Back out cd695cdb3b4f (bug 679759) because of test failures

build/automation.py.in

@@ -52,6 +52,7 @@ import sys
 import threading
 import tempfile
 import sqlite3
+import zipfile
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 sys.path.insert(0, SCRIPT_DIR)
@@ -97,6 +98,69 @@ _log.setLevel(logging.INFO)
 _log.addHandler(handler)
 
 
+class ZipFileReader(object):
+  """
+  Class to read zip files in Python 2.5 and later. Limited to only what we
+  actually use.
+  """
+
+  def __init__(self, filename):
+    self._zipfile = zipfile.ZipFile(filename, "r")
+
+  def __del__(self):
+    self._zipfile.close()
+
+  def _getnormalizedpath(self, path):
+    """
+    Gets a normalized path from 'path' (or the current working directory if
+    'path' is None). Also asserts that the path exists.
+    """
+    if path is None:
+      path = os.curdir
+    path = os.path.normpath(os.path.expanduser(path))
+    assert os.path.isdir(path)
+    return path
+
+  def _extractname(self, name, path):
+    """
+    Extracts a file with the given name from the zip file to the given path.
+    Also creates any directories needed along the way.
+    """
+    filename = os.path.normpath(os.path.join(path, name))
+    if name.endswith("/"):
+      os.makedirs(filename)
+    else:
+      path = os.path.split(filename)[0]
+      if not os.path.isdir(path):
+        os.makedirs(path)
+      with open(filename, "wb") as dest:
+        dest.write(self._zipfile.read(name))
+
+  def namelist(self):
+    return self._zipfile.namelist()
+
+  def read(self, name):
+    return self._zipfile.read(name)
+
+  def extract(self, name, path = None):
+    if hasattr(self._zipfile, "extract"):
+      return self._zipfile.extract(name, path)
+
+    # This will throw if name is not part of the zip file.
+    self._zipfile.getinfo(name)
+
+    self._extractname(name, self._getnormalizedpath(path))
+
+  def extractall(self, path = None):
+    if hasattr(self._zipfile, "extractall"):
+      return self._zipfile.extractall(path)
+
+    path = self._getnormalizedpath(path)
+
+    for name in self._zipfile.namelist():
+      self._extractname(name, path)
+
+
 #################
 # PROFILE SETUP #
 #################
@@ -988,7 +1052,7 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t
       os.makedirs(extensionsRootDir)
 
     if os.path.isfile(extensionSource):
-      reader = automationutils.ZipFileReader(extensionSource)
+      reader = ZipFileReader(extensionSource)
 
       for filename in reader.namelist():
         # Sanity check the zip file.

build/automationutils.py

@@ -36,12 +36,11 @@
 #
 # ***** END LICENSE BLOCK ***** */
 
-import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile
+import glob, logging, os, platform, shutil, subprocess, sys
 import re
 from urlparse import urlparse
 
 __all__ = [
-  "ZipFileReader",
   "addCommonOptions",
   "checkForCrashes",
   "dumpLeakLog",
@@ -71,68 +70,6 @@ DEBUGGER_INFO = {
   }
 }
 
-class ZipFileReader(object):
-  """
-  Class to read zip files in Python 2.5 and later. Limited to only what we
-  actually use.
-  """
-
-  def __init__(self, filename):
-    self._zipfile = zipfile.ZipFile(filename, "r")
-
-  def __del__(self):
-    self._zipfile.close()
-
-  def _getnormalizedpath(self, path):
-    """
-    Gets a normalized path from 'path' (or the current working directory if
-    'path' is None). Also asserts that the path exists.
-    """
-    if path is None:
-      path = os.curdir
-    path = os.path.normpath(os.path.expanduser(path))
-    assert os.path.isdir(path)
-    return path
-
-  def _extractname(self, name, path):
-    """
-    Extracts a file with the given name from the zip file to the given path.
-    Also creates any directories needed along the way.
-    """
-    filename = os.path.normpath(os.path.join(path, name))
-    if name.endswith("/"):
-      os.makedirs(filename)
-    else:
-      path = os.path.split(filename)[0]
-      if not os.path.isdir(path):
-        os.makedirs(path)
-      with open(filename, "wb") as dest:
-        dest.write(self._zipfile.read(name))
-
-  def namelist(self):
-    return self._zipfile.namelist()
-
-  def read(self, name):
-    return self._zipfile.read(name)
-
-  def extract(self, name, path = None):
-    if hasattr(self._zipfile, "extract"):
-      return self._zipfile.extract(name, path)
-
-    # This will throw if name is not part of the zip file.
-    self._zipfile.getinfo(name)
-
-    self._extractname(name, self._getnormalizedpath(path))
-
-  def extractall(self, path = None):
-    if hasattr(self._zipfile, "extractall"):
-      return self._zipfile.extractall(path)
-
-    path = self._getnormalizedpath(path)
-
-    for name in self._zipfile.namelist():
-      self._extractname(name, path)
-
 log = logging.getLogger()
 
 def isURL(thing):
@@ -165,6 +102,7 @@ def addCommonOptions(parser, defaults={}):
 
 def checkForCrashes(dumpDir, symbolsPath, testName=None):
   stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
+  stackwalkCGI = os.environ.get('MINIDUMP_STACKWALK_CGI', None)
   # try to get the caller's filename if no test name is given
   if testName is None:
     try:
@@ -172,70 +110,70 @@ def checkForCrashes(dumpDir, symbolsPath, testName=None):
     except:
       testName = "unknown"
 
-  # Check preconditions
-  dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
-  if len(dumps) == 0:
-    return False
-
   foundCrash = False
-  removeSymbolsPath = False
+  dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
-
+  for d in dumps:
-  # If our symbols are at a remote URL, download them now
+    log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
-  if isURL(symbolsPath):
+    print "Crash dump filename: " + d
-    print "Downloading symbols from: " + symbolsPath
+    if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
-    removeSymbolsPath = True
+      p = subprocess.Popen([stackwalkPath, d, symbolsPath],
-    # Get the symbols and write them to a temporary zipfile
+                           stdout=subprocess.PIPE,
-    data = urllib2.urlopen(symbolsPath)
+                           stderr=subprocess.PIPE)
-    symbolsFile = tempfile.TemporaryFile()
+      (out, err) = p.communicate()
-    symbolsFile.write(data.read())
+      if len(out) > 3:
-    # extract symbols to a temporary directory (which we'll delete after
+        # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
-    # processing all crashes)
+        print out
-    symbolsPath = tempfile.mkdtemp()
+      else:
-    zfile = ZipFileReader(symbolsFile)
+        print "stderr from minidump_stackwalk:"
-    zfile.extractall(symbolsPath)
+        print err
-
+      if p.returncode != 0:
-  try:
+        print "minidump_stackwalk exited with return code %d" % p.returncode
-    for d in dumps:
+    elif stackwalkCGI and symbolsPath and isURL(symbolsPath):
-      log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
+      f = None
-      print "Crash dump filename: " + d
+      try:
-      if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
+        f = open(d, "rb")
-        # run minidump stackwalk
+        sys.path.append(os.path.join(os.path.dirname(__file__), "poster.zip"))
-        p = subprocess.Popen([stackwalkPath, d, symbolsPath],
+        from poster.encode import multipart_encode
-                             stdout=subprocess.PIPE,
+        from poster.streaminghttp import register_openers
-                             stderr=subprocess.PIPE)
+        import urllib2
-        (out, err) = p.communicate()
+        register_openers()
-        if len(out) > 3:
+        datagen, headers = multipart_encode({"minidump": f,
-          # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
+                                             "symbols": symbolsPath})
-          print out
+        request = urllib2.Request(stackwalkCGI, datagen, headers)
+        result = urllib2.urlopen(request).read()
+        if len(result) > 3:
+          print result
         else:
-          print "stderr from minidump_stackwalk:"
+          print "stackwalkCGI returned nothing."
-          print err
+      finally:
-        if p.returncode != 0:
+        if f:
-          print "minidump_stackwalk exited with return code %d" % p.returncode
+          f.close()
+    else:
+      if not symbolsPath:
+        print "No symbols path given, can't process dump."
+      if not stackwalkPath and not stackwalkCGI:
+        print "Neither MINIDUMP_STACKWALK nor MINIDUMP_STACKWALK_CGI is set, can't process dump."
       else:
-        if not symbolsPath:
+        if stackwalkPath and not os.path.exists(stackwalkPath):
-          print "No symbols path given, can't process dump."
-        if not stackwalkPath:
-          print "MINIDUMP_STACKWALK not set, can't process dump."
-        elif stackwalkPath and not os.path.exists(stackwalkPath):
           print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath
-      dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
+        elif stackwalkCGI and not isURL(stackwalkCGI):
-      if dumpSavePath:
+          print "MINIDUMP_STACKWALK_CGI is not a URL: %s" % stackwalkCGI
-        shutil.move(d, dumpSavePath)
+        elif symbolsPath and not isURL(symbolsPath):
-        print "Saved dump as %s" % os.path.join(dumpSavePath,
+          print "symbolsPath is not a URL: %s" % symbolsPath
-                                                os.path.basename(d))
+    dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
-      else:
+    if dumpSavePath:
-        os.remove(d)
+      shutil.move(d, dumpSavePath)
-      extra = os.path.splitext(d)[0] + ".extra"
+      print "Saved dump as %s" % os.path.join(dumpSavePath,
-      if os.path.exists(extra):
+                                              os.path.basename(d))
-        os.remove(extra)
+    else:
-      foundCrash = True
+      os.remove(d)
-  finally:
+    extra = os.path.splitext(d)[0] + ".extra"
-    if removeSymbolsPath:
+    if os.path.exists(extra):
-      shutil.rmtree(symbolsPath)
+      os.remove(extra)
+    foundCrash = True
 
   return foundCrash
-
+  
 def getFullPath(directory, path):
   "Get an absolute path relative to 'directory'."
   return os.path.normpath(os.path.join(directory, os.path.expanduser(path)))

layout/tools/reftest/Makefile.in

@@ -84,6 +84,7 @@ _HARNESS_FILES = \
   $(topsrcdir)/build/mobile/devicemanagerADB.py \
   $(topsrcdir)/build/mobile/devicemanagerSUT.py \
   $(topsrcdir)/build/automationutils.py \
+  $(topsrcdir)/build/poster.zip \
   $(topsrcdir)/build/mobile/remoteautomation.py \
   $(topsrcdir)/testing/mochitest/server.js \
   $(topsrcdir)/build/pgo/server-locations.txt \

testing/mochitest/Makefile.in

@@ -82,6 +82,7 @@ _SERV_FILES = 	\
 		$(topsrcdir)/build/mobile/devicemanagerADB.py \
 		$(topsrcdir)/build/mobile/devicemanagerSUT.py \
 		$(topsrcdir)/build/automationutils.py \
+		$(topsrcdir)/build/poster.zip \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
 		gen_template.pl \
 		server.js \

testing/xpcshell/Makefile.in

@@ -65,6 +65,7 @@ EXTRA_BUILD_FILES := \
   automationutils.py \
   manifestparser.py \
   mozinfo.py \
+  poster.zip \
   $(NULL)
 
 # And files for running xpcshell remotely from $(topsrcdir)/build/mobile