Bug 1179981 - A new test harness for robocop; r=jmaher

build/mobile/remoteautomation.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import datetime
 import glob
 import time
 import re
@@ -137,8 +138,11 @@ class RemoteAutomation(Automation):
         if self._devicemanager.fileExists(traces):
             try:
                 t = self._devicemanager.pullFile(traces)
-                print "Contents of %s:" % traces
+                if t:
-                print t
+                    stripped = t.strip()
+                    if len(stripped) > 0:
+                        print "Contents of %s:" % traces
+                        print t
                 # Once reported, delete traces
                 self.deleteANRs()
             except DMError:
@@ -281,7 +285,6 @@ class RemoteAutomation(Automation):
             self.procName = cmd[0].split('/')[-1]
             if cmd[0] == 'am' and cmd[1] in RemoteAutomation._specialAmCommands:
                 self.procName = app
-                print "Robocop process name: "+self.procName
 
             # Setting timeout at 1 hour since on a remote device this takes much longer.
             # Temporarily increased to 75 minutes because no more chunks can be created.
@@ -360,19 +363,22 @@ class RemoteAutomation(Automation):
         def wait(self, timeout = None, noOutputTimeout = None):
             timer = 0
             noOutputTimer = 0
-            interval = 20
+            interval = 10
-
             if timeout == None:
                 timeout = self.timeout
-
             status = 0
-            while (self.dm.getTopActivity() == self.procName):
+            top = self.procName
-                # retrieve log updates every 60 seconds
+            slowLog = False
-                if timer % 60 == 0:
+            while (top == self.procName):
+                # Get log updates on each interval, but if it is taking
+                # too long, only do it every 60 seconds
+                if (not slowLog) or (timer % 60 == 0):
+                    startRead = datetime.datetime.now()
                     messages = self.read_stdout()
+                    if (datetime.datetime.now() - startRead) > datetime.timedelta(seconds=5):
+                        slowLog = True
                     if messages:
                         noOutputTimer = 0
-
                 time.sleep(interval)
                 timer += interval
                 noOutputTimer += interval
@@ -382,10 +388,9 @@ class RemoteAutomation(Automation):
                 if (noOutputTimeout and noOutputTimer > noOutputTimeout):
                     status = 2
                     break
-
+                top = self.dm.getTopActivity()
             # Flush anything added to stdout during the sleep
             self.read_stdout()
-
             return status
 
         def kill(self, stagedShutdown = False):

mobile/android/tests/browser/robocop/BaseTest.java

@@ -134,7 +134,7 @@ abstract class BaseTest extends BaseRobocopTest {
         } catch (Throwable t) {
             // save screenshot -- written to /mnt/sdcard/Robotium-Screenshots
             // as <filename>.jpg
-            mSolo.takeScreenshot("robocop-screenshot");
+            mSolo.takeScreenshot("robocop-screenshot-"+getClass().getName());
             if (mAsserter != null) {
                 mAsserter.dumpLog("Exception caught during test!", t);
                 mAsserter.ok(false, "Exception caught", t.toString());

mobile/android/tests/browser/robocop/UITest.java

@@ -65,7 +65,7 @@ abstract class UITest extends BaseRobocopTest
         } catch (Throwable t) {
             // save screenshot -- written to /mnt/sdcard/Robotium-Screenshots
             // as <filename>.jpg
-            mSolo.takeScreenshot("robocop-screenshot");
+            mSolo.takeScreenshot("robocop-screenshot-"+getClass().getName());
             if (mAsserter != null) {
                 mAsserter.dumpLog("Exception caught during test!", t);
                 mAsserter.ok(false, "Exception caught", t.toString());

testing/config/mozharness/android_arm_4_3_config.py

@@ -28,7 +28,7 @@ config = {
             ],
         },
         "robocop": {
-            "run_filename": "runtestsremote.py",
+            "run_filename": "runrobocop.py",
             "testsdir": "mochitest",
             "options": ["--dm_trans=adb", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
@@ -37,7 +37,6 @@ config = {
                 "--quiet", "--log-raw=%(raw_log_file)s",
                 "--total-chunks=4",
                 "--robocop-apk=../../robocop.apk",
-                "--robocop-ids=fennec_ids.txt",
                 "--robocop-ini=robocop.ini",
             ],
         },

testing/config/mozharness/android_arm_config.py

@@ -30,7 +30,7 @@ config = {
             ],
         },
         "robocop": {
-            "run_filename": "runtestsremote.py",
+            "run_filename": "runrobocop.py",
             "testsdir": "mochitest",
             "options": ["--dm_trans=sut", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
@@ -40,7 +40,6 @@ config = {
                 "--quiet", "--log-raw=%(raw_log_file)s",
                 "--total-chunks=4",
                 "--robocop-apk=../../robocop.apk",
-                "--robocop-ids=fennec_ids.txt",
                 "--robocop-ini=robocop.ini",
             ],
         },

testing/config/mozharness/android_panda_config.py

@@ -115,7 +115,7 @@ config = {
                 "--symbols-path=%(symbols_path)s",
                 "--robocop-ini=mochitest/robocop.ini"
             ],
-            "run_filename": "runtestsremote.py",
+            "run_filename": "runrobocop.py",
             "testsdir": "mochitest"
         },
         "xpcshell": {

testing/mochitest/mach_commands.py

@@ -374,6 +374,26 @@ class MochitestRunner(MozbuildObject):
 
         return runtestsremote.run_test_harness(options)
 
+    def run_robocop_test(self, context, tests, suite=None, **kwargs):
+        host_ret = verify_host_bin()
+        if host_ret != 0:
+            return host_ret
+
+        import imp
+        path = os.path.join(self.mochitest_dir, 'runrobocop.py')
+        with open(path, 'r') as fh:
+            imp.load_module('runrobocop', fh, path,
+                            ('.py', 'r', imp.PY_SOURCE))
+        import runrobocop
+
+        options = Namespace(**kwargs)
+
+        from manifestparser import TestManifest
+        manifest = TestManifest()
+        manifest.tests.extend(tests)
+        options.manifestFile = manifest
+
+        return runrobocop.run_test_harness(options)
 
 # parser
 
@@ -604,7 +624,7 @@ class RobocopCommands(MachCommandBase):
             flavor='instrumentation', subsuite='robocop'))
 
         mochitest = self._spawn(MochitestRunner)
-        return mochitest.run_android_test(self._mach_context, tests, 'robocop', **kwargs)
+        return mochitest.run_robocop_test(self._mach_context, tests, 'robocop', **kwargs)
 
 
 def REMOVED(cls):

testing/mochitest/mochitest_options.py

@@ -1016,12 +1016,6 @@ class AndroidArguments(ArgumentContainer):
           "default": "",
           "help": "name of the Robocop APK to use for ADB test running",
           }],
-        [["--robocop-ids"],
-         {"dest": "robocopIds",
-          "default": "",
-          "help": "name of the file containing the view ID map \
-                   (fennec_ids.txt)",
-          }],
         [["--remoteTestRoot"],
          {"dest": "remoteTestRoot",
           "default": None,
@@ -1130,13 +1124,6 @@ class AndroidArguments(ArgumentContainer):
                     options.robocopApk)
             options.robocopApk = os.path.abspath(options.robocopApk)
 
-        if options.robocopIds != "":
-            if not os.path.exists(options.robocopIds):
-                parser.error(
-                    "Unable to find specified robocop IDs file '%s'" %
-                    options.robocopIds)
-            options.robocopIds = os.path.abspath(options.robocopIds)
-
         # allow us to keep original application around for cleanup while
         # running robocop via 'am'
         options.remoteappname = options.app

testing/mochitest/moz.build

@@ -63,6 +63,7 @@ TEST_HARNESS_FILES.testing.mochitest += [
     'nested_setup.js',
     'pywebsocket_wrapper.py',
     'redirect.html',
+    'runrobocop.py',
     'runtests.py',
     'runtestsb2g.py',
     'runtestsremote.py',

testing/mochitest/runrobocop.py

@@ -0,0 +1,554 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import shutil
+import sys
+import tempfile
+import traceback
+
+sys.path.insert(
+    0, os.path.abspath(
+        os.path.realpath(
+            os.path.dirname(__file__))))
+
+from automation import Automation
+from remoteautomation import RemoteAutomation, fennecLogcatFilters
+from runtests import Mochitest, MessageLogger
+from mochitest_options import MochitestArgumentParser
+
+from manifestparser import TestManifest
+from manifestparser.filters import chunk_by_slice
+import devicemanager
+import mozinfo
+
+SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+
+
+class RobocopTestRunner(Mochitest):
+    """
+       A test harness for Robocop. Robocop tests are UI tests for Firefox for Android,
+       based on the Robotium test framework. This harness leverages some functionality
+       from mochitest, for convenience.
+    """
+    auto = None
+    dm = None
+    # Some robocop tests run for >60 seconds without generating any output.
+    NO_OUTPUT_TIMEOUT = 180
+
+    def __init__(self, automation, devmgr, options):
+        """
+           Simple one-time initialization.
+        """
+        Mochitest.__init__(self, options)
+
+        self.auto = automation
+        self.dm = devmgr
+        self.dm.default_timeout = 320
+        self.options = options
+        self.options.logFile = "robocop.log"
+        self.environment = self.auto.environment
+        self.deviceRoot = self.dm.getDeviceRoot()
+        self.remoteProfile = options.remoteTestRoot + "/profile"
+        self.remoteProfileCopy = options.remoteTestRoot + "/profile-copy"
+        self.auto.setRemoteProfile(self.remoteProfile)
+        self.remoteConfigFile = os.path.join(
+            self.deviceRoot, "robotium.config")
+        self.remoteLog = options.remoteLogFile
+        self.auto.setRemoteLog(self.remoteLog)
+        self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots"
+        self.remoteNSPR = os.path.join(options.remoteTestRoot, "nspr")
+        self.auto.setServerInfo(
+            self.options.webServer, self.options.httpPort, self.options.sslPort)
+        self.localLog = options.logFile
+        self.localProfile = None
+        productPieces = self.options.remoteProductName.split('.')
+        if (productPieces is not None):
+            self.auto.setProduct(productPieces[0])
+        else:
+            self.auto.setProduct(self.options.remoteProductName)
+        self.auto.setAppName(self.options.remoteappname)
+        self.certdbNew = True
+        self.remoteCopyAvailable = True
+        self.passed = 0
+        self.failed = 0
+        self.todo = 0
+
+    def startup(self):
+        """
+           Second-stage initialization: One-time initialization which may require cleanup.
+        """
+        # Despite our efforts to clean up servers started by this script, in practice
+        # we still see infrequent cases where a process is orphaned and interferes
+        # with future tests, typically because the old server is keeping the port in use.
+        # Try to avoid those failures by checking for and killing orphan servers before
+        # trying to start new ones.
+        self.killNamedOrphans('ssltunnel')
+        self.killNamedOrphans('xpcshell')
+        self.auto.deleteANRs()
+        self.auto.deleteTombstones()
+        self.dm.killProcess(self.options.app.split('/')[-1])
+        self.dm.removeDir(self.remoteScreenshots)
+        self.dm.removeDir(self.remoteNSPR)
+        self.dm.mkDir(self.remoteNSPR)
+        self.dm.mkDir(os.path.dirname(self.options.remoteLogFile))
+        # Add Android version (SDK level) to mozinfo so that manifest entries
+        # can be conditional on android_version.
+        androidVersion = self.dm.shellCheckOutput(
+            ['getprop', 'ro.build.version.sdk'])
+        self.log.info(
+            "Android sdk version '%s'; will use this to filter manifests" %
+            str(androidVersion))
+        mozinfo.info['android_version'] = androidVersion
+        if (self.options.dm_trans == 'adb' and self.options.robocopApk):
+            self.dm._checkCmd(["install", "-r", self.options.robocopApk])
+            self.log.debug("Robocop APK %s installed" %
+                           self.options.robocopApk)
+        # Display remote diagnostics; if running in mach, keep output terse.
+        if self.options.log_mach is None:
+            self.printDeviceInfo()
+        self.setupLocalPaths()
+        self.buildProfile()
+        # ignoreSSLTunnelExts is a workaround for bug 1109310
+        self.startServers(
+            self.options,
+            debuggerInfo=None,
+            ignoreSSLTunnelExts=True)
+        self.log.debug("Servers started")
+
+    def cleanup(self):
+        """
+           Cleanup at end of job run.
+        """
+        self.log.debug("Cleaning up...")
+        self.stopServers()
+        self.dm.killProcess(self.options.app.split('/')[-1])
+        blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
+        if blobberUploadDir:
+            self.log.debug("Pulling any remote nspr logs and screenshots to %s." %
+                           blobberUploadDir)
+            self.dm.getDirectory(self.remoteNSPR, blobberUploadDir)
+            self.dm.getDirectory(self.remoteScreenshots, blobberUploadDir)
+        Mochitest.cleanup(self, self.options)
+        if self.localProfile:
+            os.system("rm -Rf %s" % self.localProfile)
+        self.dm.removeDir(self.remoteProfile)
+        self.dm.removeDir(self.remoteProfileCopy)
+        self.dm.removeDir(self.remoteScreenshots)
+        self.dm.removeDir(self.remoteNSPR)
+        self.dm.removeFile(self.remoteConfigFile)
+        if self.dm.fileExists(self.remoteLog):
+            self.dm.removeFile(self.remoteLog)
+        self.log.debug("Cleanup complete.")
+
+    def findPath(self, paths, filename=None):
+        for path in paths:
+            p = path
+            if filename:
+                p = os.path.join(p, filename)
+            if os.path.exists(self.getFullPath(p)):
+                return path
+        return None
+
+    def makeLocalAutomation(self):
+        localAutomation = Automation()
+        localAutomation.IS_WIN32 = False
+        localAutomation.IS_LINUX = False
+        localAutomation.IS_MAC = False
+        localAutomation.UNIXISH = False
+        hostos = sys.platform
+        if (hostos == 'mac' or hostos == 'darwin'):
+            localAutomation.IS_MAC = True
+        elif (hostos == 'linux' or hostos == 'linux2'):
+            localAutomation.IS_LINUX = True
+            localAutomation.UNIXISH = True
+        elif (hostos == 'win32' or hostos == 'win64'):
+            localAutomation.BIN_SUFFIX = ".exe"
+            localAutomation.IS_WIN32 = True
+        return localAutomation
+
+    def setupLocalPaths(self):
+        """
+           Setup xrePath and utilityPath and verify xpcshell.
+
+           This is similar to switchToLocalPaths in runtestsremote.py.
+        """
+        localAutomation = self.makeLocalAutomation()
+        paths = [
+            self.options.xrePath,
+            localAutomation.DIST_BIN,
+            self.auto._product,
+            os.path.join('..', self.auto._product)
+        ]
+        self.options.xrePath = self.findPath(paths)
+        if self.options.xrePath is None:
+            self.log.error(
+                "unable to find xulrunner path for %s, please specify with --xre-path" %
+                os.name)
+            sys.exit(1)
+        self.log.debug("using xre path %s" % self.options.xrePath)
+        xpcshell = "xpcshell"
+        if (os.name == "nt"):
+            xpcshell += ".exe"
+        if self.options.utilityPath:
+            paths = [self.options.utilityPath, self.options.xrePath]
+        else:
+            paths = [self.options.xrePath]
+        self.options.utilityPath = self.findPath(paths, xpcshell)
+        if self.options.utilityPath is None:
+            self.log.error(
+                "unable to find utility path for %s, please specify with --utility-path" %
+                os.name)
+            sys.exit(1)
+        self.log.debug("using utility path %s" % self.options.utilityPath)
+        xpcshell_path = os.path.join(self.options.utilityPath, xpcshell)
+        if localAutomation.elf_arm(xpcshell_path):
+            self.log.error('xpcshell at %s is an ARM binary; please use '
+                           'the --utility-path argument to specify the path '
+                           'to a desktop version.' % xpcshell_path)
+            sys.exit(1)
+        self.log.debug("xpcshell found at %s" % xpcshell_path)
+
+    def buildProfile(self):
+        """
+           Build a profile locally, keep it locally for use by servers and
+           push a copy to the remote profile-copy directory.
+
+           This is similar to buildProfile in runtestsremote.py.
+        """
+        self.options.extraPrefs.append('browser.search.suggest.enabled=true')
+        self.options.extraPrefs.append('browser.search.suggest.prompted=true')
+        self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
+        self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
+        self.options.extraPrefs.append('browser.snippets.enabled=false')
+        self.options.extraPrefs.append('browser.casting.enabled=true')
+        self.options.extraPrefs.append('extensions.autoupdate.enabled=false')
+        manifest = Mochitest.buildProfile(self, self.options)
+        self.localProfile = self.options.profilePath
+        self.log.debug("Profile created at %s" % self.localProfile)
+        # some files are not needed for robocop; save time by not pushing
+        shutil.rmtree(os.path.join(self.localProfile, 'webapps'))
+        desktop_extensions = ['mochikit@mozilla.org', 'worker-test@mozilla.org', 'workerbootstrap-test@mozilla.org']
+        for ext in desktop_extensions:
+            shutil.rmtree(os.path.join(self.localProfile, 'extensions', 'staged', ext))
+        os.remove(os.path.join(self.localProfile, 'userChrome.css'))
+        try:
+            self.dm.pushDir(self.localProfile, self.remoteProfileCopy)
+        except devicemanager.DMError:
+            self.log.error(
+                "Automation Error: Unable to copy profile to device.")
+            raise
+
+        return manifest
+
+    def setupRemoteProfile(self):
+        """
+           Remove any remote profile and re-create it.
+        """
+        self.log.debug("Updating remote profile at %s" % self.remoteProfile)
+        self.dm.removeDir(self.remoteProfile)
+        if self.remoteCopyAvailable:
+            try:
+                self.dm.shellCheckOutput(
+                    ['cp', '-r', self.remoteProfileCopy, self.remoteProfile],
+                    root=True, timeout=60)
+            except devicemanager.DMError:
+                # For instance, cp is not available on some older versions of
+                # Android.
+                self.log.info(
+                    "Unable to copy remote profile; falling back to push.")
+                self.remoteCopyAvailable = False
+        if not self.remoteCopyAvailable:
+            self.dm.pushDir(self.localProfile, self.remoteProfile)
+
+    def parseLocalLog(self):
+        """
+           Read and parse the local log file, noting any failures.
+        """
+        with open(self.localLog) as currentLog:
+            data = currentLog.readlines()
+        os.unlink(self.localLog)
+        start_found = False
+        end_found = False
+        fail_found = False
+        for line in data:
+            try:
+                message = json.loads(line)
+                if not isinstance(message, dict) or 'action' not in message:
+                    continue
+            except ValueError:
+                continue
+            if message['action'] == 'test_end':
+                end_found = True
+                start_found = False
+                break
+            if start_found and not end_found:
+                if 'status' in message:
+                    if 'expected' in message:
+                        self.failed += 1
+                    elif message['status'] == 'PASS':
+                        self.passed += 1
+                    elif message['status'] == 'FAIL':
+                        self.todo += 1
+            if message['action'] == 'test_start':
+                start_found = True
+            if 'expected' in message:
+                fail_found = True
+        result = 0
+        if fail_found:
+            result = 1
+        if not end_found:
+            self.log.info(
+                "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)")
+            result = 1
+        return result
+
+    def logTestSummary(self):
+        """
+           Print a summary of all tests run to stdout, for treeherder parsing 
+           (logging via self.log does not work here).
+        """
+        print("0 INFO TEST-START | Shutdown")
+        print("1 INFO Passed: %s" % (self.passed))
+        print("2 INFO Failed: %s" % (self.failed))
+        print("3 INFO Todo: %s" % (self.todo))
+        print("4 INFO SimpleTest FINISHED")
+        if self.failed > 0:
+            return 1
+        return 0
+
+    def printDeviceInfo(self, printLogcat=False):
+        """
+           Log remote device information and logcat (if requested).
+
+           This is similar to printDeviceInfo in runtestsremote.py
+        """
+        try:
+            if printLogcat:
+                logcat = self.dm.getLogcat(
+                    filterOutRegexps=fennecLogcatFilters)
+                self.log.info(
+                    '\n' +
+                    ''.join(logcat).decode(
+                        'utf-8',
+                        'replace'))
+            self.log.info("Device info:")
+            devinfo = self.dm.getInfo()
+            for category in devinfo:
+                if type(devinfo[category]) is list:
+                    self.log.info("  %s:" % category)
+                    for item in devinfo[category]:
+                        self.log.info("     %s" % item)
+                else:
+                    self.log.info("  %s: %s" % (category, devinfo[category]))
+            self.log.info("Test root: %s" % self.dm.deviceRoot)
+        except devicemanager.DMError:
+            self.log.warning("Error getting device information")
+
+    def setupRobotiumConfig(self, browserEnv):
+        """
+           Create robotium.config and push it to the device.
+        """
+        fHandle = tempfile.NamedTemporaryFile(suffix='.config',
+                                              prefix='robotium-',
+                                              dir=os.getcwd(),
+                                              delete=False)
+        fHandle.write("profile=%s\n" % (self.remoteProfile))
+        fHandle.write("logfile=%s\n" % (self.options.remoteLogFile))
+        fHandle.write("host=http://mochi.test:8888/tests\n")
+        fHandle.write(
+            "rawhost=http://%s:%s/tests\n" %
+            (self.options.remoteWebServer, self.options.httpPort))
+        if browserEnv:
+            envstr = ""
+            delim = ""
+            for key, value in browserEnv.items():
+                try:
+                    value.index(',')
+                    self.log.error(
+                        "setupRobotiumConfig: browserEnv - Found a ',' in our value, unable to process value. key=%s,value=%s" %
+                        (key, value))
+                    self.log.error("browserEnv=%s" % browserEnv)
+                except ValueError:
+                    envstr += "%s%s=%s" % (delim, key, value)
+                    delim = ","
+            fHandle.write("envvars=%s\n" % envstr)
+        fHandle.close()
+        self.dm.removeFile(self.remoteConfigFile)
+        self.dm.pushFile(fHandle.name, self.remoteConfigFile)
+        os.unlink(fHandle.name)
+
+    def buildBrowserEnv(self):
+        """
+           Return an environment dictionary suitable for remote use.
+
+           This is similar to buildBrowserEnv in runtestsremote.py.
+        """
+        browserEnv = self.environment(
+            xrePath=None,
+            debugger=None)
+        # remove desktop environment not used on device
+        if "MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA" in browserEnv:
+            del browserEnv["MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA"]
+        if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
+            del browserEnv["XPCOM_MEM_BLOAT_LOG"]
+        browserEnv["NSPR_LOG_FILE"] = os.path.join(
+            self.remoteNSPR,
+            self.nsprLogName)
+        return browserEnv
+
+    def runSingleTest(self, test):
+        """
+           Run the specified test.
+        """
+        self.log.debug("Running test %s" % test['name'])
+        self.nsprLogName = "nspr-%s.log" % test['name']
+        browserEnv = self.buildBrowserEnv()
+        self.setupRobotiumConfig(browserEnv)
+        self.setupRemoteProfile()
+        self.options.app = "am"
+        if self.options.autorun:
+            # This launches a test (using "am instrument") and instructs
+            # Fennec to /quit/ the browser (using Robocop:Quit) and to
+            # /finish/ all opened activities.
+            browserArgs = [
+                "instrument",
+                "-w",
+                "-e", "quit_and_finish", "1",
+                "-e", "deviceroot", self.deviceRoot,
+                "-e", "class",
+                "org.mozilla.gecko.tests.%s" % test['name'].split('.java')[0],
+                "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner"]
+        else:
+            # This does not launch a test at all. It launches an activity
+            # that starts Fennec and then waits indefinitely, since cat
+            # never returns.
+            browserArgs = ["start",
+                           "-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
+                           "&&", "cat"]
+            self.dm.default_timeout = sys.maxint  # Forever.
+            self.log.info("")
+            self.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
+                         (self.options.remoteWebServer, self.options.httpPort))
+            self.log.info("")
+        result = -1
+        log_result = -1
+        try:
+            self.dm.recordLogcat()
+            result = self.auto.runApp(
+                None, browserEnv, "am", self.localProfile, browserArgs, timeout=self.NO_OUTPUT_TIMEOUT)
+            self.log.debug("runApp completes with status %d" % result)
+            if result != 0:
+                self.log.error("runApp() exited with code %s" % result)
+            if self.dm.fileExists(self.remoteLog):
+                self.dm.getFile(self.remoteLog, self.localLog)
+                self.dm.removeFile(self.remoteLog)
+                self.log.debug("Remote log %s retrieved to %s" %
+                               (self.remoteLog, self.localLog))
+            else:
+                self.log.warning(
+                    "Unable to retrieve log file (%s) from remote device" %
+                    self.remoteLog)
+            log_result = self.parseLocalLog()
+            if result != 0 or log_result != 0:
+                # Display remote diagnostics; if running in mach, keep output
+                # terse.
+                if self.options.log_mach is None:
+                    self.printDeviceInfo(printLogcat=True)
+        except:
+            self.log.error(
+                "Automation Error: Exception caught while running tests")
+            traceback.print_exc()
+            result = 1
+        self.log.debug("Test %s completes with status %d (log status %d)" %
+                       (test['name'], int(result), int(log_result)))
+        return result
+
+    def runTests(self):
+        self.startup()
+        if isinstance(self.options.manifestFile, TestManifest):
+            mp = self.options.manifestFile
+        else:
+            mp = TestManifest(strict=False)
+            mp.read(self.options.robocopIni)
+        filters = []
+        if self.options.totalChunks:
+            filters.append(
+                chunk_by_slice(self.options.thisChunk, self.options.totalChunks))
+        robocop_tests = mp.active_tests(
+            exists=False, filters=filters, **mozinfo.info)
+        if not self.options.autorun:
+            # Force a single loop iteration. The iteration will start Fennec and
+            # the httpd server, but not actually run a test.
+            self.options.test_paths = [robocop_tests[0]['name']]
+        active_tests = []
+        for test in robocop_tests:
+            if self.options.test_paths and test['name'] not in self.options.test_paths:
+                continue
+            if 'disabled' in test:
+                self.log.info('TEST-INFO | skipping %s | %s' %
+                             (test['name'], test['disabled']))
+                continue
+            active_tests.append(test)
+        self.log.suite_start([t['name'] for t in active_tests])
+        worstTestResult = None
+        for test in active_tests:
+            result = self.runSingleTest(test)
+            if worstTestResult is None or worstTestResult == 0:
+                worstTestResult = result
+        if worstTestResult is None:
+            self.log.warning(
+                "No tests run. Did you pass an invalid TEST_PATH?")
+            worstTestResult = 1
+        else:
+            print "INFO | runtests.py | Test summary: start."
+            logResult = self.logTestSummary()
+            print "INFO | runtests.py | Test summary: end."
+            if worstTestResult == 0:
+                worstTestResult = logResult
+        return worstTestResult
+
+
+def run_test_harness(options):
+    if options is None:
+        raise ValueError(
+            "Invalid options specified, use --help for a list of valid options")
+    message_logger = MessageLogger(logger=None)
+    process_args = {'messageLogger': message_logger}
+    auto = RemoteAutomation(None, "fennec", processArgs=process_args)
+    auto.setDeviceManager(options.dm)
+    runResult = -1
+    robocop = RobocopTestRunner(auto, options.dm, options)
+    try:
+        message_logger.logger = robocop.log
+        message_logger.buffering = False
+        robocop.message_logger = message_logger
+        robocop.log.debug("options=%s" % vars(options))
+        runResult = robocop.runTests()
+    except KeyboardInterrupt:
+        robocop.log.info("runrobocop.py | Received keyboard interrupt")
+        runResult = -1
+    except:
+        traceback.print_exc()
+        robocop.log.error(
+            "runrobocop.py | Received unexpected exception while running tests")
+        runResult = 1
+    finally:
+        try:
+            robocop.cleanup()
+        except devicemanager.DMError:
+            # ignore device error while cleaning up
+            pass
+        message_logger.finish()
+    return runResult
+
+
+def main(args=sys.argv[1:]):
+    parser = MochitestArgumentParser(app='android')
+    options = parser.parse_args(args)
+    return run_test_harness(options)
+
+if __name__ == "__main__":
+    sys.exit(main())

testing/mochitest/runtestsremote.py

@@ -66,10 +66,6 @@ class MochiRemote(Mochitest):
                 self.remoteLog)
         self._dm.removeDir(self.remoteProfile)
         self._dm.removeDir(self.remoteChromeTestDir)
-        # Don't leave an old robotium.config hanging around; the
-        # profile it references was just deleted!
-        deviceRoot = self._dm.getDeviceRoot()
-        self._dm.removeFile(os.path.join(deviceRoot, "robotium.config"))
         blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
         if blobberUploadDir:
             self._dm.getDirectory(self.remoteNSPR, blobberUploadDir)
@@ -178,29 +174,6 @@ class MochiRemote(Mochitest):
         self.localProfile = options.profilePath
         self._dm.removeDir(self.remoteProfile)
 
-        # we do not need this for robotium based tests, lets save a LOT of time
-        if options.robocopIni:
-            shutil.rmtree(os.path.join(options.profilePath, 'webapps'))
-            shutil.rmtree(
-                os.path.join(
-                    options.profilePath,
-                    'extensions',
-                    'staged',
-                    'mochikit@mozilla.org'))
-            shutil.rmtree(
-                os.path.join(
-                    options.profilePath,
-                    'extensions',
-                    'staged',
-                    'worker-test@mozilla.org'))
-            shutil.rmtree(
-                os.path.join(
-                    options.profilePath,
-                    'extensions',
-                    'staged',
-                    'workerbootstrap-test@mozilla.org'))
-            os.remove(os.path.join(options.profilePath, 'userChrome.css'))
-
         try:
             self._dm.pushDir(options.profilePath, self.remoteProfile)
         except devicemanager.DMError:
@@ -220,38 +193,18 @@ class MochiRemote(Mochitest):
         env["MOZ_HIDE_RESULTS_TABLE"] = "1"
         retVal = Mochitest.buildURLOptions(self, options, env)
 
-        if not options.robocopIni:
+        # we really need testConfig.js (for browser chrome)
-            # we really need testConfig.js (for browser chrome)
+        try:
-            try:
+            self._dm.pushDir(options.profilePath, self.remoteProfile)
-                self._dm.pushDir(options.profilePath, self.remoteProfile)
+        except devicemanager.DMError:
-            except devicemanager.DMError:
+            self.log.error(
-                self.log.error(
+                "Automation Error: Unable to copy profile to device.")
-                    "Automation Error: Unable to copy profile to device.")
+            raise
-                raise
 
         options.profilePath = self.remoteProfile
         options.logFile = self.localLog
         return retVal
 
-    def getTestsToRun(self, options):
-        if options.robocopIni != "":
-            # Skip over tests filtering if we run robocop tests.
-            return None
-        else:
-            return super(MochiRemote, self).getTestsToRun(options)
-
-    def buildTestPath(self, options, testsToFilter=None):
-        if options.robocopIni != "":
-            # Skip over manifest building if we just want to run
-            # robocop tests.
-            return self.buildTestURL(options)
-        else:
-            return super(
-                MochiRemote,
-                self).buildTestPath(
-                options,
-                testsToFilter)
-
     def getChromeTestDir(self, options):
         local = super(MochiRemote, self).getChromeTestDir(options)
         local = os.path.join(local, "chrome")
@@ -264,105 +217,6 @@ class MochiRemote(Mochitest):
     def getLogFilePath(self, logFile):
         return logFile
 
-    # In the future we could use LogParser:
-    # http://hg.mozilla.org/automation/logparser/
-    def addLogData(self):
-        with open(self.localLog) as currentLog:
-            data = currentLog.readlines()
-        start_found = False
-        end_found = False
-        fail_found = False
-        for line in data:
-            try:
-                message = json.loads(line)
-                if not isinstance(message, dict) or 'action' not in message:
-                    continue
-            except ValueError:
-                continue
-
-            if message['action'] == 'test_end':
-                end_found = True
-                start_found = False
-                break
-
-            if start_found and not end_found:
-                self.logMessages.append(message)
-
-            if message['action'] == 'test_start':
-                start_found = True
-            if 'expected' in message:
-                fail_found = True
-        result = 0
-        if fail_found:
-            result = 1
-        if not end_found:
-            self.log.info(
-                "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)")
-            result = 1
-        return result
-
-    def printLog(self):
-        passed = 0
-        failed = 0
-        todo = 0
-        incr = 1
-        logFile = []
-        logFile.append("0 INFO SimpleTest START")
-        for message in self.logMessages:
-            if 'status' not in message:
-                continue
-
-            if 'expected' in message:
-                failed += 1
-            elif message['status'] == 'PASS':
-                passed += 1
-            elif message['status'] == 'FAIL':
-                todo += 1
-            incr += 1
-
-        logFile.append("%s INFO TEST-START | Shutdown" % incr)
-        incr += 1
-        logFile.append("%s INFO Passed: %s" % (incr, passed))
-        incr += 1
-        logFile.append("%s INFO Failed: %s" % (incr, failed))
-        incr += 1
-        logFile.append("%s INFO Todo: %s" % (incr, todo))
-        incr += 1
-        logFile.append("%s INFO SimpleTest FINISHED" % incr)
-
-        # TODO: Consider not printing to stdout because we might be duplicating
-        # output
-        print '\n'.join(logFile)
-        with open(self.localLog, 'w') as localLog:
-            localLog.write('\n'.join(logFile))
-
-        if failed > 0:
-            return 1
-        return 0
-
-    def printScreenshots(self, screenShotDir):
-        # TODO: This can be re-written after completion of bug 749421
-        if not self._dm.dirExists(screenShotDir):
-            self.log.info(
-                "SCREENSHOT: No ScreenShots directory available: " +
-                screenShotDir)
-            return
-
-        printed = 0
-        for name in self._dm.listFiles(screenShotDir):
-            fullName = screenShotDir + "/" + name
-            self.log.info("SCREENSHOT: FOUND: [%s]" % fullName)
-            try:
-                image = self._dm.pullFile(fullName)
-                encoded = base64.b64encode(image)
-                self.log.info("SCREENSHOT: data:image/jpg;base64,%s" % encoded)
-                printed += 1
-            except:
-                self.log.info("SCREENSHOT: Could not be parsed")
-                pass
-
-        self.log.info("SCREENSHOT: TOTAL PRINTED: [%s]" % printed)
-
     def printDeviceInfo(self, printLogcat=False):
         try:
             if printLogcat:
@@ -386,44 +240,6 @@ class MochiRemote(Mochitest):
         except devicemanager.DMError:
             self.log.warning("Error getting device information")
 
-    def buildRobotiumConfig(self, options, browserEnv):
-        deviceRoot = self._dm.deviceRoot
-        fHandle = tempfile.NamedTemporaryFile(suffix='.config',
-                                              prefix='robotium-',
-                                              dir=os.getcwd(),
-                                              delete=False)
-        fHandle.write("profile=%s\n" % (self.remoteProfile))
-        fHandle.write("logfile=%s\n" % (options.remoteLogFile))
-        fHandle.write("host=http://mochi.test:8888/tests\n")
-        fHandle.write(
-            "rawhost=http://%s:%s/tests\n" %
-            (options.remoteWebServer, options.httpPort))
-
-        if browserEnv:
-            envstr = ""
-            delim = ""
-            for key, value in browserEnv.items():
-                try:
-                    value.index(',')
-                    self.log.error(
-                        "buildRobotiumConfig: browserEnv - Found a ',' in our value, unable to process value. key=%s,value=%s" %
-                        (key, value))
-                    self.log.error("browserEnv=%s" % browserEnv)
-                except ValueError:
-                    envstr += "%s%s=%s" % (delim, key, value)
-                    delim = ","
-
-            fHandle.write("envvars=%s\n" % envstr)
-        fHandle.close()
-
-        self._dm.removeFile(os.path.join(deviceRoot, "robotium.config"))
-        self._dm.pushFile(
-            fHandle.name,
-            os.path.join(
-                deviceRoot,
-                "robotium.config"))
-        os.unlink(fHandle.name)
-
     def getGMPPluginPath(self, options):
         # TODO: bug 1149374
         return None
@@ -443,7 +259,6 @@ class MochiRemote(Mochitest):
         browserEnv["NSPR_LOG_FILE"] = os.path.join(
             self.remoteNSPR,
             self.nsprLogName)
-        self.buildRobotiumConfig(options, browserEnv)
         return browserEnv
 
     def runApp(self, *args, **kwargs):
@@ -511,222 +326,22 @@ def run_test_harness(options):
     procName = options.app.split('/')[-1]
     dm.killProcess(procName)
 
-    if options.robocopIni != "":
+    mochitest.nsprLogName = "nspr.log"
-        # turning buffering off as it's not used in robocop
+    try:
-        message_logger.buffering = False
+        dm.recordLogcat()
-
+        retVal = mochitest.runTests(options)
-        # sut may wait up to 300 s for a robocop am process before returning
+    except:
-        dm.default_timeout = 320
+        log.error("Automation Error: Exception caught while running tests")
-        if isinstance(options.manifestFile, TestManifest):
+        traceback.print_exc()
-            mp = options.manifestFile
+        mochitest.stopServers()
-        else:
-            mp = TestManifest(strict=False)
-            mp.read(options.robocopIni)
-
-        filters = []
-        if options.totalChunks:
-            filters.append(
-                chunk_by_slice(options.thisChunk, options.totalChunks))
-        robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info)
-
-        options.extraPrefs.append('browser.search.suggest.enabled=true')
-        options.extraPrefs.append('browser.search.suggest.prompted=true')
-        options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
-        options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
-        options.extraPrefs.append('browser.snippets.enabled=false')
-        options.extraPrefs.append('browser.casting.enabled=true')
-        options.extraPrefs.append('extensions.autoupdate.enabled=false')
-
-        if (options.dm_trans == 'adb' and options.robocopApk):
-            dm._checkCmd(["install", "-r", options.robocopApk])
-
-        if not options.autorun:
-            # Force a single loop iteration. The iteration will start Fennec and
-            # the httpd server, but not actually run a test.
-            options.test_paths = [robocop_tests[0]['name']]
-
-        retVal = None
-        # Filtering tests
-        active_tests = []
-        for test in robocop_tests:
-            if options.test_paths and test['name'] not in options.test_paths:
-                continue
-
-            if 'disabled' in test:
-                log.info(
-                    'TEST-INFO | skipping %s | %s' %
-                    (test['name'], test['disabled']))
-                continue
-
-            active_tests.append(test)
-
-        log.suite_start([t['name'] for t in active_tests])
-
-        for test in active_tests:
-            # When running in a loop, we need to create a fresh profile for
-            # each cycle
-            if mochitest.localProfile:
-                options.profilePath = mochitest.localProfile
-                os.system("rm -Rf %s" % options.profilePath)
-                options.profilePath = None
-                mochitest.localProfile = options.profilePath
-
-            options.app = "am"
-            mochitest.nsprLogName = "nspr-%s.log" % test['name']
-            if options.autorun:
-                # This launches a test (using "am instrument") and instructs
-                # Fennec to /quit/ the browser (using Robocop:Quit) and to
-                # /finish/ all opened activities.
-                options.browserArgs = [
-                    "instrument",
-                    "-w",
-                    "-e", "quit_and_finish", "1",
-                    "-e", "deviceroot", deviceRoot,
-                    "-e",
-                    "class"]
-                options.browserArgs.append(
-                    "org.mozilla.gecko.tests.%s" %
-                    test['name'].split('.java')[0])
-                options.browserArgs.append(
-                    "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner")
-            else:
-                # This does not launch a test at all. It launches an activity
-                # that starts Fennec and then waits indefinitely, since cat
-                # never returns.
-                options.browserArgs = ["start",
-                                       "-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
-                                       "&&", "cat"]
-                dm.default_timeout = sys.maxint # Forever.
-
-                mochitest.log.info("")
-                mochitest.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
-                    (options.remoteWebServer, options.httpPort))
-                mochitest.log.info("")
-
-            # If the test is for checking the import from bookmarks then make
-            # sure there is data to import
-            if test['name'] == "testImportFromAndroid":
-
-                # Get the OS so we can run the insert in the apropriate
-                # database and following the correct table schema
-                osInfo = dm.getInfo("os")
-                devOS = " ".join(osInfo['os'])
-
-                if ("pandaboard" in devOS):
-                    delete = [
-                        'execsu',
-                        'sqlite3',
-                        "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
-                else:
-                    delete = [
-                        'execsu',
-                        'sqlite3',
-                        "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
-                if (options.dm_trans == "sut"):
-                    dm._runCmds([{"cmd": " ".join(delete)}])
-
-                # Insert the bookmarks
-                log.info(
-                    "Insert bookmarks in the default android browser database")
-                for i in range(20):
-                    if ("pandaboard" in devOS):
-                        cmd = [
-                            'execsu',
-                            'sqlite3',
-                            "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" +
-                            str(
-                                30 +
-                                i) +
-                            ",\"Bookmark" +
-                            str(i) +
-                            "\",\"http://www.bookmark" +
-                            str(i) +
-                            ".com\",0,1," +
-                            str(
-                                100 +
-                                i) +
-                            ");'"]
-                    else:
-                        cmd = [
-                            'execsu',
-                            'sqlite3',
-                            "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark" +
-                            str(i) +
-                            "\",\"http://www.bookmark" +
-                            str(i) +
-                            ".com\",1);'"]
-                    if (options.dm_trans == "sut"):
-                        dm._runCmds([{"cmd": " ".join(cmd)}])
-            try:
-                screenShotDir = "/mnt/sdcard/Robotium-Screenshots"
-                dm.removeDir(screenShotDir)
-                dm.recordLogcat()
-                result = mochitest.runTests(options)
-                if result != 0:
-                    log.error("runTests() exited with code %s" % result)
-                log_result = mochitest.addLogData()
-                if result != 0 or log_result != 0:
-                    mochitest.printDeviceInfo(printLogcat=True)
-                    mochitest.printScreenshots(screenShotDir)
-                # Ensure earlier failures aren't overwritten by success on this
-                # run
-                if retVal is None or retVal == 0:
-                    retVal = result
-            except:
-                log.error(
-                    "Automation Error: Exception caught while running tests")
-                traceback.print_exc()
-                mochitest.stopServers()
-                try:
-                    mochitest.cleanup(options)
-                except devicemanager.DMError:
-                    # device error cleaning up... oh well!
-                    pass
-                retVal = 1
-                break
-            finally:
-                # Clean-up added bookmarks
-                if test['name'] == "testImportFromAndroid":
-                    if ("pandaboard" in devOS):
-                        cmd_del = [
-                            'execsu',
-                            'sqlite3',
-                            "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
-                    else:
-                        cmd_del = [
-                            'execsu',
-                            'sqlite3',
-                            "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
-                    if (options.dm_trans == "sut"):
-                        dm._runCmds([{"cmd": " ".join(cmd_del)}])
-        if retVal is None:
-            log.warning("No tests run. Did you pass an invalid TEST_PATH?")
-            retVal = 1
-        else:
-            # if we didn't have some kind of error running the tests, make
-            # sure the tests actually passed
-            print "INFO | runtests.py | Test summary: start."
-            overallResult = mochitest.printLog()
-            print "INFO | runtests.py | Test summary: end."
-            if retVal == 0:
-                retVal = overallResult
-    else:
-        mochitest.nsprLogName = "nspr.log"
         try:
-            dm.recordLogcat()
+            mochitest.cleanup(options)
-            retVal = mochitest.runTests(options)
+        except devicemanager.DMError:
-        except:
+            # device error cleaning up... oh well!
-            log.error("Automation Error: Exception caught while running tests")
+            pass
-            traceback.print_exc()
+        retVal = 1
-            mochitest.stopServers()
-            try:
-                mochitest.cleanup(options)
-            except devicemanager.DMError:
-                # device error cleaning up... oh well!
-                pass
-            retVal = 1
 
-        mochitest.printDeviceInfo(printLogcat=True)
+    mochitest.printDeviceInfo(printLogcat=True)
 
     message_logger.finish()
 

testing/testsuite-targets.mk

@@ -60,9 +60,8 @@ RUN_MOCHITEST_REMOTE = \
 
 RUN_MOCHITEST_ROBOCOP = \
   rm -f ./$@.log && \
-  $(PYTHON) _tests/testing/mochitest/runtestsremote.py \
+  $(PYTHON) _tests/testing/mochitest/runrobocop.py \
     --robocop-apk=$(DEPTH)/build/mobile/robocop/robocop-debug.apk \
-    --robocop-ids=$(DEPTH)/mobile/android/base/fennec_ids.txt \
     --robocop-ini=_tests/testing/mochitest/robocop.ini \
     --log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \
     --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \