Bug 883281 - Add ability to skip tests based on device, r=davehunt

testing/marionette/client/marionette/b2g_update_test.py

@@ -70,7 +70,7 @@ class B2GUpdateTestRunner(MarionetteTestRunner):
         self.b2g = B2GInstance(homedir=kwargs.get('homedir'))
         self.update_tools = self.b2g.import_update_tools()
         self.adb = self.update_tools.AdbTool(path=self.b2g.adb_path,
-                                             device=self.device)
+                                             device=self.device_serial)
 
     def match(self, filename):
         return self.match_re.match(filename) is not None

testing/marionette/client/marionette/runtests.py

@@ -178,8 +178,8 @@ class MarionetteTestRunner(object):
                  es_server=None, rest_server=None, logger=None,
                  testgroup="marionette", noWindow=False, logcat_dir=None,
                  xml_output=None, repeat=0, gecko_path=None, testvars=None,
-                 tree=None, type=None, device=None, symbols_path=None, timeout=None,
-                 **kwargs):
+                 tree=None, type=None, device_serial=None, symbols_path=None,
+                 timeout=None, **kwargs):
         self.address = address
         self.emulator = emulator
         self.emulatorBinary = emulatorBinary
@@ -207,9 +207,12 @@ class MarionetteTestRunner(object):
         self.test_kwargs = kwargs
         self.tree = tree
         self.type = type
-        self.device = device
+        self.device_serial = device_serial
         self.symbols_path = symbols_path
         self.timeout = timeout
+        self._device = None
+        self._capabilities = None
+        self._appName = None
 
         if testvars:
             if not os.path.exists(testvars):
@@ -238,6 +241,32 @@ class MarionetteTestRunner(object):
         self.testvars['xml_output'] = self.xml_output
         self.results = []
 
+    @property
+    def capabilities(self):
+        if self._capabilities:
+            return self._capabilities
+
+        self.marionette.start_session()
+        self._capabilities = self.marionette.session_capabilities
+        self.marionette.delete_session()
+        return self._capabilities
+
+    @property
+    def device(self):
+        if self._device:
+            return self._device
+
+        self._device = self.capabilities.get('device')
+        return self._device
+
+    @property
+    def appName(self):
+        if self._appName:
+            return self._appName
+
+        self._appName = self.capabilities.get('browserName')
+        return self._appName
+
     def reset_test_stats(self):
         self.passed = 0
         self.failed = 0
@@ -271,11 +300,12 @@ class MarionetteTestRunner(object):
         elif self.address:
             host, port = self.address.split(':')
             try:
-		#establish a telnet connection so we can vertify the data come back
-		tlconnection = Telnet(host, port)
-	    except:
-		raise Exception("could not connect to given marionette host/port")
-	    if self.emulator:
+                #establish a telnet connection so we can vertify the data come back
+                tlconnection = Telnet(host, port)
+            except:
+                raise Exception("could not connect to given marionette host/port")
+
+            if self.emulator:
                 self.marionette = Marionette.getMarionetteOrExit(
                                              host=host, port=int(port),
                                              connectToRunningEmulator=True,
@@ -421,7 +451,18 @@ class MarionetteTestRunner(object):
             manifest = TestManifest()
             manifest.read(filepath)
 
-            manifest_tests = manifest.active_tests(disabled=False)
+            all_tests = manifest.active_tests(disabled=False)
+            manifest_tests = manifest.active_tests(disabled=False,
+                                                   device=self.device,
+                                                   app=self.appName)
+            skip_tests = list(set([x['path'] for x in all_tests]) -
+                              set([x['path'] for x in manifest_tests]))
+            for skipped in skip_tests:
+                self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
+                                 (os.path.basename(skipped),
+                                  self.device,
+                                  self.appName))
+                self.todo += 1
 
             for i in manifest.get(tests=manifest_tests, **testargs):
                 self.run_test(i["path"])
@@ -590,7 +631,7 @@ class MarionetteTestOptions(OptionParser):
                         action='store',
                         help='host:port of running Gecko instance to connect to')
         self.add_option('--device',
-                        dest='device',
+                        dest='device_serial',
                         action='store',
                         help='serial ID of a device to use for adb / fastboot')
         self.add_option('--type',

testing/marionette/marionette-server.js

@@ -42,6 +42,8 @@ loader.loadSubScript("resource://gre/modules/devtools/DevToolsUtils.js");
 loader.loadSubScript("resource://gre/modules/devtools/server/transport.js");
 
 let bypassOffline = false;
+let qemu = "0";
+let device = null;
 
 try {
   XPCOMUtils.defineLazyGetter(this, "libcutils", function () {
@@ -49,11 +51,11 @@ try {
     return libcutils;
   });
   if (libcutils) {
-    let qemu = libcutils.property_get("ro.kernel.qemu");
+    qemu = libcutils.property_get("ro.kernel.qemu");
     logger.info("B2G emulator: " + (qemu == "1" ? "yes" : "no"));
-    let platform = libcutils.property_get("ro.product.device");
-    logger.info("Platform detected is " + platform);
-    bypassOffline = (qemu == "1" || platform == "panda");
+    device = libcutils.property_get("ro.product.device");
+    logger.info("Device detected is " + device);
+    bypassOffline = (qemu == "1" || device == "panda");
   }
 }
 catch(e) {}
@@ -577,6 +579,7 @@ MarionetteServerConnection.prototype = {
           'javascriptEnabled': true,
           'nativeEvents': false,
           'platform': Services.appinfo.OS,
+          'device': qemu == "1" ? "qemu" : (!device ? "desktop" : device),
           'rotatable': rotatable,
           'takesScreenshot': false,
           'version': Services.appinfo.version