Bug 987360 - Add ability to tag tests with arbitrary strings and run them, r=chmanchester

Add a `tags` attribute to a test or DEFAULT section in a manifest:

[test_foo]
tags = foo

Then run all tests with a given tag by passing in `--tag foo` to a supported test harness. So far mochitest, xpcshell and marionette are supported.

--HG--
extra : rebase_source : 68a0931c6a8ee1df4f5c09d67c396490774aa856

testing/marionette/client/marionette/runner/base.py

@@ -17,6 +17,7 @@ import unittest
 import xml.dom.minidom as dom
 
 from manifestparser import TestManifest
+from manifestparser.filters import tags
 from marionette_driver.marionette import Marionette
 from mixins.b2g import B2GTestResultMixin, get_b2g_pid, get_dm
 from mozhttpd import MozHttpd
@@ -417,6 +418,12 @@ class BaseMarionetteOptions(OptionParser):
                         action='store_true',
                         default=False,
                         help='Enable e10s when running marionette tests.')
+        self.add_option('--tag',
+                        action='append', dest='test_tags',
+                        default=None,
+                        help="Filter out tests that don't have the given tag. Can be "
+                             "used multiple times in which case the test must contain "
+                             "at least one of the given tags.")
 
     def parse_args(self, args=None, values=None):
         options, tests = OptionParser.parse_args(self, args, values)
@@ -494,7 +501,7 @@ class BaseMarionetteTestRunner(object):
                  shuffle=False, shuffle_seed=random.randint(0, sys.maxint),
                  sdcard=None, this_chunk=1, total_chunks=1, sources=None,
                  server_root=None, gecko_log=None, result_callbacks=None,
-                 adb_host=None, adb_port=None, prefs=None,
+                 adb_host=None, adb_port=None, prefs=None, test_tags=None,
                  socket_timeout=BaseMarionetteOptions.socket_timeout_default,
                  **kwargs):
         self.address = address
@@ -540,6 +547,7 @@ class BaseMarionetteTestRunner(object):
         self._adb_host = adb_host
         self._adb_port = adb_port
         self.prefs = prefs or {}
+        self.test_tags = test_tags
 
         def gather_debug(test, status):
             rv = {}
@@ -884,11 +892,20 @@ setReq.onerror = function() {
             manifest = TestManifest()
             manifest.read(filepath)
 
+            filters = []
+            if self.test_tags:
+                filters.append(tags(self.test_tags))
             manifest_tests = manifest.active_tests(exists=False,
                                                    disabled=True,
+                                                   filters=filters,
                                                    device=self.device,
                                                    app=self.appName,
                                                    **mozinfo.info)
+            if len(manifest_tests) == 0:
+                self.logger.error("no tests to run using specified "
+                                  "combination of filters: {}".format(
+                                       manifest.fmt_filters()))
+
             unfiltered_tests = []
             for test in manifest_tests:
                 if test.get('disabled'):

testing/marionette/mach_commands.py

@@ -88,6 +88,10 @@ class B2GCommands(MachCommandBase):
     @CommandArgument('--type',
         default='b2g',
         help='Test type, usually one of: browser, b2g, b2g-qemu.')
+    @CommandArgument('--tag', action='append', dest='test_tags',
+        help='Filter out tests that don\'t have the given tag. Can be used '
+             'multiple times in which case the test must contain at least one '
+             'of the given tags.')
     @CommandArgument('tests', nargs='*', metavar='TESTS',
         help='Path to test(s) to run.')
     def run_marionette_webapi(self, tests, **kwargs):
@@ -128,6 +132,10 @@ class MachCommands(MachCommandBase):
              ' Pass in the debugger you want to use, eg pdb or ipdb.')
     @CommandArgument('--e10s', action='store_true',
         help='Enable electrolysis for marionette tests (desktop only).')
+    @CommandArgument('--tag', action='append', dest='test_tags',
+        help='Filter out tests that don\'t have the given tag. Can be used '
+             'multiple times in which case the test must contain at least one '
+             'of the given tags.')
     @CommandArgument('tests', nargs='*', metavar='TESTS',
         help='Path to test(s) to run.')
     def run_marionette_test(self, tests, **kwargs):

testing/mochitest/mach_commands.py

@@ -737,6 +737,14 @@ def MochitestCommand(func):
         help='The maximum number of timeouts permitted before halting testing')
     func = max_timeouts(func)
 
+    tags = CommandArgument(
+        "--tag",
+        dest='test_tags', action='append',
+        help="Filter out tests that don't have the given tag. Can be used "
+             "multiple times in which case the test must contain at least one "
+             "of the given tags.")
+    func = tags(func)
+
     return func
 
 
@@ -814,6 +822,14 @@ def B2GCommand(func):
         'with the --repeat parameter.')
     func = runUntilFailure(func)
 
+    tags = CommandArgument(
+        "--tag",
+        dest='test_tags', action='append',
+        help="Filter out tests that don't have the given tag. Can be used "
+             "multiple times in which case the test must contain at least one "
+             "of the given tags.")
+    func = tags(func)
+
     return func
 
 

testing/mochitest/mochitest_options.py

@@ -470,6 +470,14 @@ class MochitestOptions(optparse.OptionParser):
            "help": "maximum number of timeouts permitted before halting testing",
            "default": None,
            }],
+        [["--tag"],
+         { "action": "append",
+           "dest": "test_tags",
+           "default": None,
+           "help": "filter out tests that don't have the given tag. Can be "
+                   "used multiple times in which case the test must contain "
+                   "at least one of the given tags.",
+         }],
     ]
 
     def __init__(self, **kwargs):

testing/mochitest/runtests.py

@@ -50,6 +50,7 @@ from manifestparser.filters import (
     chunk_by_dir,
     chunk_by_slice,
     subsuite,
+    tags,
 )
 from mochitest_options import MochitestOptions
 from mozprofile import Profile, Preferences
@@ -1893,11 +1894,17 @@ class Mochitest(MochitestUtilsMixin):
                 elif options.totalChunks:
                     filters.append(chunk_by_slice(options.thisChunk,
                                                   options.totalChunks))
+
+                if options.test_tags:
+                    filters.append(tags(options.test_tags))
+
                 tests = manifest.active_tests(
                     exists=False, disabled=disabled, filters=filters, **info)
+
                 if len(tests) == 0:
-                    tests = manifest.active_tests(
-                        exists=False, disabled=True, **info)
+                    self.log.error("no tests to run using specified "
+                                   "combination of filters: {}".format(
+                                        manifest.fmt_filters()))
 
         paths = []
 

testing/mozbase/manifestparser/manifestparser/filters.py

@@ -84,10 +84,22 @@ class InstanceFilter(object):
     Generally only one instance of a class filter should be applied at a time.
     Two instances of `InstanceFilter` are considered equal if they have the
     same class name. This ensures only a single instance is ever added to
-    `filterlist`.
+    `filterlist`. This class also formats filters' __str__ method for easier
+    debugging.
     """
+    unique = True
+    def __init__(self, *args, **kwargs):
+        self.fmt_args = ', '.join(itertools.chain(
+            [str(a) for a in args],
+            ['{}={}'.format(k, v) for k, v in kwargs.iteritems()]))
+
     def __eq__(self, other):
-        return self.__class__ == other.__class__
+        if self.unique:
+            return self.__class__ == other.__class__
+        return self.__hash__() == other.__hash__()
+
+    def __str__(self):
+        return "{}({})".format(self.__class__.__name__, self.fmt_args)
 
 
 class subsuite(InstanceFilter):
@@ -105,6 +117,7 @@ class subsuite(InstanceFilter):
     :param name: The name of the subsuite to run (default None)
     """
     def __init__(self, name=None):
+        InstanceFilter.__init__(self, name=name)
         self.name = name
 
     def __call__(self, tests, values):
@@ -146,6 +159,8 @@ class chunk_by_slice(InstanceFilter):
 
     def __init__(self, this_chunk, total_chunks, disabled=False):
         assert 1 <= this_chunk <= total_chunks
+        InstanceFilter.__init__(self, this_chunk, total_chunks,
+                                disabled=disabled)
         self.this_chunk = this_chunk
         self.total_chunks = total_chunks
         self.disabled = disabled
@@ -196,6 +211,7 @@ class chunk_by_dir(InstanceFilter):
     """
 
     def __init__(self, this_chunk, total_chunks, depth):
+        InstanceFilter.__init__(self, this_chunk, total_chunks, depth)
         self.this_chunk = this_chunk
         self.total_chunks = total_chunks
         self.depth = depth
@@ -250,6 +266,7 @@ class chunk_by_runtime(InstanceFilter):
     """
 
     def __init__(self, this_chunk, total_chunks, runtimes):
+        InstanceFilter.__init__(self, this_chunk, total_chunks, runtimes)
         self.this_chunk = this_chunk
         self.total_chunks = total_chunks
 
@@ -285,6 +302,31 @@ class chunk_by_runtime(InstanceFilter):
         return (t for t in tests_by_chunk[self.this_chunk-1][1])
 
 
+class tags(InstanceFilter):
+    """
+    Removes tests that don't contain any of the given tags. This overrides
+    InstanceFilter's __eq__ method, so multiple instances can be added.
+    Multiple tag filters is equivalent to joining tags with the AND operator.
+
+    :param tags: A tag or list of tags to filter tests on
+    """
+    unique = False
+    def __init__(self, tags):
+        InstanceFilter.__init__(self, tags)
+        if isinstance(tags, basestring):
+            tags = [tags]
+        self.tags = tags
+
+    def __call__(self, tests, values):
+        for test in tests:
+            if 'tags' not in test:
+                continue
+
+            test_tags = [t.strip() for t in test['tags'].split(',')]
+            if any(t in self.tags for t in test_tags):
+                yield test
+
+
 # filter container
 
 DEFAULT_FILTERS = (

testing/mozbase/manifestparser/manifestparser/manifestparser.py

@@ -10,6 +10,7 @@ import fnmatch
 import os
 import shutil
 import sys
+import types
 
 from .ini import read_ini
 from .filters import (
@@ -710,6 +711,7 @@ class TestManifest(ManifestParser):
     def __init__(self, *args, **kwargs):
         ManifestParser.__init__(self, *args, **kwargs)
         self.filters = filterlist(DEFAULT_FILTERS)
+        self.last_used_filters = []
 
     def active_tests(self, exists=True, disabled=True, filters=None, **values):
         """
@@ -741,9 +743,20 @@ class TestManifest(ManifestParser):
         if filters:
             fltrs += filters
 
+        self.last_used_filters = fltrs[:]
         for fn in fltrs:
             tests = fn(tests, values)
         return list(tests)
 
     def test_paths(self):
         return [test['path'] for test in self.active_tests()]
+
+    def fmt_filters(self, filters=None):
+        filters = filters or self.last_used_filters
+        names = []
+        for f in filters:
+            if isinstance(f, types.FunctionType):
+                names.append(f.__name__)
+            else:
+                names.append(str(f))
+        return ', '.join(names)

testing/mozbase/manifestparser/tests/test_filters.py

@@ -4,14 +4,13 @@ from copy import deepcopy
 import os
 import unittest
 
-from manifestparser import TestManifest
 from manifestparser.filters import (
     subsuite,
+    tags,
     skip_if,
     run_if,
     fail_if,
     enabled,
-    exists,
     filterlist,
 )
 
@@ -47,12 +46,12 @@ class FilterList(unittest.TestCase):
         with self.assertRaises(IndexError):
             fl[0]
 
-    def test_add_non_callable_to_set(self):
+    def test_add_non_callable_to_list(self):
         fl = filterlist()
         with self.assertRaises(TypeError):
             fl.append('foo')
 
-    def test_add_duplicates_to_set(self):
+    def test_add_duplicates_to_list(self):
         foo = lambda x, y: x
         bar = lambda x, y: x
         sub = subsuite('foo')
@@ -66,6 +65,17 @@ class FilterList(unittest.TestCase):
         with self.assertRaises(ValueError):
             fl.append(subsuite('bar'))
 
+    def test_add_two_tags_filters(self):
+        tag1 = tags('foo')
+        tag2 = tags('bar')
+        fl = filterlist([tag1])
+
+        with self.assertRaises(ValueError):
+            fl.append(tag1)
+
+        fl.append(tag2)
+        self.assertEquals(len(fl), 2)
+
     def test_filters_run_in_order(self):
         a = lambda x, y: x
         b = lambda x, y: x
@@ -85,13 +95,15 @@ class BuiltinFilters(unittest.TestCase):
     """Test the built-in filters"""
 
     tests = (
-        { "name": "test0" },
-        { "name": "test1", "skip-if": "foo == 'bar'" },
-        { "name": "test2", "run-if": "foo == 'bar'" },
-        { "name": "test3", "fail-if": "foo == 'bar'" },
-        { "name": "test4", "disabled": "some reason" },
-        { "name": "test5", "subsuite": "baz" },
-        { "name": "test6", "subsuite": "baz,foo == 'bar'" })
+        {"name": "test0"},
+        {"name": "test1", "skip-if": "foo == 'bar'"},
+        {"name": "test2", "run-if": "foo == 'bar'"},
+        {"name": "test3", "fail-if": "foo == 'bar'"},
+        {"name": "test4", "disabled": "some reason"},
+        {"name": "test5", "subsuite": "baz"},
+        {"name": "test6", "subsuite": "baz,foo == 'bar'"},
+        {"name": "test7", "tags": "foo, bar"},
+    )
 
     def test_skip_if(self):
         tests = deepcopy(self.tests)
@@ -132,7 +144,7 @@ class BuiltinFilters(unittest.TestCase):
         tests = deepcopy(self.tests)
         tests = list(sub1(tests, {}))
         self.assertNotIn(self.tests[5], tests)
-        self.assertEquals(tests[-1]['name'], 'test6')
+        self.assertEquals(len(tests), len(self.tests)-1)
 
         tests = deepcopy(self.tests)
         tests = list(sub2(tests, {}))
@@ -154,3 +166,16 @@ class BuiltinFilters(unittest.TestCase):
         self.assertEquals(len(tests), 2)
         self.assertEquals(tests[0]['name'], 'test5')
         self.assertEquals(tests[1]['name'], 'test6')
+
+    def test_tags(self):
+        ftags1 = tags([])
+        ftags2 = tags(['bar', 'baz'])
+
+        tests = deepcopy(self.tests)
+        tests = list(ftags1(tests, {}))
+        self.assertEquals(len(tests), 0)
+
+        tests = deepcopy(self.tests)
+        tests = list(ftags2(tests, {}))
+        self.assertEquals(len(tests), 1)
+        self.assertIn(self.tests[7], tests)

testing/xpcshell/mach_commands.py

@@ -67,7 +67,7 @@ class XPCShellRunner(MozbuildObject):
                  debugger=None, debuggerArgs=None, debuggerInteractive=None,
                  jsDebugger=False, jsDebuggerPort=None,
                  rerun_failures=False, test_objects=None, verbose=False,
-                 log=None,
+                 log=None, test_tags=None,
                  # ignore parameters from other platforms' options
                  **kwargs):
         """Runs an individual xpcshell test."""
@@ -89,7 +89,7 @@ class XPCShellRunner(MozbuildObject):
                            debuggerInteractive=debuggerInteractive,
                            jsDebugger=jsDebugger, jsDebuggerPort=jsDebuggerPort,
                            rerun_failures=rerun_failures,
-                           verbose=verbose, log=log)
+                           verbose=verbose, log=log, test_tags=test_tags)
             return
         elif test_paths:
             test_paths = [self._wrap_path_argument(p).relpath() for p in test_paths]
@@ -124,6 +124,7 @@ class XPCShellRunner(MozbuildObject):
             'manifest': manifest,
             'verbose': verbose,
             'log': log,
+            'test_tags': test_tags,
         }
 
         return self._run_xpcshell_harness(**args)
@@ -133,7 +134,7 @@ class XPCShellRunner(MozbuildObject):
                               keep_going=False, sequential=False,
                               debugger=None, debuggerArgs=None, debuggerInteractive=None,
                               jsDebugger=False, jsDebuggerPort=None,
-                              rerun_failures=False, verbose=False, log=None):
+                              rerun_failures=False, verbose=False, log=None, test_tags=None):
 
         # Obtain a reference to the xpcshell test runner.
         import runxpcshelltests
@@ -171,6 +172,7 @@ class XPCShellRunner(MozbuildObject):
             'debuggerInteractive': debuggerInteractive,
             'jsDebugger': jsDebugger,
             'jsDebuggerPort': jsDebuggerPort,
+            'test_tags': test_tags,
         }
 
         if test_path is not None:
@@ -438,6 +440,10 @@ class MachCommands(MachCommandBase):
         help='Randomize the execution order of tests.')
     @CommandArgument('--rerun-failures', action='store_true',
         help='Reruns failures from last time.')
+    @CommandArgument('--tag', action='append', dest='test_tags',
+        help='Filter out tests that don\'t have the given tag. Can be used '
+             'multiple times in which case the test must contain at least one '
+             'of the given tags.')
     @CommandArgument('--devicemanager', default='adb', type=str,
         help='(Android) Type of devicemanager to use for communication: adb or sut')
     @CommandArgument('--ip', type=str, default=None,

testing/xpcshell/runxpcshelltests.py

@@ -61,7 +61,7 @@ if os.path.isdir(mozbase):
         sys.path.append(os.path.join(mozbase, package))
 
 from manifestparser import TestManifest
-from manifestparser.filters import chunk_by_slice
+from manifestparser.filters import chunk_by_slice, tags
 from mozlog import structured
 import mozcrash
 import mozinfo
@@ -752,7 +752,7 @@ class XPCShellTests(object):
         self.harness_timeout = HARNESS_TIMEOUT
         self.nodeProc = {}
 
-    def buildTestList(self):
+    def buildTestList(self, test_tags):
         """
           read the xpcshell.ini manifest and set self.alltests to be
           an array of test objects.
@@ -773,15 +773,22 @@ class XPCShellTests(object):
         self.buildTestPath()
 
         filters = []
+        if test_tags:
+            filters.append(tags(test_tags))
+
         if self.singleFile is None and self.totalChunks > 1:
             filters.append(chunk_by_slice(self.thisChunk, self.totalChunks))
-
         try:
             self.alltests = mp.active_tests(filters=filters, **mozinfo.info)
         except TypeError:
             sys.stderr.write("*** offending mozinfo.info: %s\n" % repr(mozinfo.info))
             raise
 
+        if len(self.alltests) == 0:
+            self.log.error("no tests to run using specified "
+                           "combination of filters: {}".format(
+                                mp.fmt_filters()))
+
     def setAbsPath(self):
         """
           Set the absolute path for xpcshell, httpdjspath and xrepath.
@@ -1016,7 +1023,7 @@ class XPCShellTests(object):
                  testsRootDir=None, testingModulesDir=None, pluginsPath=None,
                  testClass=XPCShellTestThread, failureManifest=None,
                  log=None, stream=None, jsDebugger=False, jsDebuggerPort=0,
-                 **otherOptions):
+                 test_tags=None, **otherOptions):
         """Run xpcshell tests.
 
         |xpcshell|, is the xpcshell executable to use to run the tests.
@@ -1164,7 +1171,7 @@ class XPCShellTests(object):
 
         pStdout, pStderr = self.getPipes()
 
-        self.buildTestList()
+        self.buildTestList(test_tags)
         if self.singleFile:
             self.sequential = True
 
@@ -1467,6 +1474,12 @@ class XPCShellOptions(OptionParser):
                         default=6000,
                         help="The port to listen on for a debugger connection if "
                              "--jsdebugger is specified.")
+        self.add_option("--tag",
+                        action="append", dest="test_tags",
+                        default=None,
+                        help="filter out tests that don't have the given tag. Can be "
+                             "used multiple times in which case the test must contain "
+                             "at least one of the given tags.")
 
 def main():
     parser = XPCShellOptions()