Bug 1193215 - Support for passing test directories through mach try, r=chmanchester

This adds support for web-platform-tests to mach try. It changes the implementation
so that instead of passing paths to manifests, the user passes arbitary paths in the
source tree, and tests under that path are run, with test discovery mainly left to
the harness.

testing/mach_commands.py

@@ -436,6 +436,9 @@ class JsapiTestsCommand(MachCommandBase):
 
         return jsapi_tests_result
 
+def autotry_parser():
+    from autotry import parser
+    return parser()
 
 @CommandProvider
 class PushToTry(MachCommandBase):
@@ -448,6 +451,18 @@ class PushToTry(MachCommandBase):
         if platforms is None:
             platforms = os.environ['AUTOTRY_PLATFORM_HINT']
 
+        rv_platforms = []
+        for item in platforms:
+            for platform in item.split(","):
+                if platform:
+                    rv_platforms.append(platform)
+
+        rv_tests = []
+        for item in tests:
+            for test in item.split(","):
+                if test:
+                    rv_tests.append(test)
+
         for p in paths:
             p = os.path.normpath(os.path.abspath(p))
             if not p.startswith(self.topsrcdir):
@@ -459,69 +474,67 @@ class PushToTry(MachCommandBase):
                       ' select all tests.' % p)
                 sys.exit(1)
 
-        return builds, platforms
+        return builds, rv_platforms, rv_tests
 
-    @Command('try', category='testing', description='Push selected tests to the try server')
-    @CommandArgument('paths', nargs='*', help='Paths to search for tests to run on try.')
-    @CommandArgument('-n', dest='verbose', action='store_true', default=False,
-                     help='Print detailed information about the resulting test selection '
-                          'and commands performed.')
-    @CommandArgument('-p', dest='platforms', required='AUTOTRY_PLATFORM_HINT' not in os.environ,
-                     help='Platforms to run. (required if not found in the environment)')
-    @CommandArgument('-u', dest='tests',
-                     help='Test jobs to run. These will be used in place of suites '
-                          'determined by test paths, if any.')
-    @CommandArgument('--extra', dest='extra_tests',
-                     help='Additional tests to run. These will be added to suites '
-                          'determined by test paths, if any.')
-    @CommandArgument('-b', dest='builds', default='do',
-                     help='Build types to run (d for debug, o for optimized)')
-    @CommandArgument('--tag', dest='tags', action='append',
-                     help='Restrict tests to the given tag (may be specified multiple times)')
-    @CommandArgument('--no-push', dest='push', action='store_false',
-                     help='Do not push to try as a result of running this command (if '
-                          'specified this command will only print calculated try '
-                          'syntax and selection info).')
+    @Command('try',
+             category='testing',
+             description='Push selected tests to the try server',
+             parser=autotry_parser)
     def autotry(self, builds=None, platforms=None, paths=None, verbose=None,
-                extra_tests=None, push=None, tags=None, tests=None):
-        """mach try is under development, please file bugs blocking 1149670.
+                push=None, tags=None, tests=None, extra_args=None, intersection=False):
+        """Autotry is in beta, please file bugs blocking 1149670.
 
-        Pushes the specified tests to try. The simplest way to specify tests is
-        by using the -u argument, which will behave as usual for try syntax.
-        This command also provides a mechanism to select test jobs and tests
-        within a job by path based on tests present in the tree under that
-        path. Mochitests, xpcshell tests, and reftests are eligible for
-        selection by this mechanism. Selected tests will be run in a single
-        chunk of the relevant suite, at this time in chunk 1.
+        Push the current tree to try, with the specified syntax.
 
-        Specifying platforms is still required with the -p argument (a default
-        is taken from the AUTOTRY_PLATFORM_HINT environment variable if set).
+        Build options, platforms and regression tests may be selected
+        using the usual try options (-b, -p and -u respectively). In
+        addition, tests in a given directory may be automatically
+        selected by passing that directory as a positional argument to the
+        command. For example:
 
-        Tests may be further filtered by passing one or more --tag to the
-        command. If one or more --tag is specified with out paths or -u,
-        tests with the given tags will be run in a single chunk of
-        applicable suites.
+        mach try -b d -p linux64 dom testing/web-platform/tests/dom
 
-        To run suites in addition to those determined from the tree, they
-        can be passed to the --extra arguent.
+        would schedule a try run for linux64 debug consisting of all
+        tests under dom/ and testing/web-platform/tests/dom.
+
+        Test selection using positional arguments is available for
+        mochitests, reftests, xpcshell tests and web-platform-tests.
+
+        Tests may be also filtered by passing --tag to the command,
+        which will run only tests marked as having the specified
+        tags e.g.
+
+        mach try -b d -p win64 --tag media
+
+        would run all tests tagged 'media' on Windows 64.
+
+        If both positional arguments or tags and -u are supplied, the
+        suites in -u will be run in full. Where tests are selected by
+        positional argument they will be run in a single chunk.
+
+        If no build option is selected, both debug and opt will be
+        scheduled. If no platform is selected a default is taken from
+        the AUTOTRY_PLATFORM_HINT environment variable, if set.
 
         The command requires either its own mercurial extension ("push-to-try",
         installable from mach mercurial-setup) or a git repo using git-cinnabar
         (available at https://github.com/glandium/git-cinnabar).
+
         """
 
         from mozbuild.testing import TestResolver
         from mozbuild.controller.building import BuildDriver
         from autotry import AutoTry
-        import pprint
-
         print("mach try is under development, please file bugs blocking 1149670.")
 
-        builds, platforms = self.validate_args(paths, tests, tags, builds, platforms)
+        if tests is None:
+            tests = []
+
+        builds, platforms, tests = self.validate_args(paths, tests, tags, builds, platforms)
         resolver = self._spawn(TestResolver)
 
         at = AutoTry(self.topsrcdir, resolver, self._mach_context)
-        if at.find_uncommited_changes():
+        if push and at.find_uncommited_changes():
             print('ERROR please commit changes before continuing')
             sys.exit(1)
 
@@ -529,27 +542,34 @@ class PushToTry(MachCommandBase):
             driver = self._spawn(BuildDriver)
             driver.install_tests(remove=False)
 
-        manifests_by_flavor = at.resolve_manifests(paths=paths, tags=tags)
+            paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
+                     for item in paths]
+            paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags)
 
-        if not manifests_by_flavor and not tests:
+            if not paths_by_flavor and not tests:
                 print("No tests were found when attempting to resolve paths:\n\n\t%s" %
                       paths)
                 sys.exit(1)
 
-        all_manifests = set()
-        for m in manifests_by_flavor.values():
-            all_manifests |= m
-        all_manifests = list(all_manifests)
+            if not intersection:
+                paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests)
+        else:
+            paths_by_flavor = {}
 
-        msg = at.calc_try_syntax(platforms, manifests_by_flavor.keys(), tests,
-                                 extra_tests, builds, all_manifests, tags)
+        try:
+            msg = at.calc_try_syntax(platforms, tests, builds, paths_by_flavor, tags,
+                                     extra_args, intersection)
+        except ValueError as e:
+            print(e.message)
+            sys.exit(1)
 
-        if verbose and manifests_by_flavor:
-            print('Tests from the following manifests will be selected: ')
-            pprint.pprint(manifests_by_flavor)
+        if verbose and paths_by_flavor:
+            print('The following tests will be selected: ')
+            for flavor, paths in paths_by_flavor.iteritems():
+                print("%s: %s" % (flavor, ",".join(paths)))
 
         if verbose or not push:
-            print('The following try syntax was calculated:\n\n\t%s\n' % msg)
+            print('The following try syntax was calculated:\n%s' % msg)
 
         if push:
             at.push_to_try(msg, verbose)

testing/mozharness/configs/b2g/emulator_automation_config.py

@@ -119,8 +119,8 @@ config = {
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
-                "%(test_path)s"
             ],
+            "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
         },
@@ -143,8 +143,8 @@ config = {
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
-                "%(test_path)s"
             ],
+            "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
         },

testing/mozharness/configs/b2g/mulet_config.py

@@ -20,8 +20,8 @@ config = {
                 "--this-chunk=%(this_chunk)s",
                 "--symbols-path=%(symbols_path)s",
                 "--enable-oop",
-                "%(test_manifest)s"
             ],
+            "tests": ["%(test_manifest)s"]
         }
     },
     "run_file_names": {

testing/mozharness/configs/unittests/linux_unittest.py

@@ -256,7 +256,7 @@ config = {
         },
     },
     "all_cppunittest_suites": {
-        "cppunittest": ["tests/cppunittest"]
+        "cppunittest": {"tests": ["tests/cppunittest"]}
     },
     "all_gtest_suites": {
         "gtest": []

testing/mozharness/mozharness/mozilla/testing/testbase.py

@@ -26,7 +26,7 @@ from mozharness.mozilla.proxxy import Proxxy
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.taskcluster_helper import TaskClusterArtifactFinderMixin
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
-from mozharness.mozilla.testing.try_tools import TryToolsMixin
+from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
 from mozharness.mozilla.tooltool import TooltoolMixin
 
 from mozharness.lib.python.authentication import get_credentials
@@ -83,7 +83,7 @@ testing_config_options = [
      "choices": ['ondemand', 'true'],
      "help": "Download and extract crash reporter symbols.",
       }],
-] + copy.deepcopy(virtualenv_config_options)
+] + copy.deepcopy(virtualenv_config_options) + copy.deepcopy(try_config_options)
 
 
 # TestingMixin {{{1
@@ -676,6 +676,35 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""")
 
         return self.minidump_stackwalk_path
 
+    def query_options(self, *args, **kwargs):
+        if "str_format_values" in kwargs:
+            str_format_values = kwargs.pop("str_format_values")
+        else:
+            str_format_values = {}
+
+        arguments = []
+
+        for arg in args:
+            if arg is not None:
+                arguments.extend(argument % str_format_values for argument in arg)
+
+        return arguments
+
+    def query_tests_args(self, *args, **kwargs):
+        if "str_format_values" in kwargs:
+            str_format_values = kwargs.pop("str_format_values")
+        else:
+            str_format_values = {}
+
+        arguments = []
+
+        for arg in reversed(args):
+            if arg:
+                arguments.append("--")
+                arguments.extend(argument % str_format_values for argument in arg)
+                break
+
+        return arguments
 
     def _run_cmd_checks(self, suites):
         if not suites:

testing/mozharness/mozharness/mozilla/testing/try_tools.py

@@ -8,17 +8,43 @@
 import argparse
 import os
 import re
+from collections import defaultdict
 
 from mozharness.base.script import PostScriptAction
 from mozharness.base.transfer import TransferMixin
 
+try_config_options = [
+    [["--try-message"],
+     {"action": "store",
+     "dest": "try_message",
+     "default": None,
+     "help": "try syntax string to select tests to run",
+      }],
+]
+
+test_flavors = {
+    'browser-chrome': {},
+    'chrome': {},
+    'devtools-chrome': {},
+    'mochitest': {},
+    'xpcshell' :{},
+    'reftest': {
+        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
+    },
+    'crashtest': {
+        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
+    },
+    'web-platform-tests': {
+        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
+    }
+}
 
 class TryToolsMixin(TransferMixin):
     """Utility functions for an interface between try syntax and out test harnesses.
     Requires log and script mixins."""
 
     harness_extra_args = None
-    try_test_paths = []
+    try_test_paths = {}
     known_try_arguments = {
         '--tag': {
             'action': 'append',
@@ -29,6 +55,9 @@ class TryToolsMixin(TransferMixin):
 
     def _extract_try_message(self):
         msg = None
+        if "try_message" in self.config and self.config["try_message"]:
+            msg = self.config["try_message"]
+        else:
             if self.buildbot_config['sourcestamp']['changes']:
                 msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments']
 
@@ -53,7 +82,7 @@ class TryToolsMixin(TransferMixin):
         return msg
 
     @PostScriptAction('download-and-extract')
-    def _set_extra_try_arguments(self, action, success=None):
+    def set_extra_try_arguments(self, action, success=None):
         """Finds a commit message and parses it for extra arguments to pass to the test
         harness command line and test paths used to filter manifests.
 
@@ -107,7 +136,7 @@ class TryToolsMixin(TransferMixin):
 
         parser.add_argument('--try-test-paths', nargs='*')
         (args, _) = parser.parse_known_args(all_try_args)
-        self.try_test_paths = args.try_test_paths
+        self.try_test_paths = self._group_test_paths(args.try_test_paths)
         del args.try_test_paths
 
         out_args = []
@@ -127,91 +156,35 @@ class TryToolsMixin(TransferMixin):
 
         self.harness_extra_args = out_args
 
-    def _resolve_specified_manifests(self):
-        if not self.try_test_paths:
-            return None
+    def _group_test_paths(self, args):
+        rv = defaultdict(list)
 
-        target_manifests = set(self.try_test_paths)
+        if args is None:
+            return rv
 
-        def filter_ini_manifest(line):
-            # Lines are formatted as [include:<path>], we care about <path>.
-            parts = line.split(':')
-            term = line
-            if len(parts) == 2:
-                term = parts[1]
-            if term.endswith(']'):
-                term = term[:-1]
-            if (term in target_manifests or
-                any(term.startswith(l) for l in target_manifests)):
-                return True
-            return False
+        for item in args:
+            suite, path = item.split(":", 1)
+            rv[suite].append(path)
+        return rv
 
-        def filter_list_manifest(line):
-            # Lines are usually formatted as "include <path>", we care about <path>.
-            parts = line.split()
-            term = line
-            if len(parts) == 2:
-                term = parts[1]
-            # Reftest master manifests also include skip-if lines and relative
-            # paths we aren't doing to resolve here, so unlike above this is just
-            # a substring check.
-            if (term in target_manifests or
-                any(l in term for l in target_manifests)):
-                return True
-            return False
-
-        # The master manifests we need to filter for target manifests.
-        # TODO: All this needs to go in a config file somewhere and get sewn
-        # into the job definition so its less likely to break as things are
-        # modified. One straightforward way to achieve this would be with a key
-        # in the tree manifest for the master manifest path, however the current
-        # tree manifests don't distinguish between flavors of mochitests to this
-        # isn't straightforward.
-        master_manifests = [
-            ('mochitest/chrome/chrome.ini', filter_ini_manifest),
-            ('mochitest/browser/browser-chrome.ini', filter_ini_manifest),
-            ('mochitest/tests/mochitest.ini', filter_ini_manifest),
-            ('xpcshell/tests/all-test-dirs.list', filter_ini_manifest),
-            ('xpcshell/tests/xpcshell.ini', filter_ini_manifest),
-            ('reftest/tests/layout/reftests/reftest.list', filter_list_manifest),
-            ('reftest/tests/testing/crashtest/crashtests.list', filter_list_manifest),
-        ]
-
-        dirs = self.query_abs_dirs()
-        tests_dir = dirs.get('abs_test_install_dir',
-                             os.path.join(dirs['abs_work_dir'], 'tests'))
-        master_manifests = [(os.path.join(tests_dir, name), filter_fn) for (name, filter_fn) in
-                            master_manifests]
-        for m, filter_fn in master_manifests:
-            if not os.path.isfile(m):
-                continue
-
-            self.info("Filtering master manifest at: %s" % m)
-            lines = self.read_from_file(m).splitlines()
-
-            out_lines = [line for line in lines if filter_fn(line)]
-
-            self.rmtree(m)
-            self.write_to_file(m, '\n'.join(out_lines))
-
-
-    def append_harness_extra_args(self, cmd):
-        """Append arguments derived from try syntax to a command."""
+    def try_args(self, flavor):
+        """Get arguments, test_list derived from try syntax to apply to a command"""
         # TODO: Detect and reject incompatible arguments
-        extra_args = self.harness_extra_args[:] if self.harness_extra_args else []
-        if self.try_test_paths:
-            self._resolve_specified_manifests()
+        args = self.harness_extra_args[:] if self.harness_extra_args else []
+
+        if self.try_test_paths.get(flavor):
             self.info('TinderboxPrint: Tests will be run from the following '
-                      'manifests: %s.' % ','.join(self.try_test_paths))
-            extra_args.extend(['--this-chunk=1', '--total-chunks=1'])
+                      'files: %s.' % ','.join(self.try_test_paths[flavor]))
+            args.extend(['--this-chunk=1', '--total-chunks=1'])
 
-        if not extra_args:
-            return cmd
+            path_func = test_flavors[flavor].get("path", lambda x:x)
+            tests = [path_func(item) for item in self.try_test_paths[flavor]]
+        else:
+            tests = []
 
-        out_cmd = cmd[:]
-        out_cmd.extend(extra_args)
+        if args or tests:
             self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
-                  'to the test command:\nTinderboxPrint: \t%s' %
-                  extra_args)
+                      'to the test command:\nTinderboxPrint: \t%s -- %s' %
+                      (" ".join(args), " ".join(tests)))
 
-        return out_cmd
+        return args, tests

testing/mozharness/scripts/android_emulator_unittest.py

@@ -483,14 +483,12 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
                 continue
             cmd.append(arg)
 
-        tests = None
-        if "tests" in self.test_suite_definitions[self.test_suite]:
-            tests = self.test_suite_definitions[self.test_suite]["tests"]
-        elif "tests" in self.config["suite_definitions"][suite_category]:
-            tests = self.config["suite_definitions"][suite_category]["tests"]
-
-        if tests:
-            cmd.extend(tests)
+        try_options, try_tests = self.try_args(suite_category)
+        cmd.extend(try_options)
+        cmd.extend(self.query_tests_args(
+            self.config["suite_definitions"][suite_category].get("tests"),
+            self.test_suite_definitions[self.test_suite].get("tests"),
+            try_tests))
 
         return cmd
 
@@ -664,7 +662,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
         Run the tests
         """
         cmd = self._build_command()
-        cmd = self.append_harness_extra_args(cmd)
+
         try:
             cwd = self._query_tests_dir()
         except:

testing/mozharness/scripts/android_panda.py

@@ -217,6 +217,20 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu
                                  if self._query_specified_suites(cat) is not None]
         super(PandaTest, self).download_and_extract(suite_categories=target_categories)
 
+    def _query_try_flavor(self, category, suite):
+        flavors = {
+            "mochitest": [("plain.*", "mochitest"),
+                          ("browser-chrome.*", "browser-chrome"),
+                          ("mochitest-devtools-chrome.*", "devtools-chrome"),
+                          ("chrome", "chrome")],
+            "xpcshell": [("xpcshell", "xpcshell")],
+            "reftest": [("reftest", "reftest"),
+                        ("crashtest", "crashtest")]
+        }
+        for suite_pattern, flavor in flavors.get(category, []):
+            if re.compile(suite_pattern).match(suite):
+                return flavor
+
     def _run_category_suites(self, suite_category, preflight_run_method=None):
         """run suite(s) to a specific category"""
 
@@ -246,11 +260,13 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu
                 if should_install_app:
                     self._install_app()
                 cmd = abs_base_cmd[:]
-                replace_dict = {}
-                for arg in suites[suite]:
-                    cmd.append(arg % replace_dict)
 
-                cmd = self.append_harness_extra_args(cmd)
+                flavor = self._query_try_flavor(suite_category, suite)
+                try_options, try_tests = self.try_args(flavor)
+
+                cmd.extend(self.query_options(suites[suite],
+                                              try_options))
+                cmd.extend(self.query_tests_args(try_tests))
 
                 tests = self.config["suite_definitions"][suite_category].get("tests", [])
                 cmd += tests

testing/mozharness/scripts/androidx86_emulator_unittest.py

@@ -452,10 +452,12 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
                 continue
             cmd.append(arg)
 
-        if "tests" in self.test_suite_definitions[suite_name]:
-            cmd.extend(self.suite_definitions[suite_name]["tests"])
-        elif "tests" in c["suite_definitions"][suite_category]:
-            cmd.extend(self.config["suite_definitions"][suite_category]["tests"])
+        try_options, try_tests = self.try_args(suite_category)
+        cmd.extend(try_options)
+        cmd.extend(self.query_tests_args(
+            self.config["suite_definitions"][suite_category].get("tests"),
+            self.test_suite_definitions[suite_name].get("tests"),
+            try_tests))
 
         return cmd
 
@@ -477,7 +479,6 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
         """
         dirs = self.query_abs_dirs()
         cmd = self._build_command(self.emulators[emulator_index], suite_name)
-        cmd = self.append_harness_extra_args(cmd)
 
         try:
             cwd = self._query_tests_dir(suite_name)

testing/mozharness/scripts/b2g_desktop_unittest.py

@@ -139,6 +139,7 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript):
                                     '%s_raw.log' % suite)
         error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
                                           '%s_errorsummary.log' % suite)
+
         str_format_values = {
             'application': self.binary_path,
             'test_manifest': self.test_manifest,
@@ -156,16 +157,15 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript):
         if suite not in self.config["suite_definitions"]:
             self.fatal("'%s' not defined in the config!" % suite)
 
-        options = self.config["suite_definitions"][suite]["options"]
-        if options:
-            for option in options:
-                option = option % str_format_values
-                if not option.endswith('None'):
-                    cmd.append(option)
+        try_options, try_tests = self.try_args(suite)
 
-        tests = self.config["suite_definitions"][suite].get("tests")
-        if tests:
-            cmd.extend(tests)
+        cmd.extend(self.query_options(self.config["suite_definitions"][suite]["options"],
+                                      try_options,
+                                      str_format_values=str_format_values))
+
+        cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
+                                         try_tests,
+                                         str_format_values=str_format_values))
 
         return cmd
 
@@ -200,7 +200,6 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript):
             self.fatal("Don't know how to run --test-suite '%s'!" % suite)
 
         cmd = self._query_abs_base_cmd(suite)
-        cmd = self.append_harness_extra_args(cmd)
 
         cwd = dirs['abs_%s_dir' % suite]
 

testing/mozharness/scripts/b2g_emulator_unittest.py

@@ -273,18 +273,19 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
         }
 
         if suite not in self.config["suite_definitions"]:
-            self.fatal("Key '%s' not defined in the config!" % suite)
+            self.fatal("'%s' not defined in the config!" % suite)
 
-        options = self.config["suite_definitions"][suite]["options"]
-        if options:
-            for option in options:
-                option = option % str_format_values
-                if not option.endswith('None'):
-                    cmd.append(option)
+        try_options, try_tests = self.try_args(suite)
 
-        tests = self.config["suite_definitions"][suite].get("tests", [])
-        if tests:
-            cmd.extend(tests)
+        options = self.query_options(self.config["suite_definitions"][suite]["options"],
+                                     try_options,
+                                     str_format_values=str_format_values)
+        cmd.extend(opt for opt in options if not opt.endswith('None'))
+
+        tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
+                                      try_tests,
+                                      str_format_values=str_format_values)
+        cmd.extend(opt for opt in tests if not opt.endswith('None'))
 
         return cmd
 
@@ -341,8 +342,6 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
             self.fatal("Don't know how to run --test-suite '%s'!" % suite)
 
         cmd = self._query_abs_base_cmd(suite)
-        cmd = self.append_harness_extra_args(cmd)
-
         cwd = dirs['abs_%s_dir' % suite]
 
         # TODO we probably have to move some of the code in

testing/mozharness/scripts/desktop_unittest.py

@@ -34,7 +34,6 @@ from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_opt
 
 SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill', 'webapprt']
 
-
 # DesktopUnittest {{{1
 class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin):
     config_options = [
@@ -299,32 +298,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
         self.symbols_url = symbols_url
         return self.symbols_url
 
-    def get_webapprt_path(self, res_dir, mochitest_dir):
-        """Get the path to the webapp runtime binary.
-        On Mac, we copy the stub from the resources dir to the test app bundle,
-        since we have to run it from the executable directory of a bundle
-        in order for its windows to appear.  Ideally, the build system would do
-        this for us at build time, and we should find a way for it to do that.
-        """
-        exe_suffix = self.config.get('exe_suffix', '')
-        app_name = 'webapprt-stub' + exe_suffix
-        app_path = os.path.join(res_dir, app_name)
-        if self._is_darwin():
-            mac_dir_name = os.path.join(
-                mochitest_dir,
-                'webapprtChrome',
-                'webapprt',
-                'test',
-                'chrome',
-                'TestApp.app',
-                'Contents',
-                'MacOS')
-            mac_app_name = 'webapprt' + exe_suffix
-            mac_app_path = os.path.join(mac_dir_name, mac_app_name)
-            self.copyfile(app_path, mac_app_path, copystat=True)
-            return mac_app_path
-        return app_path
-
     def _query_abs_base_cmd(self, suite_category, suite):
         if self.binary_path:
             c = self.config
@@ -335,6 +308,11 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
             abs_app_dir = self.query_abs_app_dir()
             abs_res_dir = self.query_abs_res_dir()
 
+            webapprt_path = os.path.join(os.path.dirname(self.binary_path),
+                                         'webapprt-stub')
+            if c.get('exe_suffix'):
+                webapprt_path += c['exe_suffix']
+
             raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
                                         '%s_raw.log' % suite)
 
@@ -344,7 +322,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
                 'binary_path': self.binary_path,
                 'symbols_path': self._query_symbols_url(),
                 'abs_app_dir': abs_app_dir,
-                'abs_res_dir': abs_res_dir,
+                'app_path': webapprt_path,
                 'raw_log_file': raw_log_file,
                 'error_summary_file': error_summary_file,
                 'gtest_dir': os.path.join(dirs['abs_test_install_dir'],
@@ -356,9 +334,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
             if self.symbols_path:
                 str_format_values['symbols_path'] = self.symbols_path
 
-            if suite_category == 'webapprt':
-                str_format_values['app_path'] = self.get_webapprt_path(abs_res_dir, dirs['abs_mochitest_dir'])
-
             if c['e10s']:
                 base_cmd.append('--e10s')
 
@@ -398,6 +373,13 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
                              "please make sure they are specified in your "
                              "config under %s_options" %
                              (suite_category, suite_category))
+
+
+            for option in options:
+                option = option % str_format_values
+                if not option.endswith('None'):
+                    base_cmd.append(option)
+
             return base_cmd
         else:
             self.fatal("'binary_path' could not be determined.\n This should "
@@ -434,6 +416,20 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
 
         return suites
 
+    def _query_try_flavor(self, category, suite):
+        flavors = {
+            "mochitest": [("plain.*", "mochitest"),
+                          ("browser-chrome.*", "browser-chrome"),
+                          ("mochitest-devtools-chrome.*", "devtools-chrome"),
+                          ("chrome", "chrome")],
+            "xpcshell": [("xpcshell", "xpcshell")],
+            "reftest": [("reftest", "reftest"),
+                        ("crashtest", "crashtest")]
+        }
+        for suite_pattern, flavor in flavors.get(category, []):
+            if re.compile(suite_pattern).match(suite):
+                return flavor
+
     # Actions {{{2
 
     # clobber defined in BaseScript, deletes mozharness/build if exists
@@ -588,15 +584,22 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
                 options_list = []
                 env = {}
                 if isinstance(suites[suite], dict):
-                    options_list = suites[suite]['options'] + suites[suite].get("tests", [])
+                    options_list = suites[suite].get('options', [])
+                    tests_list = suites[suite].get('tests', [])
                     env = copy.deepcopy(suites[suite].get('env', {}))
                 else:
                     options_list = suites[suite]
+                    tests_list = []
 
-                for arg in options_list:
-                    cmd.append(arg % replace_dict)
+                flavor = self._query_try_flavor(suite_category, suite)
+                try_options, try_tests = self.try_args(flavor)
 
-                cmd = self.append_harness_extra_args(cmd)
+                cmd.extend(self.query_options(options_list,
+                                              try_options,
+                                              str_format_values=replace_dict))
+                cmd.extend(self.query_tests_args(tests_list,
+                                                 try_tests,
+                                                 str_format_values=replace_dict))
 
                 suite_name = suite_category + '-' + suite
                 tbpl_status, log_level = None, None

testing/mozharness/scripts/marionette.py

@@ -439,7 +439,10 @@ class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMix
             self.fatal("Could not create blobber upload directory")
 
         cmd.append(manifest)
-        cmd = self.append_harness_extra_args(cmd)
+
+        try_options, try_tests = self.try_args("marionette")
+        cmd.extend(self.query_tests_args(try_tests,
+                                         str_format_values=config_fmt_args))
 
         env = {}
         if self.query_minidump_stackwalk():

testing/mozharness/scripts/web_platform_tests.py

@@ -115,15 +115,15 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
         abs_app_dir = self.query_abs_app_dir()
         run_file_name = "runtests.py"
 
-        base_cmd = [self.query_python_path('python'), '-u']
-        base_cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
+        cmd = [self.query_python_path('python'), '-u']
+        cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
 
         # Make sure that the logging directory exists
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             self.fatal("Could not create blobber upload directory")
             # Exit
 
-        base_cmd += ["--log-raw=-",
+        cmd += ["--log-raw=-",
                 "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
                                               "wpt_raw.log"),
                 "--binary=%s" % self.binary_path,
@@ -131,15 +131,15 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
                 "--stackwalk-binary=%s" % self.query_minidump_stackwalk()]
 
         for test_type in c.get("test_type", []):
-            base_cmd.append("--test-type=%s" % test_type)
+            cmd.append("--test-type=%s" % test_type)
 
         if c.get("e10s"):
-            base_cmd.append("--e10s")
+            cmd.append("--e10s")
 
         for opt in ["total_chunks", "this_chunk"]:
             val = c.get(opt)
             if val:
-                base_cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
+                cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
 
         options = list(c.get("options", []))
 
@@ -151,9 +151,15 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
             'abs_work_dir': dirs["abs_work_dir"]
             }
 
-        opt_cmd = [item % str_format_values for item in options]
+        try_options, try_tests = self.try_args("web-platform-tests")
 
-        return base_cmd + opt_cmd
+        cmd.extend(self.query_options(options,
+                                      try_options,
+                                      str_format_values=str_format_values))
+        cmd.extend(self.query_tests_args(try_tests,
+                                         str_format_values=str_format_values))
+
+        return cmd
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
@@ -167,7 +173,6 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
     def run_tests(self):
         dirs = self.query_abs_dirs()
         cmd = self._query_cmd()
-        cmd = self.append_harness_extra_args(cmd)
 
         parser = StructuredOutputParser(config=self.config,
                                         log_obj=self.log_obj)

testing/tools/autotry/autotry.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import argparse
 import sys
 import os
 import itertools
@@ -10,22 +11,60 @@ import which
 
 from collections import defaultdict
 
+import ConfigParser
+
+
+def parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.')
+    parser.add_argument('-p', dest='platforms', action="append",
+                        help='Platforms to run. (required if not found in the environment)')
+    parser.add_argument('-u', dest='tests', action="append",
+                        help='Test suites to run in their entirety')
+    parser.add_argument('-b', dest='builds', default='do',
+                        help='Build types to run (d for debug, o for optimized)')
+    parser.add_argument('--tag', dest='tags', action='append',
+                        help='Restrict tests to the given tag (may be specified multiple times)')
+    parser.add_argument('--and', action='store_true', dest="intersection",
+                        help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments')
+    parser.add_argument('--no-push', dest='push', action='store_false',
+                        help='Do not push to try as a result of running this command (if '
+                        'specified this command will only print calculated try '
+                        'syntax and selection info).')
+    parser.add_argument('extra_args', nargs=argparse.REMAINDER,
+                        help='Extra arguments to put in the try push')
+    parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False,
+                        help='Print detailed information about the resulting test selection '
+                        'and commands performed.')
+    return parser
 
-TRY_SYNTAX_TMPL = """
-try: -b %s -p %s -u %s -t none %s %s
-"""
 
 class AutoTry(object):
 
-    test_flavors = [
-        'browser-chrome',
-        'chrome',
-        'devtools-chrome',
-        'mochitest',
-        'xpcshell',
-        'reftest',
-        'crashtest',
-    ]
+    # Maps from flavors to the job names needed to run that flavour
+    flavor_jobs = {
+        'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
+        'xpcshell': ['xpcshell'],
+        'chrome': ['mochitest-o'],
+        'browser-chrome': ['mochitest-browser-chrome-1',
+                           'mochitest-e10s-browser-chrome-1'],
+        'devtools-chrome': ['mochitest-dt',
+                            'mochitest-e10s-devtools-chrome'],
+        'crashtest': ['crashtest', 'crashtest-e10s'],
+        'reftest': ['reftest', 'reftest-e10s'],
+        'web-platform-tests': ['web-platform-tests-1'],
+    }
+
+    flavor_suites = {
+        "mochitest": "mochitests",
+        "xpcshell": "xpcshell",
+        "chrome": "mochitest-o",
+        "browser-chrome": "mochitest-bc",
+        "devtools-chrome": "mochitest-dt",
+        "crashtest": "crashtest",
+        "reftest": "reftest",
+        "web-platform-tests": "web-platform-tests",
+    }
 
     def __init__(self, topsrcdir, resolver, mach_context):
         self.topsrcdir = topsrcdir
@@ -37,57 +76,69 @@ class AutoTry(object):
         else:
             self._use_git = True
 
-    def resolve_manifests(self, paths=None, tags=None):
-        if not (paths or tags):
-            return {}
+    def paths_by_flavor(self, paths=None, tags=None):
+        paths_by_flavor = defaultdict(set)
 
-        tests = list(self.resolver.resolve_tests(tags=tags,
-                                                 paths=paths,
-                                                 cwd=self.mach_context.cwd))
-        manifests_by_flavor = defaultdict(set)
+        if not (paths or tags):
+            return dict(paths_by_flavor)
+
+        tests = list(self.resolver.resolve_tests(paths=paths,
+                                                 tags=tags))
 
         for t in tests:
-            if t['flavor'] in AutoTry.test_flavors:
+            if t['flavor'] in self.flavor_suites:
                 flavor = t['flavor']
                 if 'subsuite' in t and t['subsuite'] == 'devtools':
                     flavor = 'devtools-chrome'
-                manifest = os.path.relpath(t['manifest'], self.topsrcdir)
-                manifests_by_flavor[flavor].add(manifest)
 
-        return dict(manifests_by_flavor)
+                for path in paths:
+                    if flavor in ["crashtest", "reftest"]:
+                        manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
+                        if manifest_relpath.startswith(path):
+                            paths_by_flavor[flavor].add(manifest_relpath)
+                    else:
+                        if t['file_relpath'].startswith(path):
+                            paths_by_flavor[flavor].add(path)
 
-    def calc_try_syntax(self, platforms, flavors, tests, extra_tests, builds,
-                        manifests, tags):
+        return dict(paths_by_flavor)
 
-        # Maps from flavors to the try syntax snippets implied by that flavor.
-        # TODO: put selected tests under their own builder/label to avoid
-        # confusion reading results on treeherder.
-        flavor_suites = {
-            'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
-            'xpcshell': ['xpcshell'],
-            'chrome': ['mochitest-o'],
-            'browser-chrome': ['mochitest-browser-chrome-1',
-                               'mochitest-e10s-browser-chrome-1'],
-            'devtools-chrome': ['mochitest-dt',
-                                'mochitest-e10s-devtools-chrome'],
-            'crashtest': ['crashtest', 'crashtest-e10s'],
-            'reftest': ['reftest', 'reftest-e10s'],
-        }
+    def remove_duplicates(self, paths_by_flavor, tests):
+        rv = {}
+        for item in paths_by_flavor:
+            if self.flavor_suites[item] not in tests:
+                rv[item] = paths_by_flavor[item].copy()
+        return rv
+
+    def calc_try_syntax(self, platforms, tests, builds, paths_by_flavor, tags, extra_args,
+                        intersection):
+        parts = ["try:", "-b", builds, "-p", ",".join(platforms)]
+
+        suites = set(tests) if not intersection else set()
+        paths = set()
+        for flavor, flavor_tests in paths_by_flavor.iteritems():
+            suite = self.flavor_suites[flavor]
+            if suite not in suites and (not intersection or suite in tests):
+                for job_name in self.flavor_jobs[flavor]:
+                    for test in flavor_tests:
+                        paths.add("%s:%s" % (flavor, test))
+                    suites.add(job_name)
+
+        if not suites:
+            raise ValueError("No tests found matching filters")
+
+        parts.append("-u")
+        parts.append(",".join(sorted(suites)))
 
         if tags:
-            tags = ' '.join('--tag %s' % t for t in tags)
-        else:
-            tags = ''
+            parts.append(' '.join('--tag %s' % t for t in tags))
 
-        if not tests:
-            tests = ','.join(itertools.chain(*(flavor_suites[f] for f in flavors)))
-            if extra_tests:
-                tests += ',%s' % (extra_tests)
+        if extra_args is not None:
+            parts.extend(extra_args)
 
-        manifests = ' '.join(manifests)
-        if manifests:
-            manifests = '--try-test-paths %s' % manifests
-        return TRY_SYNTAX_TMPL % (builds, platforms, tests, manifests, tags)
+        if paths:
+            parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
+
+        return " ".join(parts)
 
     def _run_git(self, *args):
         args = ['git'] + list(args)