bug 1540655: remote: add mach command for running Puppeteer tests; r=remote-protocol-reviewers,jdescottes

Introduces "./mach puppeteer-test" command for running the Puppeteer
tests against the remote agent.  This has to be a top-level command
because the automatic test detection system in mach does not allow
us to delegate to a subcommand such as "./mach remote puppeteer-test".

The tests run against a fork of Puppeteer with hotfixes needed for
it to work  with the CDP implementation in Firefox.  This fork is
located at https://github.com/andreastt/puppeteer/tree/firefox, and
vendored under remote/test/puppeteer/ in a previous commit in this series.

Differential Revision: https://phabricator.services.mozilla.com/D37009

--HG--
extra : moz-landing-system : lando

remote/mach_commands.py

@@ -21,7 +21,12 @@ from mach.decorators import (
     SubCommand,
 )
 
-from mozbuild.base import MachCommandBase
+from mozbuild.base import (
+    MachCommandBase,
+    MozbuildObject,
+)
+from mozbuild import nodeutil
+import mozprofile
 
 
 EX_CONFIG = 78
@@ -32,6 +37,15 @@ DEFAULT_REPO = "https://github.com/andreastt/puppeteer.git"
 DEFAULT_COMMITISH = "firefox"
 
 
+def setup():
+    # add node and npm from mozbuild to front of system path
+    npm, _ = nodeutil.find_npm_executable()
+    if not npm:
+        exit(EX_CONFIG, "could not find npm executable")
+    path = os.path.abspath(os.path.join(npm, os.pardir))
+    os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
+
+
 @CommandProvider
 class RemoteCommands(MachCommandBase):
     def __init__(self, context):
@@ -125,6 +139,21 @@ def git(*args, **kwargs):
     return out
 
 
+def npm(*args, **kwargs):
+    env = None
+    if kwargs.get("env"):
+        env = os.environ.copy()
+        env.update(kwargs["env"])
+
+    p = subprocess.Popen(("npm",) + args,
+                         cwd=kwargs.get("cwd"),
+                         env=env)
+
+    p.wait()
+    if p.returncode > 0:
+        exit(p.returncode, "npm: exit code {}".format(p.returncode))
+
+
 # tempfile.TemporaryDirectory missing from Python 2.7
 class TemporaryDirectory(object):
     def __init__(self):
@@ -149,6 +178,129 @@ class TemporaryDirectory(object):
             self._closed = True
 
 
+class PuppeteerRunner(MozbuildObject):
+    def __init__(self, *args, **kwargs):
+        super(PuppeteerRunner, self).__init__(*args, **kwargs)
+
+        self.profile = mozprofile.Profile()
+
+        self.remotedir = os.path.join(self.topsrcdir, "remote")
+        self.puppeteerdir = os.path.join(self.remotedir, "test", "puppeteer")
+
+    def run_test(self, *tests, **params):
+        """
+        Runs Puppeteer unit tests with npm.
+
+        Possible optional test parameters:
+
+        `binary`:
+          Path for the browser binary to use.  Defaults to the local
+          build.
+        `jobs`:
+          Number of tests to run in parallel.  Defaults to not
+          parallelise, e.g. `-j1`.
+        `headless`:
+          Boolean to indicate whether to activate Firefox' headless mode.
+        `extra_prefs`:
+          Dictionary of extra preferences to write to the profile,
+          before invoking npm.  Overrides default preferences.
+        """
+        setup()
+
+        binary = params.get("binary") or self.get_binary_path()
+
+        # currently runs against puppeteer-chrome
+        # but future intention is to run against puppeteer-firefox
+        # when it targets the Mozilla remote agent instead of Juggler
+        env = {"CHROME": binary,
+               "DUMPIO": "1"}
+
+        if params.get("jobs"):
+            env["PPTR_PARALLEL_TESTS"] = str(params["jobs"])
+
+        if params.get("headless"):
+            env["MOZ_HEADLESS"] = "1"
+
+        prefs = params.get("extra_prefs", {})
+        for k, v in params.get("extra_prefs", {}).items():
+            prefs[k] = mozprofile.Preferences.cast(v)
+
+        prefs.update({
+            # https://bugzilla.mozilla.org/show_bug.cgi?id=1544393
+            "remote.enabled": True,
+            # https://bugzilla.mozilla.org/show_bug.cgi?id=1543115
+            "browser.dom.window.dump.enabled": True,
+        })
+
+        self.profile.set_preferences(prefs)
+
+        # PROFILE is a Puppeteer workaround (see ab302d6)
+        # for passing the --profile flag to Firefox
+        env["PROFILE"] = self.profile.profile
+
+        return npm("run", "unit", "--verbose", *tests,
+                   cwd=self.puppeteerdir,
+                   env=env)
+
+
+@CommandProvider
+class PuppeteerTest(MachCommandBase):
+    @Command("puppeteer-test", category="testing",
+             description="Run Puppeteer unit tests.")
+    @CommandArgument("--binary",
+                     type=str,
+                     help="Path to Firefox binary.  Defaults to local build.")
+    @CommandArgument("-z", "--headless",
+                     action="store_true",
+                     help="Run browser in headless mode (default).")
+    @CommandArgument("--setpref",
+                     action="append",
+                     dest="extra_prefs",
+                     metavar="<pref>=<value>",
+                     help="Defines additional user preferences.")
+    @CommandArgument("-j",
+                     dest="jobs",
+                     type=int,
+                     metavar="<N>",
+                     help="Optionally run tests in parallel.")
+    @CommandArgument("tests", nargs="*")
+    def puppeteer_test(self, binary=None, headless=True, extra_prefs=None,
+                       jobs=1, tests=None, **kwargs):
+        # moztest calls this programmatically with test objects or manifests
+        if "test_objects" in kwargs and tests is not None:
+            raise ValueError("Expected either 'test_objects' or 'tests'")
+
+        if "test_objects" in kwargs:
+            tests = []
+            for test in kwargs["test_objects"]:
+                tests.append(test["path"])
+
+        prefs = {}
+        for s in (extra_prefs or []):
+            kv = s.split("=")
+            if len(kv) != 2:
+                exit(EX_USAGE, "syntax error in --setpref={}".format(s))
+            prefs[kv[0]] = kv[1].strip()
+
+        self.install_puppeteer()
+
+        params = {"binary": binary,
+                  "headless": headless,
+                  "extra_prefs": prefs,
+                  "jobs": jobs}
+        puppeteer = self._spawn(PuppeteerRunner)
+        try:
+            return puppeteer.run_test(*tests, **params)
+        except Exception as e:
+            exit(EX_SOFTWARE, e)
+
+    def install_puppeteer(self):
+        setup()
+        npm("install",
+            cwd=os.path.join(self.topsrcdir, "remote", "test", "puppeteer"),
+            env={"PUPPETEER_SKIP_CHROMIUM_DOWNLOAD": "1"})
+
+
 def exit(code, error=None):
     if error is not None:
         if isinstance(error, Exception):