Bug 1288225 - Exclude eslint-plugin-mozilla from tooltool. r=ahal

This removes the in-tree plugin from the tooltool archive and uses that code
directly from the Gecko checkout instead.

For automation, we now get ESLint and external plugins from tooltool and then
symbolic link to the in-tree plugin.

For local development, we install ESLint and external plugins following the
shrinkwrap file created from the last change to the tooltool archive.  The local
plugin is then installed.

This change also removes the list of module versions from mach_commands.py, so
there is only one place to update module versions for the future.

MozReview-Commit-ID: AhbZ8lVPmN4

taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml

@@ -29,6 +29,7 @@ task:
           /build/tooltool.py fetch -m manifest.tt &&
           tar xvfz eslint.tar.gz &&
           rm eslint.tar.gz &&
+          ln -s ../eslint-plugin-mozilla node_modules &&
           cd ../../.. &&
           tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
 

tools/lint/eslint/manifest.tt

@@ -1,8 +1,8 @@
 [
 {
-"size": 2349680,
+"size": 2371391,
 "visibility": "public",
-"digest": "2b02ae6dd4bc735990660f97a831f05e604c28120977e4120cf59619fb02be22cbd42be26ec2bd176f172f4566f3dfb445082e8d9651346662b8fb8fde407b8c",
+"digest": "31421d2c6dbf6d8ab6ee45f29b191d1932b7f84a5f8a9cdf24d071a39be75c7013c7db07d79921cbb300678820bd0abca600edbd0b51bf907f0a7eafd3f91382",
 "algorithm": "sha512",
 "filename": "eslint.tar.gz"
 }

tools/lint/eslint/npm-shrinkwrap.json

@@ -1,9 +1,10 @@
 {
+  "name": "mach-eslint",
   "dependencies": {
     "acorn": {
-      "version": "3.2.0",
+      "version": "3.3.0",
       "from": "acorn@>=3.1.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.2.0.tgz"
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz"
     },
     "acorn-jsx": {
       "version": "3.0.1",
@@ -206,45 +207,35 @@
     },
     "escope": {
       "version": "3.6.0",
-      "from": "escope@>=3.6.0 <4.0.0",
+      "from": "escope@>=3.2.0 <4.0.0",
       "resolved": "https://registry.npmjs.org/escope/-/escope-3.6.0.tgz"
     },
     "eslint": {
       "version": "2.9.0",
       "from": "eslint@2.9.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-2.9.0.tgz"
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-2.9.0.tgz",
+      "dependencies": {
+        "espree": {
+          "version": "3.1.4",
+          "from": "espree@3.1.4",
+          "resolved": "https://registry.npmjs.org/espree/-/espree-3.1.4.tgz"
+        }
+      }
     },
     "eslint-plugin-html": {
       "version": "1.4.0",
       "from": "eslint-plugin-html@1.4.0",
       "resolved": "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-1.4.0.tgz"
     },
-    "eslint-plugin-mozilla": {
-      "version": "0.2.0",
-      "from": "eslint-plugin-mozilla",
-      "resolved": "file:eslint-plugin-mozilla",
-      "dependencies": {
-        "espree": {
-          "version": "2.2.5",
-          "from": "espree@>=2.2.4 <3.0.0",
-          "resolved": "https://registry.npmjs.org/espree/-/espree-2.2.5.tgz"
-        }
-      }
-    },
     "eslint-plugin-react": {
       "version": "4.2.3",
       "from": "eslint-plugin-react@4.2.3",
       "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-4.2.3.tgz"
     },
     "espree": {
-      "version": "3.1.4",
+      "version": "2.2.5",
-      "from": "espree@3.1.4",
+      "from": "espree@>=2.2.4 <3.0.0",
-      "resolved": "https://registry.npmjs.org/espree/-/espree-3.1.4.tgz"
+      "resolved": "https://registry.npmjs.org/espree/-/espree-2.2.5.tgz"
-    },
-    "esprima": {
-      "version": "2.7.2",
-      "from": "esprima@>=2.6.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.2.tgz"
     },
     "esrecurse": {
       "version": "4.1.0",
@@ -260,7 +251,7 @@
     },
     "estraverse": {
       "version": "4.2.0",
-      "from": "estraverse@>=4.2.0 <5.0.0",
+      "from": "estraverse@>=4.1.1 <5.0.0",
       "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz"
     },
     "esutils": {
@@ -329,9 +320,9 @@
       "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz"
     },
     "graceful-fs": {
-      "version": "4.1.4",
+      "version": "4.1.5",
       "from": "graceful-fs@>=4.1.2 <5.0.0",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.4.tgz"
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.5.tgz"
     },
     "has-ansi": {
       "version": "2.0.0",
@@ -411,7 +402,14 @@
     "js-yaml": {
       "version": "3.6.1",
       "from": "js-yaml@>=3.5.1 <4.0.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz"
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz",
+      "dependencies": {
+        "esprima": {
+          "version": "2.7.2",
+          "from": "esprima@>=2.6.0 <3.0.0",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.2.tgz"
+        }
+      }
     },
     "json-stable-stringify": {
       "version": "1.0.1",
@@ -434,9 +432,9 @@
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz"
     },
     "lodash": {
-      "version": "4.13.1",
+      "version": "4.14.0",
       "from": "lodash@>=4.0.0 <5.0.0",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.13.1.tgz"
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.14.0.tgz"
     },
     "minimatch": {
       "version": "3.0.2",

tools/lint/eslint/package.json

@@ -1,12 +1,15 @@
 {
-  "name": "",
+  "name": "mach-eslint",
-  "description": "None",
+  "description": "ESLint and external plugins for use with mach",
   "repository": {},
   "license": "MPL-2.0",
   "dependencies": {
-    "eslint": "*",
+    "eslint": "2.9.0",
-    "eslint-plugin-html": "*",
+    "eslint-plugin-html": "1.4.0",
-    "eslint-plugin-mozilla": "*",
+    "eslint-plugin-react": "4.2.3",
-    "eslint-plugin-react": "*"
+    "escope": "^3.2.0",
+    "espree": "^2.2.4",
+    "estraverse": "^4.1.1",
+    "sax": "^1.1.4"
   }
 }

tools/lint/eslint/update

@@ -37,8 +37,13 @@ echo "Removing node_modules and npm_shrinkwrap.json..."
 rm -rf node_modules/
 rm npm-shrinkwrap.json
 
-echo "Installing eslint and dependencies..."
+echo "Installing eslint and external plugins..."
-../../../mach eslint --setup
+# ESLint and all _external_ plugins are listed in this directory's package.json,
+# so a regular `npm install` will install them at the specified versions.
+# The in-tree eslint-plugin-mozilla is kept out of this tooltool archive on
+# purpose so that it can be changed by any developer without requiring tooltool
+# access to make changes.
+npm install
 
 echo "Creating npm shrinkwrap..."
 npm shrinkwrap

tools/lint/mach_commands.py

@@ -9,6 +9,7 @@ import json
 import logging
 import os
 import platform
+import re
 import subprocess
 import sys
 import which
@@ -29,13 +30,6 @@ from mach.decorators import (
 here = os.path.abspath(os.path.dirname(__file__))
 
 
-ESLINT_PACKAGES = [
-    "eslint@2.9.0",
-    "eslint-plugin-html@1.4.0",
-    "eslint-plugin-mozilla@0.2.0",
-    "eslint-plugin-react@4.2.3"
-]
-
 ESLINT_NOT_FOUND_MESSAGE = '''
 Could not find eslint!  We looked at the --binary option, at the ESLINT
 environment variable, and then at your local node_modules path. Please Install
@@ -61,6 +55,9 @@ option in the node installation) and try again.
 Valid installation paths:
 '''.strip()
 
+VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$")
+CARET_VERSION_RANGE_RE = re.compile(r"^\^((\d+)\.\d+\.\d+)$")
+
 
 def setup_argument_parser():
     from mozlint import cli
@@ -84,7 +81,7 @@ class MachCommands(MachCommandBase):
     @Command('eslint', category='devenv',
              description='Run eslint or help configure eslint for optimal development.')
     @CommandArgument('-s', '--setup', default=False, action='store_true',
-                     help='configure eslint for optimal development.')
+                     help='Configure eslint for optimal development.')
     @CommandArgument('-e', '--ext', default='[.js,.jsm,.jsx,.xml,.html]',
                      help='Filename extensions to lint, default: "[.js,.jsm,.jsx,.xml,.html]".')
     @CommandArgument('-b', '--binary', default=None,
@@ -103,8 +100,8 @@ class MachCommands(MachCommandBase):
         if setup:
             return self.eslint_setup()
 
-        npmPath = self.get_node_or_npm_path("npm")
+        npm_path = self.get_node_or_npm_path("npm")
-        if not npmPath:
+        if not npm_path:
             return 1
 
         if self.eslint_module_has_issues():
@@ -162,7 +159,7 @@ class MachCommands(MachCommandBase):
                  'Finished eslint. {msg} encountered.')
         return success
 
-    def eslint_setup(self, update_only=False):
+    def eslint_setup(self):
         """Ensure eslint is optimally configured.
 
         This command will inspect your eslint configuration and
@@ -178,30 +175,22 @@ class MachCommands(MachCommandBase):
         # we manually switch folders here instead.
         os.chdir(module_path)
 
-        npmPath = self.get_node_or_npm_path("npm")
+        npm_path = self.get_node_or_npm_path("npm")
-        if not npmPath:
+        if not npm_path:
             return 1
 
-        # Install eslint and necessary plugins.
+        # Install ESLint and external plugins
-        for pkg in ESLINT_PACKAGES:
+        cmd = [npm_path, "install"]
-            name, version = pkg.split("@")
+        print("Installing eslint for mach using \"%s\"..." % (" ".join(cmd)))
-            success = False
+        if not self.call_process("eslint", cmd):
+            return 1
 
-            if self.node_package_installed(pkg, cwd=module_path):
+        # Install in-tree ESLint plugin
-                success = True
+        cmd = [npm_path, "install",
-            else:
+               os.path.join(module_path, "eslint-plugin-mozilla")]
-                if pkg.startswith("eslint-plugin-mozilla"):
+        print("Installing eslint-plugin-mozilla using \"%s\"..." % (" ".join(cmd)))
-                    cmd = [npmPath, "install",
+        if not self.call_process("eslint-plugin-mozilla", cmd):
-                           os.path.join(module_path, "eslint-plugin-mozilla")]
+            return 1
-                else:
-                    cmd = [npmPath, "install", pkg]
-
-                print("Installing %s v%s using \"%s\"..."
-                      % (name, version, " ".join(cmd)))
-                success = self.call_process(pkg, cmd)
-
-            if not success:
-                return 1
 
         eslint_path = os.path.join(module_path, "node_modules", ".bin", "eslint")
 
@@ -224,42 +213,67 @@ class MachCommands(MachCommandBase):
 
         return True
 
+    def expected_eslint_modules(self):
+        # Read the expected version of ESLint and external modules
+        expected_modules_path = os.path.join(self.get_eslint_module_path(), "package.json")
+        with open(expected_modules_path, "r") as f:
+            expected_modules = json.load(f)["dependencies"]
+
+        # Also read the in-tree ESLint plugin version
+        mozilla_json_path = os.path.join(self.get_eslint_module_path(),
+                                         "eslint-plugin-mozilla", "package.json")
+        with open(mozilla_json_path, "r") as f:
+            expected_modules["eslint-plugin-mozilla"] = json.load(f)["version"]
+
+        return expected_modules
+
     def eslint_module_has_issues(self):
         has_issues = False
-        node_module_path = os.path.join(self.get_eslint_module_path(), "node_modules")
+        node_modules_path = os.path.join(self.get_eslint_module_path(), "node_modules")
 
-        for pkg in ESLINT_PACKAGES:
+        for name, version_range in self.expected_eslint_modules().iteritems():
-            name, req_version = pkg.split("@")
+            path = os.path.join(node_modules_path, name, "package.json")
-            path = os.path.join(node_module_path, name, "package.json")
 
             if not os.path.exists(path):
-                print("%s v%s needs to be installed locally." % (name, req_version))
+                print("%s v%s needs to be installed locally." % (name, version_range))
                 has_issues = True
                 continue
 
             data = json.load(open(path))
 
-            if data["version"] != req_version:
+            if not self.version_in_range(data["version"], version_range):
-                print("%s v%s should be v%s." % (name, data["version"], req_version))
+                print("%s v%s should be v%s." % (name, data["version"], version_range))
                 has_issues = True
 
         return has_issues
 
-    def node_package_installed(self, package_name="", globalInstall=False, cwd=None):
+    def version_in_range(self, version, version_range):
-        try:
+        """
-            npmPath = self.get_node_or_npm_path("npm")
+        Check if a module version is inside a version range.  Only supports explicit versions and
-
+        caret ranges for the moment, since that's all we've used so far.
-            cmd = [npmPath, "ls", "--parseable", package_name]
+        """
-
+        if version == version_range:
-            if globalInstall:
-                cmd.append("-g")
-
-            with open(os.devnull, "w") as fnull:
-                subprocess.check_call(cmd, stdout=fnull, stderr=fnull, cwd=cwd)
-
             return True
-        except subprocess.CalledProcessError:
+
-            return False
+        version_match = VERSION_RE.match(version)
+        if not version_match:
+            raise RuntimeError("mach eslint doesn't understand module version %s" % version)
+        version = LooseVersion(version)
+
+        # Caret ranges as specified by npm allow changes that do not modify the left-most non-zero
+        # digit in the [major, minor, patch] tuple.  The code below assumes the major digit is
+        # non-zero.
+        range_match = CARET_VERSION_RANGE_RE.match(version_range)
+        if range_match:
+            range_version = range_match.group(1)
+            range_major = int(range_match.group(2))
+
+            range_min = LooseVersion(range_version)
+            range_max = LooseVersion("%d.0.0" % (range_major + 1))
+
+            return range_min <= version < range_max
+
+        return False
 
     def get_possible_node_paths_win(self):
         """
@@ -282,17 +296,17 @@ class MachCommands(MachCommandBase):
         if platform.system() == "Windows":
             for ext in [".cmd", ".exe", ""]:
                 try:
-                    nodeOrNpmPath = which.which(filename + ext,
+                    node_or_npm_path = which.which(filename + ext,
-                                                path=self.get_possible_node_paths_win())
+                                                   path=self.get_possible_node_paths_win())
-                    if self.is_valid(nodeOrNpmPath, minversion):
+                    if self.is_valid(node_or_npm_path, minversion):
-                        return nodeOrNpmPath
+                        return node_or_npm_path
                 except which.WhichError:
                     pass
         else:
             try:
-                nodeOrNpmPath = which.which(filename)
+                node_or_npm_path = which.which(filename)
-                if self.is_valid(nodeOrNpmPath, minversion):
+                if self.is_valid(node_or_npm_path, minversion):
-                    return nodeOrNpmPath
+                    return node_or_npm_path
             except which.WhichError:
                 pass