From c9822d13cd56769b3fd6dd37e23ad72f186bac5d Mon Sep 17 00:00:00 2001 From: "J. Ryan Stinnett" Date: Fri, 22 Jul 2016 18:18:58 -0500 Subject: [PATCH] Bug 1288225 - Exclude eslint-plugin-mozilla from tooltool. r=ahal This removes the in-tree plugin from the tooltool archive and uses that code directly from the Gecko checkout instead. For automation, we now get ESLint and external plugins from tooltool and then symbolic link to the in-tree plugin. For local development, we install ESLint and external plugins following the shrinkwrap file created from the last change to the tooltool archive. The local plugin is then installed. This change also removes the list of module versions from mach_commands.py, so there is only one place to update module versions for the future. MozReview-Commit-ID: AhbZ8lVPmN4 --- .../ci/legacy/tasks/tests/eslint-gecko.yml | 1 + tools/lint/eslint/manifest.tt | 4 +- tools/lint/eslint/npm-shrinkwrap.json | 58 ++++---- tools/lint/eslint/package.json | 15 +- tools/lint/eslint/update | 9 +- tools/lint/mach_commands.py | 134 ++++++++++-------- 6 files changed, 121 insertions(+), 100 deletions(-) diff --git a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml index 9dea3ede0394..fd59e2a25df6 100644 --- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml +++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml @@ -29,6 +29,7 @@ task: /build/tooltool.py fetch -m manifest.tt && tar xvfz eslint.tar.gz && rm eslint.tar.gz && + ln -s ../eslint-plugin-mozilla node_modules && cd ../../.. && tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter . diff --git a/tools/lint/eslint/manifest.tt b/tools/lint/eslint/manifest.tt index 1eabd483c5c2..6f70b55ff2e8 100644 --- a/tools/lint/eslint/manifest.tt +++ b/tools/lint/eslint/manifest.tt @@ -1,8 +1,8 @@ [ { -"size": 2349680, +"size": 2371391, "visibility": "public", -"digest": "2b02ae6dd4bc735990660f97a831f05e604c28120977e4120cf59619fb02be22cbd42be26ec2bd176f172f4566f3dfb445082e8d9651346662b8fb8fde407b8c", +"digest": "31421d2c6dbf6d8ab6ee45f29b191d1932b7f84a5f8a9cdf24d071a39be75c7013c7db07d79921cbb300678820bd0abca600edbd0b51bf907f0a7eafd3f91382", "algorithm": "sha512", "filename": "eslint.tar.gz" } diff --git a/tools/lint/eslint/npm-shrinkwrap.json b/tools/lint/eslint/npm-shrinkwrap.json index deec183be73c..bfcd08833456 100644 --- a/tools/lint/eslint/npm-shrinkwrap.json +++ b/tools/lint/eslint/npm-shrinkwrap.json @@ -1,9 +1,10 @@ { + "name": "mach-eslint", "dependencies": { "acorn": { - "version": "3.2.0", + "version": "3.3.0", "from": "acorn@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.2.0.tgz" + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz" }, "acorn-jsx": { "version": "3.0.1", @@ -206,45 +207,35 @@ }, "escope": { "version": "3.6.0", - "from": "escope@>=3.6.0 <4.0.0", + "from": "escope@>=3.2.0 <4.0.0", "resolved": "https://registry.npmjs.org/escope/-/escope-3.6.0.tgz" }, "eslint": { "version": "2.9.0", "from": "eslint@2.9.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-2.9.0.tgz" + "resolved": "https://registry.npmjs.org/eslint/-/eslint-2.9.0.tgz", + "dependencies": { + "espree": { + "version": "3.1.4", + "from": "espree@3.1.4", + "resolved": "https://registry.npmjs.org/espree/-/espree-3.1.4.tgz" + } + } }, "eslint-plugin-html": { "version": "1.4.0", "from": "eslint-plugin-html@1.4.0", "resolved": "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-1.4.0.tgz" }, - "eslint-plugin-mozilla": { - "version": "0.2.0", - "from": "eslint-plugin-mozilla", - "resolved": "file:eslint-plugin-mozilla", - "dependencies": { - "espree": { - "version": "2.2.5", - "from": "espree@>=2.2.4 <3.0.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-2.2.5.tgz" - } - } - }, "eslint-plugin-react": { "version": "4.2.3", "from": "eslint-plugin-react@4.2.3", "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-4.2.3.tgz" }, "espree": { - "version": "3.1.4", - "from": "espree@3.1.4", - "resolved": "https://registry.npmjs.org/espree/-/espree-3.1.4.tgz" - }, - "esprima": { - "version": "2.7.2", - "from": "esprima@>=2.6.0 <3.0.0", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.2.tgz" + "version": "2.2.5", + "from": "espree@>=2.2.4 <3.0.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-2.2.5.tgz" }, "esrecurse": { "version": "4.1.0", @@ -260,7 +251,7 @@ }, "estraverse": { "version": "4.2.0", - "from": "estraverse@>=4.2.0 <5.0.0", + "from": "estraverse@>=4.1.1 <5.0.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz" }, "esutils": { @@ -329,9 +320,9 @@ "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz" }, "graceful-fs": { - "version": "4.1.4", + "version": "4.1.5", "from": "graceful-fs@>=4.1.2 <5.0.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.4.tgz" + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.5.tgz" }, "has-ansi": { "version": "2.0.0", @@ -411,7 +402,14 @@ "js-yaml": { "version": "3.6.1", "from": "js-yaml@>=3.5.1 <4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz" + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz", + "dependencies": { + "esprima": { + "version": "2.7.2", + "from": "esprima@>=2.6.0 <3.0.0", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.2.tgz" + } + } }, "json-stable-stringify": { "version": "1.0.1", @@ -434,9 +432,9 @@ "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz" }, "lodash": { - "version": "4.13.1", + "version": "4.14.0", "from": "lodash@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.13.1.tgz" + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.14.0.tgz" }, "minimatch": { "version": "3.0.2", diff --git a/tools/lint/eslint/package.json b/tools/lint/eslint/package.json index 415b3418d12c..4b6bb7d195ea 100644 --- a/tools/lint/eslint/package.json +++ b/tools/lint/eslint/package.json @@ -1,12 +1,15 @@ { - "name": "", - "description": "None", + "name": "mach-eslint", + "description": "ESLint and external plugins for use with mach", "repository": {}, "license": "MPL-2.0", "dependencies": { - "eslint": "*", - "eslint-plugin-html": "*", - "eslint-plugin-mozilla": "*", - "eslint-plugin-react": "*" + "eslint": "2.9.0", + "eslint-plugin-html": "1.4.0", + "eslint-plugin-react": "4.2.3", + "escope": "^3.2.0", + "espree": "^2.2.4", + "estraverse": "^4.1.1", + "sax": "^1.1.4" } } diff --git a/tools/lint/eslint/update b/tools/lint/eslint/update index f8b47d789642..47758423644d 100755 --- a/tools/lint/eslint/update +++ b/tools/lint/eslint/update @@ -37,8 +37,13 @@ echo "Removing node_modules and npm_shrinkwrap.json..." rm -rf node_modules/ rm npm-shrinkwrap.json -echo "Installing eslint and dependencies..." -../../../mach eslint --setup +echo "Installing eslint and external plugins..." +# ESLint and all _external_ plugins are listed in this directory's package.json, +# so a regular `npm install` will install them at the specified versions. +# The in-tree eslint-plugin-mozilla is kept out of this tooltool archive on +# purpose so that it can be changed by any developer without requiring tooltool +# access to make changes. +npm install echo "Creating npm shrinkwrap..." npm shrinkwrap diff --git a/tools/lint/mach_commands.py b/tools/lint/mach_commands.py index 4a075673e8dc..d11534abd296 100644 --- a/tools/lint/mach_commands.py +++ b/tools/lint/mach_commands.py @@ -9,6 +9,7 @@ import json import logging import os import platform +import re import subprocess import sys import which @@ -29,13 +30,6 @@ from mach.decorators import ( here = os.path.abspath(os.path.dirname(__file__)) -ESLINT_PACKAGES = [ - "eslint@2.9.0", - "eslint-plugin-html@1.4.0", - "eslint-plugin-mozilla@0.2.0", - "eslint-plugin-react@4.2.3" -] - ESLINT_NOT_FOUND_MESSAGE = ''' Could not find eslint! We looked at the --binary option, at the ESLINT environment variable, and then at your local node_modules path. Please Install @@ -61,6 +55,9 @@ option in the node installation) and try again. Valid installation paths: '''.strip() +VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") +CARET_VERSION_RANGE_RE = re.compile(r"^\^((\d+)\.\d+\.\d+)$") + def setup_argument_parser(): from mozlint import cli @@ -84,7 +81,7 @@ class MachCommands(MachCommandBase): @Command('eslint', category='devenv', description='Run eslint or help configure eslint for optimal development.') @CommandArgument('-s', '--setup', default=False, action='store_true', - help='configure eslint for optimal development.') + help='Configure eslint for optimal development.') @CommandArgument('-e', '--ext', default='[.js,.jsm,.jsx,.xml,.html]', help='Filename extensions to lint, default: "[.js,.jsm,.jsx,.xml,.html]".') @CommandArgument('-b', '--binary', default=None, @@ -103,8 +100,8 @@ class MachCommands(MachCommandBase): if setup: return self.eslint_setup() - npmPath = self.get_node_or_npm_path("npm") - if not npmPath: + npm_path = self.get_node_or_npm_path("npm") + if not npm_path: return 1 if self.eslint_module_has_issues(): @@ -162,7 +159,7 @@ class MachCommands(MachCommandBase): 'Finished eslint. {msg} encountered.') return success - def eslint_setup(self, update_only=False): + def eslint_setup(self): """Ensure eslint is optimally configured. This command will inspect your eslint configuration and @@ -178,30 +175,22 @@ class MachCommands(MachCommandBase): # we manually switch folders here instead. os.chdir(module_path) - npmPath = self.get_node_or_npm_path("npm") - if not npmPath: + npm_path = self.get_node_or_npm_path("npm") + if not npm_path: return 1 - # Install eslint and necessary plugins. - for pkg in ESLINT_PACKAGES: - name, version = pkg.split("@") - success = False + # Install ESLint and external plugins + cmd = [npm_path, "install"] + print("Installing eslint for mach using \"%s\"..." % (" ".join(cmd))) + if not self.call_process("eslint", cmd): + return 1 - if self.node_package_installed(pkg, cwd=module_path): - success = True - else: - if pkg.startswith("eslint-plugin-mozilla"): - cmd = [npmPath, "install", - os.path.join(module_path, "eslint-plugin-mozilla")] - else: - cmd = [npmPath, "install", pkg] - - print("Installing %s v%s using \"%s\"..." - % (name, version, " ".join(cmd))) - success = self.call_process(pkg, cmd) - - if not success: - return 1 + # Install in-tree ESLint plugin + cmd = [npm_path, "install", + os.path.join(module_path, "eslint-plugin-mozilla")] + print("Installing eslint-plugin-mozilla using \"%s\"..." % (" ".join(cmd))) + if not self.call_process("eslint-plugin-mozilla", cmd): + return 1 eslint_path = os.path.join(module_path, "node_modules", ".bin", "eslint") @@ -224,42 +213,67 @@ class MachCommands(MachCommandBase): return True + def expected_eslint_modules(self): + # Read the expected version of ESLint and external modules + expected_modules_path = os.path.join(self.get_eslint_module_path(), "package.json") + with open(expected_modules_path, "r") as f: + expected_modules = json.load(f)["dependencies"] + + # Also read the in-tree ESLint plugin version + mozilla_json_path = os.path.join(self.get_eslint_module_path(), + "eslint-plugin-mozilla", "package.json") + with open(mozilla_json_path, "r") as f: + expected_modules["eslint-plugin-mozilla"] = json.load(f)["version"] + + return expected_modules + def eslint_module_has_issues(self): has_issues = False - node_module_path = os.path.join(self.get_eslint_module_path(), "node_modules") + node_modules_path = os.path.join(self.get_eslint_module_path(), "node_modules") - for pkg in ESLINT_PACKAGES: - name, req_version = pkg.split("@") - path = os.path.join(node_module_path, name, "package.json") + for name, version_range in self.expected_eslint_modules().iteritems(): + path = os.path.join(node_modules_path, name, "package.json") if not os.path.exists(path): - print("%s v%s needs to be installed locally." % (name, req_version)) + print("%s v%s needs to be installed locally." % (name, version_range)) has_issues = True continue data = json.load(open(path)) - if data["version"] != req_version: - print("%s v%s should be v%s." % (name, data["version"], req_version)) + if not self.version_in_range(data["version"], version_range): + print("%s v%s should be v%s." % (name, data["version"], version_range)) has_issues = True return has_issues - def node_package_installed(self, package_name="", globalInstall=False, cwd=None): - try: - npmPath = self.get_node_or_npm_path("npm") - - cmd = [npmPath, "ls", "--parseable", package_name] - - if globalInstall: - cmd.append("-g") - - with open(os.devnull, "w") as fnull: - subprocess.check_call(cmd, stdout=fnull, stderr=fnull, cwd=cwd) - + def version_in_range(self, version, version_range): + """ + Check if a module version is inside a version range. Only supports explicit versions and + caret ranges for the moment, since that's all we've used so far. + """ + if version == version_range: return True - except subprocess.CalledProcessError: - return False + + version_match = VERSION_RE.match(version) + if not version_match: + raise RuntimeError("mach eslint doesn't understand module version %s" % version) + version = LooseVersion(version) + + # Caret ranges as specified by npm allow changes that do not modify the left-most non-zero + # digit in the [major, minor, patch] tuple. The code below assumes the major digit is + # non-zero. + range_match = CARET_VERSION_RANGE_RE.match(version_range) + if range_match: + range_version = range_match.group(1) + range_major = int(range_match.group(2)) + + range_min = LooseVersion(range_version) + range_max = LooseVersion("%d.0.0" % (range_major + 1)) + + return range_min <= version < range_max + + return False def get_possible_node_paths_win(self): """ @@ -282,17 +296,17 @@ class MachCommands(MachCommandBase): if platform.system() == "Windows": for ext in [".cmd", ".exe", ""]: try: - nodeOrNpmPath = which.which(filename + ext, - path=self.get_possible_node_paths_win()) - if self.is_valid(nodeOrNpmPath, minversion): - return nodeOrNpmPath + node_or_npm_path = which.which(filename + ext, + path=self.get_possible_node_paths_win()) + if self.is_valid(node_or_npm_path, minversion): + return node_or_npm_path except which.WhichError: pass else: try: - nodeOrNpmPath = which.which(filename) - if self.is_valid(nodeOrNpmPath, minversion): - return nodeOrNpmPath + node_or_npm_path = which.which(filename) + if self.is_valid(node_or_npm_path, minversion): + return node_or_npm_path except which.WhichError: pass