diff --git a/build/virtualenv_packages.txt b/build/virtualenv_packages.txt index dabca237ab45..7fab1dd9080c 100644 --- a/build/virtualenv_packages.txt +++ b/build/virtualenv_packages.txt @@ -23,6 +23,7 @@ mozilla.pth:third_party/python/distro mozilla.pth:third_party/python/dlmanager mozilla.pth:third_party/python/ecdsa/src mozilla.pth:third_party/python/enum34 +mozilla.pth:third_party/python/esprima mozilla.pth:third_party/python/fluent.migrate mozilla.pth:third_party/python/fluent.syntax mozilla.pth:third_party/python/funcsigs diff --git a/python/mozperftest/mozperftest/argparser.py b/python/mozperftest/mozperftest/argparser.py index bdc8cfac89bf..188a169b75a4 100644 --- a/python/mozperftest/mozperftest/argparser.py +++ b/python/mozperftest/mozperftest/argparser.py @@ -19,10 +19,18 @@ from mozperftest.system import get_layers as system_layers from mozperftest.browser import get_layers as browser_layers from mozperftest.metrics import get_layers as metrics_layers +FLAVORS = ["script", "doc"] + class Options: general_args = { + "--flavor": { + "choices": FLAVORS, + "metavar": "{{{}}}".format(", ".join(FLAVORS)), + "default": None, + "help": "Only run tests of this flavor.", + }, "tests": { "nargs": "*", "metavar": "TEST", diff --git a/python/mozperftest/mozperftest/mach_commands.py b/python/mozperftest/mozperftest/mach_commands.py index 6829181ef84b..7cda839761f3 100644 --- a/python/mozperftest/mozperftest/mach_commands.py +++ b/python/mozperftest/mozperftest/mach_commands.py @@ -1,6 +1,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import random from functools import partial from mach.decorators import CommandProvider, Command from mozbuild.base import MachCommandBase, MachCommandConditions as conditions @@ -14,6 +16,25 @@ def get_perftest_parser(): @CommandProvider class Perftest(MachCommandBase): + def _build_test_list(self, tests, randomized=False): + res = [] + for test in tests: + if os.path.isfile(test): + tests.append(test) + elif os.path.isdir(test): + for root, dirs, files in os.walk(test): + for file in files: + if not file.startswith("perftest"): + continue + res.append(os.path.join(root, file)) + if not randomized: + res.sort() + else: + # random shuffling is used to make sure + # we don't always run tests in the same order + random.shuffle(res) + return res + @Command( "perftest", category="testing", @@ -24,6 +45,23 @@ class Perftest(MachCommandBase): def run_perftest( self, flavor="script", test_objects=None, resolve_tests=True, **kwargs ): + + MachCommandBase._activate_virtualenv(self) + kwargs["tests"] = self._build_test_list( + kwargs["tests"], randomized=flavor != "doc" + ) + + if flavor == "doc": + from mozperftest.utils import install_package + + install_package(self.virtualenv_manager, "esprima") + + from mozperftest.scriptinfo import ScriptInfo + + for test in kwargs["tests"]: + print(ScriptInfo(test)) + return + from mozperftest import MachEnvironment, Metadata kwargs["test_objects"] = test_objects diff --git a/python/mozperftest/mozperftest/scriptinfo.py b/python/mozperftest/mozperftest/scriptinfo.py new file mode 100644 index 000000000000..1f3837c34f14 --- /dev/null +++ b/python/mozperftest/mozperftest/scriptinfo.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from collections import defaultdict +import re +import os +import textwrap + +# This import will fail if esprima is not installed. +# The package is vendored in python/third_party and also available at PyPI +# The mach perftest command will make sure it's installed when --flavor doc +# is being used. +import esprima + + +_INFO = """\ +%(filename)s +%(filename_underline)s + +%(description)s + +Owner: %(owner)s +Usage: +%(usage)s + +Description: +%(long_description)s +""" + + +class MetadataDict(defaultdict): + def __missing__(self, key): + return "N/A" + + +class ScriptInfo(MetadataDict): + def __init__(self, script): + super(ScriptInfo, self).__init__() + filename = os.path.basename(script) + self["filename"] = script, filename + self["filename_underline"] = None, "-" * len(filename) + self.script = script + with open(script) as f: + self.parsed = esprima.parseScript(f.read()) + + # looking for the exports statement + for stmt in self.parsed.body: + if ( + stmt.type != "ExpressionStatement" + or stmt.expression.left is None + or stmt.expression.left.property.name != "exports" + or stmt.expression.right is None + or stmt.expression.right.properties is None + ): + continue + + # now scanning the properties + for prop in stmt.expression.right.properties: + if prop.value.type == "Identifier": + value = prop.value.name + elif prop.value.type == "Literal": + value = prop.value.value + elif prop.value.type == "TemplateLiteral": + # ugly + value = prop.value.quasis[0].value.cooked.replace("\n", " ") + value = re.sub("\s+", " ", value).strip() + elif prop.value.type == "ArrayExpression": + value = [e.value for e in prop.value.elements] + else: + raise ValueError(prop.value.type) + # line wrapping + if isinstance(value, str): + repr = "\n".join(textwrap.wrap(value, break_on_hyphens=False)) + elif isinstance(value, list): + repr = ", ".join(value) + + self[prop.key.name] = value, repr + + def __str__(self): + reprs = dict((k, v[1]) for k, v in self.items()) + d = MetadataDict() + d.update(reprs) + return _INFO % d diff --git a/python/mozperftest/mozperftest/tests/example.js b/python/mozperftest/mozperftest/tests/example.js deleted file mode 100644 index b7e98218954d..000000000000 --- a/python/mozperftest/mozperftest/tests/example.js +++ /dev/null @@ -1,19 +0,0 @@ -async function setUp(context) { - context.log.info("setUp example!"); -} - -async function test(context, commands) { - context.log.info("Test with setUp/tearDown example!"); - await commands.measure.start("https://www.sitespeed.io/"); - await commands.measure.start("https://www.mozilla.org/en-US/"); -} - -async function tearDown(context) { - context.log.info("tearDown example!"); -} - -module.exports = { // eslint-disable-line - setUp, - tearDown, - test, -}; diff --git a/python/mozperftest/mozperftest/tests/samples/perftest_example.js b/python/mozperftest/mozperftest/tests/samples/perftest_example.js new file mode 100644 index 000000000000..34d876ac82fe --- /dev/null +++ b/python/mozperftest/mozperftest/tests/samples/perftest_example.js @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; + +var someVar; + +async function setUp(context) { + context.log.info("setUp example!"); +} + +async function test(context, commands) { + context.log.info("Test with setUp/tearDown example!"); + await commands.measure.start("https://www.sitespeed.io/"); + await commands.measure.start("https://www.mozilla.org/en-US/"); +} + +async function tearDown(context) { + context.log.info("tearDown example!"); +} + + +module.exports = { + setUp, + tearDown, + test, + owner: "Performance Team", + description: "Measures cold process applink time", + long_description: ` + This test launches the appropriate android app, simulating an app link + workflow. The application is launched with the intent action + android.intent.action.VIEW loading a trivially simple website. The reported + metric is the time from process start to navigationStart, reported as processLaunchToNavStart + `, + usage: ` + ./mach perftest testing/performance/perftest_applink.js \ + --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk \ + --hooks testing/performance/hooks_applink.py \ + --android-app-name org.mozilla.fennec_aurora \ + --perfherder-metrics processLaunchToNavStart + `, + supported_browser: ["Fenix nightly", "Geckoview_example", "Fennec"], + platform: ["Android"], +}; diff --git a/python/mozperftest/mozperftest/tests/support.py b/python/mozperftest/mozperftest/tests/support.py index 326de703ef6a..01b2f7ae1779 100644 --- a/python/mozperftest/mozperftest/tests/support.py +++ b/python/mozperftest/mozperftest/tests/support.py @@ -8,6 +8,11 @@ from mozperftest.metadata import Metadata from mozperftest.environment import MachEnvironment +HERE = os.path.dirname(__file__) +EXAMPLE_TESTS_DIR = os.path.join(HERE, "samples") +EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js") + + @contextlib.contextmanager def temp_file(name="temp", content=None): tempdir = tempfile.mkdtemp() diff --git a/python/mozperftest/mozperftest/tests/test_browsertime.py b/python/mozperftest/mozperftest/tests/test_browsertime.py index 282cb6889c60..7d0752209cc9 100644 --- a/python/mozperftest/mozperftest/tests/test_browsertime.py +++ b/python/mozperftest/mozperftest/tests/test_browsertime.py @@ -4,7 +4,7 @@ import mozunit import mock import shutil -from mozperftest.tests.support import get_running_env +from mozperftest.tests.support import get_running_env, EXAMPLE_TEST from mozperftest.environment import BROWSER from mozperftest.browser.browsertime import add_options from mozperftest.utils import silence @@ -28,7 +28,7 @@ def fetch(self, url): def test_browser(): mach_cmd, metadata, env = get_running_env() browser = env.layers[BROWSER] - env.set_arg("tests", [os.path.join(HERE, "example.js")]) + env.set_arg("tests", [EXAMPLE_TEST]) try: with browser as b, silence(): @@ -38,7 +38,7 @@ def test_browser(): assert mach_cmd.run_process.call_count == 1 # XXX more checks - assert mach_cmd.run_process.call_args[0][-1][-1] == os.path.join(HERE, "example.js") + assert mach_cmd.run_process.call_args[0][-1][-1] == EXAMPLE_TEST def test_add_options(): diff --git a/python/mozperftest/mozperftest/tests/test_consoleoutput.py b/python/mozperftest/mozperftest/tests/test_consoleoutput.py index 4b255383d62d..195c959d19c6 100644 --- a/python/mozperftest/mozperftest/tests/test_consoleoutput.py +++ b/python/mozperftest/mozperftest/tests/test_consoleoutput.py @@ -2,7 +2,7 @@ import os import mozunit -from mozperftest.tests.support import get_running_env, temp_dir +from mozperftest.tests.support import EXAMPLE_TEST, get_running_env, temp_dir from mozperftest.environment import METRICS from mozperftest.utils import silence @@ -26,7 +26,7 @@ def test_console_output(): mach_cmd.run_process = _run_process metrics = env.layers[METRICS] - env.set_arg("tests", [os.path.join(HERE, "example.js")]) + env.set_arg("tests", [EXAMPLE_TEST]) metadata.set_result(os.path.join(HERE, "browsertime-results")) with metrics as console, silence(): diff --git a/python/mozperftest/mozperftest/tests/test_mach_commands.py b/python/mozperftest/mozperftest/tests/test_mach_commands.py index a2d96f513b09..05e68fc1d91b 100644 --- a/python/mozperftest/mozperftest/tests/test_mach_commands.py +++ b/python/mozperftest/mozperftest/tests/test_mach_commands.py @@ -7,6 +7,7 @@ import os import mock import tempfile import shutil +from contextlib import contextmanager from mach.registrar import Registrar @@ -16,6 +17,7 @@ Registrar.commands_by_category = {"testing": set()} from mozperftest.environment import MachEnvironment from mozperftest.mach_commands import Perftest +from mozperftest.tests.support import EXAMPLE_TESTS_DIR class _TestMachEnvironment(MachEnvironment): @@ -29,8 +31,8 @@ class _TestMachEnvironment(MachEnvironment): pass -@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment) -def test_command(): +@contextmanager +def _get_perftest(): from mozbuild.base import MozbuildObject config = MozbuildObject.from_environment() @@ -43,11 +45,25 @@ def test_command(): state_dir = tempfile.mkdtemp() try: - test = Perftest(context()) - test.run_perftest() + yield Perftest(context()) finally: shutil.rmtree(context.state_dir) +@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment) +@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv") +def test_command(mocked_func): + with _get_perftest() as test: + test.run_perftest(tests=[EXAMPLE_TESTS_DIR]) + # XXX add assertions + + +@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment) +@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv") +def test_doc_flavor(mocked_func): + with _get_perftest() as test: + test.run_perftest(tests=[EXAMPLE_TESTS_DIR], flavor="doc") + + if __name__ == "__main__": mozunit.main() diff --git a/python/mozperftest/mozperftest/tests/test_perfherder.py b/python/mozperftest/mozperftest/tests/test_perfherder.py index f1cf86b7445b..6de6355c3e27 100644 --- a/python/mozperftest/mozperftest/tests/test_perfherder.py +++ b/python/mozperftest/mozperftest/tests/test_perfherder.py @@ -3,7 +3,7 @@ import os import mozunit import json -from mozperftest.tests.support import get_running_env, temp_file +from mozperftest.tests.support import get_running_env, temp_file, EXAMPLE_TEST from mozperftest.environment import METRICS from mozperftest.utils import silence @@ -22,7 +22,7 @@ def test_metrics(): mach_cmd.run_process = _run_process metrics = env.layers[METRICS] - env.set_arg("tests", [os.path.join(HERE, "example.js")]) + env.set_arg("tests", [EXAMPLE_TEST]) metadata.set_result(os.path.join(HERE, "browsertime-results")) with temp_file() as output: diff --git a/python/mozperftest/mozperftest/tests/test_scriptinfo.py b/python/mozperftest/mozperftest/tests/test_scriptinfo.py new file mode 100644 index 000000000000..65b22f6562b2 --- /dev/null +++ b/python/mozperftest/mozperftest/tests/test_scriptinfo.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import mozunit + +from mozperftest.scriptinfo import ScriptInfo +from mozperftest.tests.support import EXAMPLE_TEST + + +def test_scriptinfo(): + info = ScriptInfo(EXAMPLE_TEST) + assert info["author"] == "N/A" + + display = str(info) + assert "appropriate android app" in display + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mozperftest/mozperftest/utils.py b/python/mozperftest/mozperftest/utils.py index 9c45a37645ce..8991803a2ac9 100644 --- a/python/mozperftest/mozperftest/utils.py +++ b/python/mozperftest/mozperftest/utils.py @@ -4,6 +4,8 @@ import logging import contextlib import sys +import os + from six import StringIO @@ -54,3 +56,20 @@ class MachLogger: def error(self, msg, name="mozperftest", **kwargs): self._logger(logging.ERROR, name, kwargs, msg) + + +def install_package(virtualenv_manager, package): + from pip._internal.req.constructors import install_req_from_line + + req = install_req_from_line(package) + req.check_if_exists(use_user_site=False) + # already installed, check if it's in our venv + if req.satisfied_by is not None: + venv_site_lib = os.path.abspath( + os.path.join(virtualenv_manager.bin_path, "..", "lib") + ) + site_packages = os.path.abspath(req.satisfied_by.location) + if site_packages.startswith(venv_site_lib): + # already installed in this venv, we can skip + return + virtualenv_manager._run_pip(["install", package]) diff --git a/testing/performance/hooks_applink.py b/testing/performance/hooks_applink.py index 823ec73fd8d9..171c220a1e7c 100644 --- a/testing/performance/hooks_applink.py +++ b/testing/performance/hooks_applink.py @@ -5,23 +5,28 @@ from mozperftest.browser.browsertime import add_options url = "'https://www.example.com'" -common_options = [("processStartTime", "true"), - ("firefox.disableBrowsertimeExtension", "true"), - ("firefox.android.intentArgument", "'-a'"), - ("firefox.android.intentArgument", "'android.intent.action.VIEW'"), - ("firefox.android.intentArgument", "'-d'"), - ("firefox.android.intentArgument", url)] +common_options = [ + ("processStartTime", "true"), + ("firefox.disableBrowsertimeExtension", "true"), + ("firefox.android.intentArgument", "'-a'"), + ("firefox.android.intentArgument", "'android.intent.action.VIEW'"), + ("firefox.android.intentArgument", "'-d'"), + ("firefox.android.intentArgument", url), +] app_options = { "org.mozilla.geckoview_example": [ - ("firefox.android.activity", "'org.mozilla.geckoview_example.GeckoViewActivity'") + ( + "firefox.android.activity", + "'org.mozilla.geckoview_example.GeckoViewActivity'", + ) ], "org.mozilla.fennec_aurora": [ ("firefox.android.activity", "'org.mozilla.fenix.IntentReceiverActivity'") ], "org.mozilla.firefox": [ ("firefox.android.activity", "'org.mozilla.gecko.BrowserApp'") - ] + ], } diff --git a/testing/performance/perftest_applink.js b/testing/performance/perftest_applink.js index f372466c0065..d05f7b362f7a 100644 --- a/testing/performance/perftest_applink.js +++ b/testing/performance/perftest_applink.js @@ -1,8 +1,10 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. /* eslint-env node */ +"use strict"; -module.exports = async function(context, commands) { - "use strict"; - +async function test(context, commands) { // This violates all sorts of abstraction boundaries, but I don't see supported APIs for "just // waiting" nor for allowing navigation scripts to produce measurements. await commands.measure.start(); @@ -19,4 +21,25 @@ module.exports = async function(context, commands) { console.log("processLaunchToNavStart: " + processLaunchToNavStart); return true; +} + +module.exports = { + test, + owner: "Performance Team", + description: "Measures cold process applink time", + long_description: ` + This test launches the appropriate android app, simulating an app link + workflow. The application is launched with the intent action + android.intent.action.VIEW loading a trivially simple website. The reported + metric is the time from process start to navigationStart, reported as processLaunchToNavStart + `, + usage: ` + ./mach perftest testing/performance/perftest_applink.js \ + --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk \ + --hooks testing/performance/hooks_applink.py \ + --android-app-name org.mozilla.fennec_aurora \ + --perfherder-metrics processLaunchToNavStart + `, + supported_browser: ["Fenix nightly", "Geckoview_example", "Fennec"], + platform: ["Android"], }; diff --git a/third_party/python/esprima/PKG-INFO b/third_party/python/esprima/PKG-INFO new file mode 100644 index 000000000000..c2fee6ace6f6 --- /dev/null +++ b/third_party/python/esprima/PKG-INFO @@ -0,0 +1,143 @@ +Metadata-Version: 1.1 +Name: esprima +Version: 4.0.1 +Summary: ECMAScript parsing infrastructure for multipurpose analysis in Python +Home-page: https://github.com/Kronuz/esprima-python +Author: German M. Bravo (Kronuz) +Author-email: german.mb@gmail.com +License: BSD License +Description: |Donate| |PyPI Version| |PyPI License| |PyPI Format| |PyPI Status| + + **Esprima** (`esprima.org `__, BSD license) is a + high performance, standard-compliant + `ECMAScript `__ + parser officially written in ECMAScript (also popularly known as + `JavaScript `__) and ported to + Python. Esprima is created and maintained by `Ariya + Hidayat `__, with the help of `many + contributors `__. + + Python port is a line-by-line manual translation and was created and is + maintained by `German Mendez Bravo + (Kronuz) `__. + + Features + ~~~~~~~~ + + - Full support for ECMAScript 2017 (`ECMA-262 8th + Edition `__) + - Sensible `syntax tree + format `__ as + standardized by `ESTree project `__ + - Experimental support for `JSX `__, a + syntax extension for `React `__ + - Optional tracking of syntax node location (index-based and + line-column) + - `Heavily tested `__ (~1500 `unit + tests `__ + with `full code + coverage `__) + + Installation + ~~~~~~~~~~~~ + + .. code:: shell + + pip install esprima + + API + ~~~ + + Esprima can be used to perform `lexical + analysis `__ + (tokenization) or `syntactic + analysis `__ (parsing) of a + JavaScript program. + + A simple example: + + .. code:: javascript + + >>> import esprima + >>> program = 'const answer = 42' + + >>> esprima.tokenize(program) + [{ + type: "Keyword", + value: "const" + }, { + type: "Identifier", + value: "answer" + }, { + type: "Punctuator", + value: "=" + }, { + type: "Numeric", + value: "42" + }] + + >>> esprima.parseScript(program) + { + body: [ + { + kind: "const", + declarations: [ + { + init: { + raw: "42", + type: "Literal", + value: 42 + }, + type: "VariableDeclarator", + id: { + type: "Identifier", + name: "answer" + } + } + ], + type: "VariableDeclaration" + } + ], + type: "Program", + sourceType: "script" + } + + For more information, please read the `complete + documentation `__. + + .. |Donate| image:: https://img.shields.io/badge/Donate-PayPal-green.svg + :target: https://www.paypal.me/Kronuz/25 + .. |PyPI Version| image:: https://img.shields.io/pypi/v/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI License| image:: https://img.shields.io/pypi/l/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Wheel| image:: https://img.shields.io/pypi/wheel/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Format| image:: https://img.shields.io/pypi/format/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Implementation| image:: https://img.shields.io/pypi/implementation/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Status| image:: https://img.shields.io/pypi/status/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Downloads| image:: https://img.shields.io/pypi/dm/esprima.svg + :target: https://pypi.python.org/pypi/esprima +Keywords: esprima ecmascript javascript parser ast +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Topic :: Software Development :: Code Generators +Classifier: Topic :: Software Development :: Compilers +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: General +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 diff --git a/third_party/python/esprima/README b/third_party/python/esprima/README new file mode 100644 index 000000000000..442fbc7b11fa --- /dev/null +++ b/third_party/python/esprima/README @@ -0,0 +1,117 @@ +|Donate| |PyPI Version| |PyPI License| |PyPI Format| |PyPI Status| + +**Esprima** (`esprima.org `__, BSD license) is a +high performance, standard-compliant +`ECMAScript `__ +parser officially written in ECMAScript (also popularly known as +`JavaScript `__) and ported to +Python. Esprima is created and maintained by `Ariya +Hidayat `__, with the help of `many +contributors `__. + +Python port is a line-by-line manual translation and was created and is +maintained by `German Mendez Bravo +(Kronuz) `__. + +Features +~~~~~~~~ + +- Full support for ECMAScript 2017 (`ECMA-262 8th + Edition `__) +- Sensible `syntax tree + format `__ as + standardized by `ESTree project `__ +- Experimental support for `JSX `__, a + syntax extension for `React `__ +- Optional tracking of syntax node location (index-based and + line-column) +- `Heavily tested `__ (~1500 `unit + tests `__ + with `full code + coverage `__) + +Installation +~~~~~~~~~~~~ + +.. code:: shell + + pip install esprima + +API +~~~ + +Esprima can be used to perform `lexical +analysis `__ +(tokenization) or `syntactic +analysis `__ (parsing) of a +JavaScript program. + +A simple example: + +.. code:: javascript + + >>> import esprima + >>> program = 'const answer = 42' + + >>> esprima.tokenize(program) + [{ + type: "Keyword", + value: "const" + }, { + type: "Identifier", + value: "answer" + }, { + type: "Punctuator", + value: "=" + }, { + type: "Numeric", + value: "42" + }] + + >>> esprima.parseScript(program) + { + body: [ + { + kind: "const", + declarations: [ + { + init: { + raw: "42", + type: "Literal", + value: 42 + }, + type: "VariableDeclarator", + id: { + type: "Identifier", + name: "answer" + } + } + ], + type: "VariableDeclaration" + } + ], + type: "Program", + sourceType: "script" + } + +For more information, please read the `complete +documentation `__. + +.. |Donate| image:: https://img.shields.io/badge/Donate-PayPal-green.svg + :target: https://www.paypal.me/Kronuz/25 +.. |PyPI Version| image:: https://img.shields.io/pypi/v/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI License| image:: https://img.shields.io/pypi/l/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Wheel| image:: https://img.shields.io/pypi/wheel/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Format| image:: https://img.shields.io/pypi/format/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Implementation| image:: https://img.shields.io/pypi/implementation/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Status| image:: https://img.shields.io/pypi/status/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Downloads| image:: https://img.shields.io/pypi/dm/esprima.svg + :target: https://pypi.python.org/pypi/esprima diff --git a/third_party/python/esprima/esprima/__init__.py b/third_party/python/esprima/esprima/__init__.py new file mode 100644 index 000000000000..0dcdf99e5ec2 --- /dev/null +++ b/third_party/python/esprima/esprima/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import + +version = '4.0.1' +__version__ = (4, 0, 1) + +from .esprima import * # NOQA diff --git a/third_party/python/esprima/esprima/__main__.py b/third_party/python/esprima/esprima/__main__.py new file mode 100644 index 000000000000..92f2aa2ec596 --- /dev/null +++ b/third_party/python/esprima/esprima/__main__.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals, print_function, division + +import sys + +from .esprima import parse, tokenize, Error, toDict +from . import version + + +def main(): + import json + import time + import optparse + + usage = "usage: %prog [options] [file.js]" + parser = optparse.OptionParser(usage=usage, version=version) + parser.add_option("--comment", dest="comment", + action="store_true", default=False, + help="Gather all line and block comments in an array") + parser.add_option("--attachComment", dest="attachComment", + action="store_true", default=False, + help="Attach comments to nodes") + parser.add_option("--loc", dest="loc", default=False, + action="store_true", + help="Include line-column location info for each syntax node") + parser.add_option("--range", dest="range", default=False, + action="store_true", + help="Include index-based range for each syntax node") + parser.add_option("--raw", dest="raw", default=False, + action="store_true", + help="Display the raw value of literals") + parser.add_option("--tokens", dest="tokens", default=False, + action="store_true", + help="List all tokens in an array") + parser.add_option("--tolerant", dest="tolerant", default=False, + action="store_true", + help="Tolerate errors on a best-effort basis (experimental)") + parser.add_option("--tokenize", dest="tokenize", default=False, + action="store_true", + help="Only tokenize, do not parse.") + parser.add_option("--module", dest="sourceType", default='string', + action="store_const", const='module', + help="Tolerate errors on a best-effort basis (experimental)") + parser.set_defaults(jsx=True, classProperties=True) + opts, args = parser.parse_args() + + if len(args) == 1: + with open(args[0], 'rb') as f: + code = f.read().decode('utf-8') + elif sys.stdin.isatty(): + parser.print_help() + return 64 + else: + code = sys.stdin.read().decode('utf-8') + + options = opts.__dict__ + do_tokenize = options.pop('tokenize') + + t = time.time() + try: + if do_tokenize: + del options['sourceType'] + del options['tokens'] + del options['raw'] + del options['jsx'] + res = toDict(tokenize(code, options=options)) + else: + res = toDict(parse(code, options=options)) + except Error as e: + res = e.toDict() + dt = time.time() - t + 0.000000001 + + print(json.dumps(res, indent=4)) + print() + print('Parsed everyting in', round(dt, 5), 'seconds.') + print('Thats %d characters per second' % (len(code) // dt)) + + return 0 + + +if __name__ == '__main__': + retval = main() + sys.exit(retval) diff --git a/third_party/python/esprima/esprima/character.py b/third_party/python/esprima/esprima/character.py new file mode 100644 index 000000000000..a650a714a949 --- /dev/null +++ b/third_party/python/esprima/esprima/character.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +import sys + +import unicodedata +from collections import defaultdict + +from .compat import uchr, xrange + +# http://stackoverflow.com/questions/14245893/efficiently-list-all-characters-in-a-given-unicode-category +U_CATEGORIES = defaultdict(list) +for c in map(uchr, xrange(sys.maxunicode + 1)): + U_CATEGORIES[unicodedata.category(c)].append(c) +UNICODE_LETTER = set( + U_CATEGORIES['Lu'] + U_CATEGORIES['Ll'] + + U_CATEGORIES['Lt'] + U_CATEGORIES['Lm'] + + U_CATEGORIES['Lo'] + U_CATEGORIES['Nl'] +) +UNICODE_OTHER_ID_START = set(( + # Other_ID_Start + '\u1885', '\u1886', '\u2118', '\u212E', '\u309B', '\u309C', + # New in Unicode 8.0 + '\u08B3', '\u0AF9', '\u13F8', '\u9FCD', '\uAB60', '\U00010CC0', '\U000108E0', '\U0002B820', + # New in Unicode 9.0 + '\u1C80', '\U000104DB', '\U0001E922', + '\U0001EE00', '\U0001EE06', '\U0001EE0A', +)) +UNICODE_OTHER_ID_CONTINUE = set(( + # Other_ID_Continue + '\xB7', '\u0387', '\u1369', '\u136A', '\u136B', '\u136C', + '\u136D', '\u136E', '\u136F', '\u1370', '\u1371', '\u19DA', + # New in Unicode 8.0 + '\u08E3', '\uA69E', '\U00011730', + # New in Unicode 9.0 + '\u08D4', '\u1DFB', '\uA8C5', '\U00011450', + '\U0001EE03', '\U0001EE0B', +)) +UNICODE_COMBINING_MARK = set(U_CATEGORIES['Mn'] + U_CATEGORIES['Mc']) +UNICODE_DIGIT = set(U_CATEGORIES['Nd']) +UNICODE_CONNECTOR_PUNCTUATION = set(U_CATEGORIES['Pc']) +IDENTIFIER_START = UNICODE_LETTER.union(UNICODE_OTHER_ID_START).union(set(('$', '_', '\\'))) +IDENTIFIER_PART = IDENTIFIER_START.union(UNICODE_COMBINING_MARK).union(UNICODE_DIGIT).union(UNICODE_CONNECTOR_PUNCTUATION).union(set(('\u200D', '\u200C'))).union(UNICODE_OTHER_ID_CONTINUE) + +WHITE_SPACE = set(( + '\x09', '\x0B', '\x0C', '\x20', '\xA0', + '\u1680', '\u180E', '\u2000', '\u2001', '\u2002', + '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', + '\u2008', '\u2009', '\u200A', '\u202F', '\u205F', + '\u3000', '\uFEFF', +)) +LINE_TERMINATOR = set(('\x0A', '\x0D', '\u2028', '\u2029')) + +DECIMAL_CONV = dict((c, n) for n, c in enumerate('0123456789')) +OCTAL_CONV = dict((c, n) for n, c in enumerate('01234567')) +HEX_CONV = dict((c, n) for n, c in enumerate('0123456789abcdef')) +for n, c in enumerate('ABCDEF', 10): + HEX_CONV[c] = n +DECIMAL_DIGIT = set(DECIMAL_CONV.keys()) +OCTAL_DIGIT = set(OCTAL_CONV.keys()) +HEX_DIGIT = set(HEX_CONV.keys()) + + +class Character: + @staticmethod + def fromCodePoint(code): + return uchr(code) + + # https://tc39.github.io/ecma262/#sec-white-space + + @staticmethod + def isWhiteSpace(ch): + return ch in WHITE_SPACE + + # https://tc39.github.io/ecma262/#sec-line-terminators + + @staticmethod + def isLineTerminator(ch): + return ch in LINE_TERMINATOR + + # https://tc39.github.io/ecma262/#sec-names-and-keywords + + @staticmethod + def isIdentifierStart(ch): + return ch in IDENTIFIER_START + + @staticmethod + def isIdentifierPart(ch): + return ch in IDENTIFIER_PART + + # https://tc39.github.io/ecma262/#sec-literals-numeric-literals + + @staticmethod + def isDecimalDigit(ch): + return ch in DECIMAL_DIGIT + + @staticmethod + def isHexDigit(ch): + return ch in HEX_DIGIT + + @staticmethod + def isOctalDigit(ch): + return ch in OCTAL_DIGIT diff --git a/third_party/python/esprima/esprima/comment_handler.py b/third_party/python/esprima/esprima/comment_handler.py new file mode 100644 index 000000000000..09a37a5fd2f0 --- /dev/null +++ b/third_party/python/esprima/esprima/comment_handler.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, self.list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, self.list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# self.SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES +# LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# self.SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .objects import Object +from .nodes import Node +from .syntax import Syntax + + +class Comment(Node): + def __init__(self, type, value, range=None, loc=None): + self.type = type + self.value = value + self.range = range + self.loc = loc + + +class Entry(Object): + def __init__(self, comment, start): + self.comment = comment + self.start = start + + +class NodeInfo(Object): + def __init__(self, node, start): + self.node = node + self.start = start + + +class CommentHandler(object): + def __init__(self): + self.attach = False + self.comments = [] + self.stack = [] + self.leading = [] + self.trailing = [] + + def insertInnerComments(self, node, metadata): + # innnerComments for properties empty block + # `function a(:/** comments **\/}` + if node.type is Syntax.BlockStatement and not node.body: + innerComments = [] + for i, entry in enumerate(self.leading): + if metadata.end.offset >= entry.start: + innerComments.append(entry.comment) + self.leading[i] = None + self.trailing[i] = None + if innerComments: + node.innerComments = innerComments + self.leading = [v for v in self.leading if v is not None] + self.trailing = [v for v in self.trailing if v is not None] + + def findTrailingComments(self, metadata): + trailingComments = [] + + if self.trailing: + for i, entry in enumerate(self.trailing): + if entry.start >= metadata.end.offset: + trailingComments.append(entry.comment) + if trailingComments: + self.trailing = [] + return trailingComments + + last = self.stack and self.stack[-1] + if last and last.node.trailingComments: + firstComment = last.node.trailingComments[0] + if firstComment and firstComment.range[0] >= metadata.end.offset: + trailingComments = last.node.trailingComments + del last.node.trailingComments + return trailingComments + + def findLeadingComments(self, metadata): + leadingComments = [] + + target = None + while self.stack: + entry = self.stack and self.stack[-1] + if entry and entry.start >= metadata.start.offset: + target = entry.node + self.stack.pop() + else: + break + + if target: + if target.leadingComments: + for i, comment in enumerate(target.leadingComments): + if comment.range[1] <= metadata.start.offset: + leadingComments.append(comment) + target.leadingComments[i] = None + if leadingComments: + target.leadingComments = [v for v in target.leadingComments if v is not None] + if not target.leadingComments: + del target.leadingComments + return leadingComments + + for i, entry in enumerate(self.leading): + if entry.start <= metadata.start.offset: + leadingComments.append(entry.comment) + self.leading[i] = None + if leadingComments: + self.leading = [v for v in self.leading if v is not None] + + return leadingComments + + def visitNode(self, node, metadata): + if node.type is Syntax.Program and node.body: + return + + self.insertInnerComments(node, metadata) + trailingComments = self.findTrailingComments(metadata) + leadingComments = self.findLeadingComments(metadata) + if leadingComments: + node.leadingComments = leadingComments + if trailingComments: + node.trailingComments = trailingComments + + self.stack.append(NodeInfo( + node=node, + start=metadata.start.offset + )) + + def visitComment(self, node, metadata): + type = 'Line' if node.type[0] == 'L' else 'Block' + comment = Comment( + type=type, + value=node.value + ) + if node.range: + comment.range = node.range + if node.loc: + comment.loc = node.loc + self.comments.append(comment) + + if self.attach: + entry = Entry( + comment=Comment( + type=type, + value=node.value, + range=[metadata.start.offset, metadata.end.offset] + ), + start=metadata.start.offset + ) + if node.loc: + entry.comment.loc = node.loc + node.type = type + self.leading.append(entry) + self.trailing.append(entry) + + def visit(self, node, metadata): + if node.type == 'LineComment': + self.visitComment(node, metadata) + elif node.type == 'BlockComment': + self.visitComment(node, metadata) + elif self.attach: + self.visitNode(node, metadata) diff --git a/third_party/python/esprima/esprima/compat.py b/third_party/python/esprima/esprima/compat.py new file mode 100644 index 000000000000..79543255e377 --- /dev/null +++ b/third_party/python/esprima/esprima/compat.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +import sys + +PY3 = sys.version_info >= (3, 0) + +if PY3: + # Python 3: + basestring = str + long = int + xrange = range + unicode = str + uchr = chr + + def uord(ch): + return ord(ch[0]) + +else: + basestring = basestring + long = long + xrange = xrange + unicode = unicode + + try: + # Python 2 UCS4: + unichr(0x10000) + uchr = unichr + + def uord(ch): + return ord(ch[0]) + + except ValueError: + # Python 2 UCS2: + def uchr(code): + # UTF-16 Encoding + if code <= 0xFFFF: + return unichr(code) + cu1 = ((code - 0x10000) >> 10) + 0xD800 + cu2 = ((code - 0x10000) & 1023) + 0xDC00 + return unichr(cu1) + unichr(cu2) + + def uord(ch): + cp = ord(ch[0]) + if cp >= 0xD800 and cp <= 0xDBFF: + second = ord(ch[1]) + if second >= 0xDC00 and second <= 0xDFFF: + first = cp + cp = (first - 0xD800) * 0x400 + second - 0xDC00 + 0x10000 + return cp diff --git a/third_party/python/esprima/esprima/error_handler.py b/third_party/python/esprima/esprima/error_handler.py new file mode 100644 index 000000000000..9b0f5cb843f3 --- /dev/null +++ b/third_party/python/esprima/esprima/error_handler.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import unicode_literals + +from .compat import unicode + + +class Error(Exception): + def __init__(self, message, name=None, index=None, lineNumber=None, column=None, description=None): + super(Error, self).__init__(message) + self.message = message + self.name = name + self.index = index + self.lineNumber = lineNumber + self.column = column + # self.description = description + + def toString(self): + return '%s: %s' % (self.__class__.__name__, self) + + def toDict(self): + d = dict((unicode(k), v) for k, v in self.__dict__.items() if v is not None) + d['message'] = self.toString() + return d + + +class ErrorHandler: + def __init__(self): + self.errors = [] + self.tolerant = False + + def recordError(self, error): + self.errors.append(error.toDict()) + + def tolerate(self, error): + if self.tolerant: + self.recordError(error) + else: + raise error + + def createError(self, index, line, col, description): + msg = 'Line %s: %s' % (line, description) + return Error(msg, index=index, lineNumber=line, column=col, description=description) + + def throwError(self, index, line, col, description): + raise self.createError(index, line, col, description) + + def tolerateError(self, index, line, col, description): + error = self.createError(index, line, col, description) + if self.tolerant: + self.recordError(error) + else: + raise error diff --git a/third_party/python/esprima/esprima/esprima.py b/third_party/python/esprima/esprima/esprima.py new file mode 100644 index 000000000000..faea0c2dda06 --- /dev/null +++ b/third_party/python/esprima/esprima/esprima.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .comment_handler import CommentHandler +from .error_handler import Error +from .jsx_parser import JSXParser +from .jsx_syntax import JSXSyntax +from .objects import Array, toDict +from .parser import Parser +from .syntax import Syntax +from .tokenizer import Tokenizer +from .visitor import NodeVisitor +from . import nodes +from . import jsx_nodes + + +__all__ = ['Syntax', 'JSXSyntax', 'Error', 'NodeVisitor', 'nodes', 'jsx_nodes', + 'parse', 'parseModule', 'parseScript', 'tokenize', 'toDict'] + + +def parse(code, options=None, delegate=None, **kwargs): + options = {} if options is None else options.copy() + options.update(kwargs) + + # ESNext presset: + if options.get('esnext', False): + options['jsx'] = True + options['classProperties'] = True + + commentHandler = None + + def proxyDelegate(node, metadata): + if delegate: + new_node = delegate(node, metadata) + if new_node is not None: + node = new_node + if commentHandler: + commentHandler.visit(node, metadata) + return node + + parserDelegate = None if delegate is None else proxyDelegate + collectComment = options.get('comment', False) + attachComment = options.get('attachComment', False) + if collectComment or attachComment: + commentHandler = CommentHandler() + commentHandler.attach = attachComment + options['comment'] = True + parserDelegate = proxyDelegate + + isModule = options.get('sourceType', 'script') == 'module' + + if options.get('jsx', False): + parser = JSXParser(code, options=options, delegate=parserDelegate) + else: + parser = Parser(code, options=options, delegate=parserDelegate) + + ast = parser.parseModule() if isModule else parser.parseScript() + + if collectComment and commentHandler: + ast.comments = commentHandler.comments + + if parser.config.tokens: + ast.tokens = parser.tokens + + if parser.config.tolerant: + ast.errors = parser.errorHandler.errors + + return ast + + +def parseModule(code, options=None, delegate=None, **kwargs): + kwargs['sourceType'] = 'module' + return parse(code, options, delegate, **kwargs) + + +def parseScript(code, options=None, delegate=None, **kwargs): + kwargs['sourceType'] = 'script' + return parse(code, options, delegate, **kwargs) + + +def tokenize(code, options=None, delegate=None, **kwargs): + options = {} if options is None else options.copy() + options.update(kwargs) + + tokenizer = Tokenizer(code, options) + + tokens = Array() + + try: + while True: + token = tokenizer.getNextToken() + if not token: + break + if delegate: + token = delegate(token) + tokens.append(token) + except Error as e: + tokenizer.errorHandler.tolerate(e) + + if tokenizer.errorHandler.tolerant: + tokens.errors = tokenizer.errors() + + return tokens diff --git a/third_party/python/esprima/esprima/jsx_nodes.py b/third_party/python/esprima/esprima/jsx_nodes.py new file mode 100644 index 000000000000..f195653ab581 --- /dev/null +++ b/third_party/python/esprima/esprima/jsx_nodes.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .nodes import Node +from .jsx_syntax import JSXSyntax + + +class JSXClosingElement(Node): + def __init__(self, name): + self.type = JSXSyntax.JSXClosingElement + self.name = name + + +class JSXElement(Node): + def __init__(self, openingElement, children, closingElement): + self.type = JSXSyntax.JSXElement + self.openingElement = openingElement + self.children = children + self.closingElement = closingElement + + +class JSXEmptyExpression(Node): + def __init__(self): + self.type = JSXSyntax.JSXEmptyExpression + + +class JSXExpressionContainer(Node): + def __init__(self, expression): + self.type = JSXSyntax.JSXExpressionContainer + self.expression = expression + + +class JSXIdentifier(Node): + def __init__(self, name): + self.type = JSXSyntax.JSXIdentifier + self.name = name + + +class JSXMemberExpression(Node): + def __init__(self, object, property): + self.type = JSXSyntax.JSXMemberExpression + self.object = object + self.property = property + + +class JSXAttribute(Node): + def __init__(self, name, value): + self.type = JSXSyntax.JSXAttribute + self.name = name + self.value = value + + +class JSXNamespacedName(Node): + def __init__(self, namespace, name): + self.type = JSXSyntax.JSXNamespacedName + self.namespace = namespace + self.name = name + + +class JSXOpeningElement(Node): + def __init__(self, name, selfClosing, attributes): + self.type = JSXSyntax.JSXOpeningElement + self.name = name + self.selfClosing = selfClosing + self.attributes = attributes + + +class JSXSpreadAttribute(Node): + def __init__(self, argument): + self.type = JSXSyntax.JSXSpreadAttribute + self.argument = argument + + +class JSXText(Node): + def __init__(self, value, raw): + self.type = JSXSyntax.JSXText + self.value = value + self.raw = raw diff --git a/third_party/python/esprima/esprima/jsx_parser.py b/third_party/python/esprima/esprima/jsx_parser.py new file mode 100644 index 000000000000..ec71b9251df1 --- /dev/null +++ b/third_party/python/esprima/esprima/jsx_parser.py @@ -0,0 +1,584 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .compat import uchr +from .character import Character +from . import jsx_nodes as JSXNode +from .jsx_syntax import JSXSyntax +from . import nodes as Node +from .parser import Marker, Parser +from .token import Token, TokenName +from .xhtml_entities import XHTMLEntities + + +class MetaJSXElement(object): + def __init__(self, node=None, opening=None, closing=None, children=None): + self.node = node + self.opening = opening + self.closing = closing + self.children = children + + +class JSXToken(object): + Identifier = 100 + Text = 101 + + +class RawJSXToken(object): + def __init__(self, type=None, value=None, lineNumber=None, lineStart=None, start=None, end=None): + self.type = type + self.value = value + self.lineNumber = lineNumber + self.lineStart = lineStart + self.start = start + self.end = end + + +TokenName[JSXToken.Identifier] = "JSXIdentifier" +TokenName[JSXToken.Text] = "JSXText" + + +# Fully qualified element name, e.g. returns "svg:path" +def getQualifiedElementName(elementName): + typ = elementName.type + if typ is JSXSyntax.JSXIdentifier: + id = elementName + qualifiedName = id.name + elif typ is JSXSyntax.JSXNamespacedName: + ns = elementName + qualifiedName = getQualifiedElementName(ns.namespace) + ':' + getQualifiedElementName(ns.name) + elif typ is JSXSyntax.JSXMemberExpression: + expr = elementName + qualifiedName = getQualifiedElementName(expr.object) + '.' + getQualifiedElementName(expr.property) + + return qualifiedName + + +class JSXParser(Parser): + def __init__(self, code, options, delegate): + super(JSXParser, self).__init__(code, options, delegate) + + def parsePrimaryExpression(self): + return self.parseJSXRoot() if self.match('<') else super(JSXParser, self).parsePrimaryExpression() + + def startJSX(self): + # Unwind the scanner before the lookahead token. + self.scanner.index = self.startMarker.index + self.scanner.lineNumber = self.startMarker.line + self.scanner.lineStart = self.startMarker.index - self.startMarker.column + + def finishJSX(self): + # Prime the next lookahead. + self.nextToken() + + def reenterJSX(self): + self.startJSX() + self.expectJSX('}') + + # Pop the closing '}' added from the lookahead. + if self.config.tokens: + self.tokens.pop() + + def createJSXNode(self): + self.collectComments() + return Marker( + index=self.scanner.index, + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart + ) + + def createJSXChildNode(self): + return Marker( + index=self.scanner.index, + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart + ) + + def scanXHTMLEntity(self, quote): + result = '&' + + valid = True + terminated = False + numeric = False + hex = False + + while not self.scanner.eof() and valid and not terminated: + ch = self.scanner.source[self.scanner.index] + if ch == quote: + break + + terminated = (ch == ';') + result += ch + self.scanner.index += 1 + if not terminated: + length = len(result) + if length == 2: + # e.g. '{' + numeric = (ch == '#') + elif length == 3: + if numeric: + # e.g. 'A' + hex = ch == 'x' + valid = hex or Character.isDecimalDigit(ch) + numeric = numeric and not hex + else: + valid = valid and not (numeric and not Character.isDecimalDigit(ch)) + valid = valid and not (hex and not Character.isHexDigit(ch)) + + if valid and terminated and len(result) > 2: + # e.g. 'A' becomes just '#x41' + st = result[1:-1] + if numeric and len(st) > 1: + result = uchr(int(st[1:], 10)) + elif hex and len(st) > 2: + result = uchr(int(st[2:], 16)) + elif not numeric and not hex and st in XHTMLEntities: + result = XHTMLEntities[st] + + return result + + # Scan the next JSX token. This replaces Scanner#lex when in JSX mode. + + def lexJSX(self): + ch = self.scanner.source[self.scanner.index] + + # < > / : = { } + if ch in ('<', '>', '/', ':', '=', '{', '}'): + value = self.scanner.source[self.scanner.index] + self.scanner.index += 1 + return RawJSXToken( + type=Token.Punctuator, + value=value, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=self.scanner.index - 1, + end=self.scanner.index + ) + + # " ' + if ch in ('\'', '"'): + start = self.scanner.index + quote = self.scanner.source[self.scanner.index] + self.scanner.index += 1 + str = '' + while not self.scanner.eof(): + ch = self.scanner.source[self.scanner.index] + self.scanner.index += 1 + if ch == quote: + break + elif ch == '&': + str += self.scanXHTMLEntity(quote) + else: + str += ch + + return RawJSXToken( + type=Token.StringLiteral, + value=str, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + # ... or . + if ch == '.': + start = self.scanner.index + if self.scanner.source[start + 1:start + 3] == '..': + value = '...' + self.scanner.index += 3 + else: + value = '.' + self.scanner.index += 1 + return RawJSXToken( + type=Token.Punctuator, + value=value, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + # ` + if ch == '`': + # Only placeholder, since it will be rescanned as a real assignment expression. + return RawJSXToken( + type=Token.Template, + value='', + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=self.scanner.index, + end=self.scanner.index + ) + + # Identifer can not contain backslash (char code 92). + if Character.isIdentifierStart(ch) and ch != '\\': + start = self.scanner.index + self.scanner.index += 1 + while not self.scanner.eof(): + ch = self.scanner.source[self.scanner.index] + if Character.isIdentifierPart(ch) and ch != '\\': + self.scanner.index += 1 + elif ch == '-': + # Hyphen (char code 45) can be part of an identifier. + self.scanner.index += 1 + else: + break + + id = self.scanner.source[start:self.scanner.index] + return RawJSXToken( + type=JSXToken.Identifier, + value=id, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + return self.scanner.lex() + + def nextJSXToken(self): + self.collectComments() + + self.startMarker.index = self.scanner.index + self.startMarker.line = self.scanner.lineNumber + self.startMarker.column = self.scanner.index - self.scanner.lineStart + token = self.lexJSX() + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + + if self.config.tokens: + self.tokens.append(self.convertToken(token)) + + return token + + def nextJSXText(self): + self.startMarker.index = self.scanner.index + self.startMarker.line = self.scanner.lineNumber + self.startMarker.column = self.scanner.index - self.scanner.lineStart + + start = self.scanner.index + + text = '' + while not self.scanner.eof(): + ch = self.scanner.source[self.scanner.index] + if ch in ('{', '<'): + break + + self.scanner.index += 1 + text += ch + if Character.isLineTerminator(ch): + self.scanner.lineNumber += 1 + if ch == '\r' and self.scanner.source[self.scanner.index] == '\n': + self.scanner.index += 1 + + self.scanner.lineStart = self.scanner.index + + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + + token = RawJSXToken( + type=JSXToken.Text, + value=text, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + if text and self.config.tokens: + self.tokens.append(self.convertToken(token)) + + return token + + def peekJSXToken(self): + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.lexJSX() + self.scanner.restoreState(state) + + return next + + # Expect the next JSX token to match the specified punctuator. + # If not, an exception will be thrown. + + def expectJSX(self, value): + token = self.nextJSXToken() + if token.type is not Token.Punctuator or token.value != value: + self.throwUnexpectedToken(token) + + # Return True if the next JSX token matches the specified punctuator. + + def matchJSX(self, *value): + next = self.peekJSXToken() + return next.type is Token.Punctuator and next.value in value + + def parseJSXIdentifier(self): + node = self.createJSXNode() + token = self.nextJSXToken() + if token.type is not JSXToken.Identifier: + self.throwUnexpectedToken(token) + + return self.finalize(node, JSXNode.JSXIdentifier(token.value)) + + def parseJSXElementName(self): + node = self.createJSXNode() + elementName = self.parseJSXIdentifier() + + if self.matchJSX(':'): + namespace = elementName + self.expectJSX(':') + name = self.parseJSXIdentifier() + elementName = self.finalize(node, JSXNode.JSXNamespacedName(namespace, name)) + elif self.matchJSX('.'): + while self.matchJSX('.'): + object = elementName + self.expectJSX('.') + property = self.parseJSXIdentifier() + elementName = self.finalize(node, JSXNode.JSXMemberExpression(object, property)) + + return elementName + + def parseJSXAttributeName(self): + node = self.createJSXNode() + + identifier = self.parseJSXIdentifier() + if self.matchJSX(':'): + namespace = identifier + self.expectJSX(':') + name = self.parseJSXIdentifier() + attributeName = self.finalize(node, JSXNode.JSXNamespacedName(namespace, name)) + else: + attributeName = identifier + + return attributeName + + def parseJSXStringLiteralAttribute(self): + node = self.createJSXNode() + token = self.nextJSXToken() + if token.type is not Token.StringLiteral: + self.throwUnexpectedToken(token) + + raw = self.getTokenRaw(token) + return self.finalize(node, Node.Literal(token.value, raw)) + + def parseJSXExpressionAttribute(self): + node = self.createJSXNode() + + self.expectJSX('{') + self.finishJSX() + + if self.match('}'): + self.tolerateError('JSX attributes must only be assigned a non-empty expression') + + expression = self.parseAssignmentExpression() + self.reenterJSX() + + return self.finalize(node, JSXNode.JSXExpressionContainer(expression)) + + def parseJSXAttributeValue(self): + if self.matchJSX('{'): + return self.parseJSXExpressionAttribute() + if self.matchJSX('<'): + return self.parseJSXElement() + + return self.parseJSXStringLiteralAttribute() + + def parseJSXNameValueAttribute(self): + node = self.createJSXNode() + name = self.parseJSXAttributeName() + value = None + if self.matchJSX('='): + self.expectJSX('=') + value = self.parseJSXAttributeValue() + + return self.finalize(node, JSXNode.JSXAttribute(name, value)) + + def parseJSXSpreadAttribute(self): + node = self.createJSXNode() + self.expectJSX('{') + self.expectJSX('...') + + self.finishJSX() + argument = self.parseAssignmentExpression() + self.reenterJSX() + + return self.finalize(node, JSXNode.JSXSpreadAttribute(argument)) + + def parseJSXAttributes(self): + attributes = [] + + while not self.matchJSX('/', '>'): + attribute = self.parseJSXSpreadAttribute() if self.matchJSX('{') else self.parseJSXNameValueAttribute() + attributes.append(attribute) + + return attributes + + def parseJSXOpeningElement(self): + node = self.createJSXNode() + + self.expectJSX('<') + name = self.parseJSXElementName() + attributes = self.parseJSXAttributes() + selfClosing = self.matchJSX('/') + if selfClosing: + self.expectJSX('/') + + self.expectJSX('>') + + return self.finalize(node, JSXNode.JSXOpeningElement(name, selfClosing, attributes)) + + def parseJSXBoundaryElement(self): + node = self.createJSXNode() + + self.expectJSX('<') + if self.matchJSX('/'): + self.expectJSX('/') + elementName = self.parseJSXElementName() + self.expectJSX('>') + return self.finalize(node, JSXNode.JSXClosingElement(elementName)) + + name = self.parseJSXElementName() + attributes = self.parseJSXAttributes() + selfClosing = self.matchJSX('/') + if selfClosing: + self.expectJSX('/') + + self.expectJSX('>') + + return self.finalize(node, JSXNode.JSXOpeningElement(name, selfClosing, attributes)) + + def parseJSXEmptyExpression(self): + node = self.createJSXChildNode() + self.collectComments() + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + return self.finalize(node, JSXNode.JSXEmptyExpression()) + + def parseJSXExpressionContainer(self): + node = self.createJSXNode() + self.expectJSX('{') + + if self.matchJSX('}'): + expression = self.parseJSXEmptyExpression() + self.expectJSX('}') + else: + self.finishJSX() + expression = self.parseAssignmentExpression() + self.reenterJSX() + + return self.finalize(node, JSXNode.JSXExpressionContainer(expression)) + + def parseJSXChildren(self): + children = [] + + while not self.scanner.eof(): + node = self.createJSXChildNode() + token = self.nextJSXText() + if token.start < token.end: + raw = self.getTokenRaw(token) + child = self.finalize(node, JSXNode.JSXText(token.value, raw)) + children.append(child) + + if self.scanner.source[self.scanner.index] == '{': + container = self.parseJSXExpressionContainer() + children.append(container) + else: + break + + return children + + def parseComplexJSXElement(self, el): + stack = [] + + while not self.scanner.eof(): + el.children.extend(self.parseJSXChildren()) + node = self.createJSXChildNode() + element = self.parseJSXBoundaryElement() + if element.type is JSXSyntax.JSXOpeningElement: + opening = element + if opening.selfClosing: + child = self.finalize(node, JSXNode.JSXElement(opening, [], None)) + el.children.append(child) + else: + stack.append(el) + el = MetaJSXElement( + node=node, + opening=opening, + closing=None, + children=[], + ) + + if element.type is JSXSyntax.JSXClosingElement: + el.closing = element + open = getQualifiedElementName(el.opening.name) + close = getQualifiedElementName(el.closing.name) + if open != close: + self.tolerateError('Expected corresponding JSX closing tag for %0', open) + + if stack: + child = self.finalize(el.node, JSXNode.JSXElement(el.opening, el.children, el.closing)) + el = stack[-1] + el.children.append(child) + stack.pop() + else: + break + + return el + + def parseJSXElement(self): + node = self.createJSXNode() + + opening = self.parseJSXOpeningElement() + children = [] + closing = None + + if not opening.selfClosing: + el = self.parseComplexJSXElement(MetaJSXElement( + node=node, + opening=opening, + closing=closing, + children=children + )) + children = el.children + closing = el.closing + + return self.finalize(node, JSXNode.JSXElement(opening, children, closing)) + + def parseJSXRoot(self): + # Pop the opening '<' added from the lookahead. + if self.config.tokens: + self.tokens.pop() + + self.startJSX() + element = self.parseJSXElement() + self.finishJSX() + + return element + + def isStartOfExpression(self): + return super(JSXParser, self).isStartOfExpression() or self.match('<') diff --git a/third_party/python/esprima/esprima/jsx_syntax.py b/third_party/python/esprima/esprima/jsx_syntax.py new file mode 100644 index 000000000000..808cc8b02724 --- /dev/null +++ b/third_party/python/esprima/esprima/jsx_syntax.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import unicode_literals + + +class JSXSyntax: + JSXAttribute = "JSXAttribute" + JSXClosingElement = "JSXClosingElement" + JSXElement = "JSXElement" + JSXEmptyExpression = "JSXEmptyExpression" + JSXExpressionContainer = "JSXExpressionContainer" + JSXIdentifier = "JSXIdentifier" + JSXMemberExpression = "JSXMemberExpression" + JSXNamespacedName = "JSXNamespacedName" + JSXOpeningElement = "JSXOpeningElement" + JSXSpreadAttribute = "JSXSpreadAttribute" + JSXText = "JSXText" diff --git a/third_party/python/esprima/esprima/messages.py b/third_party/python/esprima/esprima/messages.py new file mode 100644 index 000000000000..bb6314e1ea6a --- /dev/null +++ b/third_party/python/esprima/esprima/messages.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import unicode_literals + + +# Error messages should be identical to V8. +class Messages: + ObjectPatternAsRestParameter = "Unexpected token {" + BadImportCallArity = "Unexpected token" + BadGetterArity = "Getter must not have any formal parameters" + BadSetterArity = "Setter must have exactly one formal parameter" + BadSetterRestParameter = "Setter function argument must not be a rest parameter" + ConstructorIsAsync = "Class constructor may not be an async method" + ConstructorSpecialMethod = "Class constructor may not be an accessor" + DeclarationMissingInitializer = "Missing initializer in %0 declaration" + DefaultRestParameter = "Unexpected token =" + DefaultRestProperty = "Unexpected token =" + DuplicateBinding = "Duplicate binding %0" + DuplicateConstructor = "A class may only have one constructor" + DuplicateProtoProperty = "Duplicate __proto__ fields are not allowed in object literals" + ForInOfLoopInitializer = "%0 loop variable declaration may not have an initializer" + GeneratorInLegacyContext = "Generator declarations are not allowed in legacy contexts" + IllegalBreak = "Illegal break statement" + IllegalContinue = "Illegal continue statement" + IllegalExportDeclaration = "Unexpected token" + IllegalImportDeclaration = "Unexpected token" + IllegalLanguageModeDirective = "Illegal 'use strict' directive in function with non-simple parameter list" + IllegalReturn = "Illegal return statement" + InvalidEscapedReservedWord = "Keyword must not contain escaped characters" + InvalidHexEscapeSequence = "Invalid hexadecimal escape sequence" + InvalidLHSInAssignment = "Invalid left-hand side in assignment" + InvalidLHSInForIn = "Invalid left-hand side in for-in" + InvalidLHSInForLoop = "Invalid left-hand side in for-loop" + InvalidModuleSpecifier = "Unexpected token" + InvalidRegExp = "Invalid regular expression" + LetInLexicalBinding = "let is disallowed as a lexically bound name" + MissingFromClause = "Unexpected token" + MultipleDefaultsInSwitch = "More than one default clause in switch statement" + NewlineAfterThrow = "Illegal newline after throw" + NoAsAfterImportNamespace = "Unexpected token" + NoCatchOrFinally = "Missing catch or finally after try" + ParameterAfterRestParameter = "Rest parameter must be last formal parameter" + PropertyAfterRestProperty = "Unexpected token" + Redeclaration = "%0 '%1' has already been declared" + StaticPrototype = "Classes may not have static property named prototype" + StrictCatchVariable = "Catch variable may not be eval or arguments in strict mode" + StrictDelete = "Delete of an unqualified identifier in strict mode." + StrictFunction = "In strict mode code, functions can only be declared at top level or inside a block" + StrictFunctionName = "Function name may not be eval or arguments in strict mode" + StrictLHSAssignment = "Assignment to eval or arguments is not allowed in strict mode" + StrictLHSPostfix = "Postfix increment/decrement may not have eval or arguments operand in strict mode" + StrictLHSPrefix = "Prefix increment/decrement may not have eval or arguments operand in strict mode" + StrictModeWith = "Strict mode code may not include a with statement" + StrictOctalLiteral = "Octal literals are not allowed in strict mode." + StrictParamDupe = "Strict mode function may not have duplicate parameter names" + StrictParamName = "Parameter name eval or arguments is not allowed in strict mode" + StrictReservedWord = "Use of future reserved word in strict mode" + StrictVarName = "Variable name may not be eval or arguments in strict mode" + TemplateOctalLiteral = "Octal literals are not allowed in template strings." + UnexpectedEOS = "Unexpected end of input" + UnexpectedIdentifier = "Unexpected identifier" + UnexpectedNumber = "Unexpected number" + UnexpectedReserved = "Unexpected reserved word" + UnexpectedString = "Unexpected string" + UnexpectedTemplate = "Unexpected quasi %0" + UnexpectedToken = "Unexpected token %0" + UnexpectedTokenIllegal = "Unexpected token ILLEGAL" + UnknownLabel = "Undefined label '%0'" + UnterminatedRegExp = "Invalid regular expression: missing /" diff --git a/third_party/python/esprima/esprima/nodes.py b/third_party/python/esprima/esprima/nodes.py new file mode 100644 index 000000000000..bbbbdb893b44 --- /dev/null +++ b/third_party/python/esprima/esprima/nodes.py @@ -0,0 +1,620 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .objects import Object +from .syntax import Syntax +from .scanner import RegExp + + +class Node(Object): + def __dir__(self): + return list(self.__dict__.keys()) + + def __iter__(self): + return self.__iter__ + + def keys(self): + return self.__dict__.keys() + + def items(self): + return self.__dict__.items() + + +class ArrayExpression(Node): + def __init__(self, elements): + self.type = Syntax.ArrayExpression + self.elements = elements + + +class ArrayPattern(Node): + def __init__(self, elements): + self.type = Syntax.ArrayPattern + self.elements = elements + + +class ArrowFunctionExpression(Node): + def __init__(self, params, body, expression): + self.type = Syntax.ArrowFunctionExpression + self.generator = False + self.isAsync = False + self.params = params + self.body = body + self.expression = expression + + +class AssignmentExpression(Node): + def __init__(self, operator, left, right): + self.type = Syntax.AssignmentExpression + self.operator = operator + self.left = left + self.right = right + + +class AssignmentPattern(Node): + def __init__(self, left, right): + self.type = Syntax.AssignmentPattern + self.left = left + self.right = right + + +class AsyncArrowFunctionExpression(Node): + def __init__(self, params, body, expression): + self.type = Syntax.ArrowFunctionExpression + self.generator = False + self.isAsync = True + self.params = params + self.body = body + self.expression = expression + + +class AsyncFunctionDeclaration(Node): + def __init__(self, id, params, body): + self.type = Syntax.FunctionDeclaration + self.generator = False + self.expression = False + self.isAsync = True + self.id = id + self.params = params + self.body = body + + +class AsyncFunctionExpression(Node): + def __init__(self, id, params, body): + self.type = Syntax.FunctionExpression + self.generator = False + self.expression = False + self.isAsync = True + self.id = id + self.params = params + self.body = body + + +class AwaitExpression(Node): + def __init__(self, argument): + self.type = Syntax.AwaitExpression + self.argument = argument + + +class BinaryExpression(Node): + def __init__(self, operator, left, right): + self.type = Syntax.LogicalExpression if operator in ('||', '&&') else Syntax.BinaryExpression + self.operator = operator + self.left = left + self.right = right + + +class BlockStatement(Node): + def __init__(self, body): + self.type = Syntax.BlockStatement + self.body = body + + +class BreakStatement(Node): + def __init__(self, label): + self.type = Syntax.BreakStatement + self.label = label + + +class CallExpression(Node): + def __init__(self, callee, args): + self.type = Syntax.CallExpression + self.callee = callee + self.arguments = args + + +class CatchClause(Node): + def __init__(self, param, body): + self.type = Syntax.CatchClause + self.param = param + self.body = body + + +class ClassBody(Node): + def __init__(self, body): + self.type = Syntax.ClassBody + self.body = body + + +class ClassDeclaration(Node): + def __init__(self, id, superClass, body): + self.type = Syntax.ClassDeclaration + self.id = id + self.superClass = superClass + self.body = body + + +class ClassExpression(Node): + def __init__(self, id, superClass, body): + self.type = Syntax.ClassExpression + self.id = id + self.superClass = superClass + self.body = body + + +class ComputedMemberExpression(Node): + def __init__(self, object, property): + self.type = Syntax.MemberExpression + self.computed = True + self.object = object + self.property = property + + +class ConditionalExpression(Node): + def __init__(self, test, consequent, alternate): + self.type = Syntax.ConditionalExpression + self.test = test + self.consequent = consequent + self.alternate = alternate + + +class ContinueStatement(Node): + def __init__(self, label): + self.type = Syntax.ContinueStatement + self.label = label + + +class DebuggerStatement(Node): + def __init__(self): + self.type = Syntax.DebuggerStatement + + +class Directive(Node): + def __init__(self, expression, directive): + self.type = Syntax.ExpressionStatement + self.expression = expression + self.directive = directive + + +class DoWhileStatement(Node): + def __init__(self, body, test): + self.type = Syntax.DoWhileStatement + self.body = body + self.test = test + + +class EmptyStatement(Node): + def __init__(self): + self.type = Syntax.EmptyStatement + + +class ExportAllDeclaration(Node): + def __init__(self, source): + self.type = Syntax.ExportAllDeclaration + self.source = source + + +class ExportDefaultDeclaration(Node): + def __init__(self, declaration): + self.type = Syntax.ExportDefaultDeclaration + self.declaration = declaration + + +class ExportNamedDeclaration(Node): + def __init__(self, declaration, specifiers, source): + self.type = Syntax.ExportNamedDeclaration + self.declaration = declaration + self.specifiers = specifiers + self.source = source + + +class ExportSpecifier(Node): + def __init__(self, local, exported): + self.type = Syntax.ExportSpecifier + self.exported = exported + self.local = local + + +class ExportDefaultSpecifier(Node): + def __init__(self, local): + self.type = Syntax.ExportDefaultSpecifier + self.local = local + + +class ExpressionStatement(Node): + def __init__(self, expression): + self.type = Syntax.ExpressionStatement + self.expression = expression + + +class ForInStatement(Node): + def __init__(self, left, right, body): + self.type = Syntax.ForInStatement + self.each = False + self.left = left + self.right = right + self.body = body + + +class ForOfStatement(Node): + def __init__(self, left, right, body): + self.type = Syntax.ForOfStatement + self.left = left + self.right = right + self.body = body + + +class ForStatement(Node): + def __init__(self, init, test, update, body): + self.type = Syntax.ForStatement + self.init = init + self.test = test + self.update = update + self.body = body + + +class FunctionDeclaration(Node): + def __init__(self, id, params, body, generator): + self.type = Syntax.FunctionDeclaration + self.expression = False + self.isAsync = False + self.id = id + self.params = params + self.body = body + self.generator = generator + + +class FunctionExpression(Node): + def __init__(self, id, params, body, generator): + self.type = Syntax.FunctionExpression + self.expression = False + self.isAsync = False + self.id = id + self.params = params + self.body = body + self.generator = generator + + +class Identifier(Node): + def __init__(self, name): + self.type = Syntax.Identifier + self.name = name + + +class IfStatement(Node): + def __init__(self, test, consequent, alternate): + self.type = Syntax.IfStatement + self.test = test + self.consequent = consequent + self.alternate = alternate + + +class Import(Node): + def __init__(self): + self.type = Syntax.Import + + +class ImportDeclaration(Node): + def __init__(self, specifiers, source): + self.type = Syntax.ImportDeclaration + self.specifiers = specifiers + self.source = source + + +class ImportDefaultSpecifier(Node): + def __init__(self, local): + self.type = Syntax.ImportDefaultSpecifier + self.local = local + + +class ImportNamespaceSpecifier(Node): + def __init__(self, local): + self.type = Syntax.ImportNamespaceSpecifier + self.local = local + + +class ImportSpecifier(Node): + def __init__(self, local, imported): + self.type = Syntax.ImportSpecifier + self.local = local + self.imported = imported + + +class LabeledStatement(Node): + def __init__(self, label, body): + self.type = Syntax.LabeledStatement + self.label = label + self.body = body + + +class Literal(Node): + def __init__(self, value, raw): + self.type = Syntax.Literal + self.value = value + self.raw = raw + + +class MetaProperty(Node): + def __init__(self, meta, property): + self.type = Syntax.MetaProperty + self.meta = meta + self.property = property + + +class MethodDefinition(Node): + def __init__(self, key, computed, value, kind, isStatic): + self.type = Syntax.MethodDefinition + self.key = key + self.computed = computed + self.value = value + self.kind = kind + self.static = isStatic + + +class FieldDefinition(Node): + def __init__(self, key, computed, value, kind, isStatic): + self.type = Syntax.FieldDefinition + self.key = key + self.computed = computed + self.value = value + self.kind = kind + self.static = isStatic + + +class Module(Node): + def __init__(self, body): + self.type = Syntax.Program + self.sourceType = 'module' + self.body = body + + +class NewExpression(Node): + def __init__(self, callee, args): + self.type = Syntax.NewExpression + self.callee = callee + self.arguments = args + + +class ObjectExpression(Node): + def __init__(self, properties): + self.type = Syntax.ObjectExpression + self.properties = properties + + +class ObjectPattern(Node): + def __init__(self, properties): + self.type = Syntax.ObjectPattern + self.properties = properties + + +class Property(Node): + def __init__(self, kind, key, computed, value, method, shorthand): + self.type = Syntax.Property + self.key = key + self.computed = computed + self.value = value + self.kind = kind + self.method = method + self.shorthand = shorthand + + +class RegexLiteral(Node): + def __init__(self, value, raw, pattern, flags): + self.type = Syntax.Literal + self.value = value + self.raw = raw + self.regex = RegExp( + pattern=pattern, + flags=flags, + ) + + +class RestElement(Node): + def __init__(self, argument): + self.type = Syntax.RestElement + self.argument = argument + + +class ReturnStatement(Node): + def __init__(self, argument): + self.type = Syntax.ReturnStatement + self.argument = argument + + +class Script(Node): + def __init__(self, body): + self.type = Syntax.Program + self.sourceType = 'script' + self.body = body + + +class SequenceExpression(Node): + def __init__(self, expressions): + self.type = Syntax.SequenceExpression + self.expressions = expressions + + +class SpreadElement(Node): + def __init__(self, argument): + self.type = Syntax.SpreadElement + self.argument = argument + + +class StaticMemberExpression(Node): + def __init__(self, object, property): + self.type = Syntax.MemberExpression + self.computed = False + self.object = object + self.property = property + + +class Super(Node): + def __init__(self): + self.type = Syntax.Super + + +class SwitchCase(Node): + def __init__(self, test, consequent): + self.type = Syntax.SwitchCase + self.test = test + self.consequent = consequent + + +class SwitchStatement(Node): + def __init__(self, discriminant, cases): + self.type = Syntax.SwitchStatement + self.discriminant = discriminant + self.cases = cases + + +class TaggedTemplateExpression(Node): + def __init__(self, tag, quasi): + self.type = Syntax.TaggedTemplateExpression + self.tag = tag + self.quasi = quasi + + +class TemplateElement(Node): + class Value(Object): + def __init__(self, raw, cooked): + self.raw = raw + self.cooked = cooked + + def __init__(self, raw, cooked, tail): + self.type = Syntax.TemplateElement + self.value = TemplateElement.Value(raw, cooked) + self.tail = tail + + +class TemplateLiteral(Node): + def __init__(self, quasis, expressions): + self.type = Syntax.TemplateLiteral + self.quasis = quasis + self.expressions = expressions + + +class ThisExpression(Node): + def __init__(self): + self.type = Syntax.ThisExpression + + +class ThrowStatement(Node): + def __init__(self, argument): + self.type = Syntax.ThrowStatement + self.argument = argument + + +class TryStatement(Node): + def __init__(self, block, handler, finalizer): + self.type = Syntax.TryStatement + self.block = block + self.handler = handler + self.finalizer = finalizer + + +class UnaryExpression(Node): + def __init__(self, operator, argument): + self.type = Syntax.UnaryExpression + self.prefix = True + self.operator = operator + self.argument = argument + + +class UpdateExpression(Node): + def __init__(self, operator, argument, prefix): + self.type = Syntax.UpdateExpression + self.operator = operator + self.argument = argument + self.prefix = prefix + + +class VariableDeclaration(Node): + def __init__(self, declarations, kind): + self.type = Syntax.VariableDeclaration + self.declarations = declarations + self.kind = kind + + +class VariableDeclarator(Node): + def __init__(self, id, init): + self.type = Syntax.VariableDeclarator + self.id = id + self.init = init + + +class WhileStatement(Node): + def __init__(self, test, body): + self.type = Syntax.WhileStatement + self.test = test + self.body = body + + +class WithStatement(Node): + def __init__(self, object, body): + self.type = Syntax.WithStatement + self.object = object + self.body = body + + +class YieldExpression(Node): + def __init__(self, argument, delegate): + self.type = Syntax.YieldExpression + self.argument = argument + self.delegate = delegate + + +class ArrowParameterPlaceHolder(Node): + def __init__(self, params): + self.type = Syntax.ArrowParameterPlaceHolder + self.params = params + self.isAsync = False + + +class AsyncArrowParameterPlaceHolder(Node): + def __init__(self, params): + self.type = Syntax.ArrowParameterPlaceHolder + self.params = params + self.isAsync = True + + +class BlockComment(Node): + def __init__(self, value): + self.type = Syntax.BlockComment + self.value = value + + +class LineComment(Node): + def __init__(self, value): + self.type = Syntax.LineComment + self.value = value diff --git a/third_party/python/esprima/esprima/objects.py b/third_party/python/esprima/esprima/objects.py new file mode 100644 index 000000000000..a8acca1b63a3 --- /dev/null +++ b/third_party/python/esprima/esprima/objects.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + + +def toDict(value): + from .visitor import ToDictVisitor + return ToDictVisitor().visit(value) + + +class Array(list): + pass + + +class Object(object): + def toDict(self): + from .visitor import ToDictVisitor + return ToDictVisitor().visit(self) + + def __repr__(self): + from .visitor import ReprVisitor + return ReprVisitor().visit(self) + + def __getattr__(self, name): + return None diff --git a/third_party/python/esprima/esprima/parser.py b/third_party/python/esprima/esprima/parser.py new file mode 100644 index 000000000000..2309e7b6fb8f --- /dev/null +++ b/third_party/python/esprima/esprima/parser.py @@ -0,0 +1,3104 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .objects import Object +from .compat import basestring, unicode +from .utils import format +from .error_handler import ErrorHandler +from .messages import Messages +from .scanner import RawToken, Scanner, SourceLocation, Position, RegExp +from .token import Token, TokenName +from .syntax import Syntax +from . import nodes as Node + + +class Value(object): + def __init__(self, value): + self.value = value + + +class Params(object): + def __init__(self, simple=None, message=None, stricted=None, firstRestricted=None, inFor=None, paramSet=None, params=None, get=None): + self.simple = simple + self.message = message + self.stricted = stricted + self.firstRestricted = firstRestricted + self.inFor = inFor + self.paramSet = paramSet + self.params = params + self.get = get + + +class Config(Object): + def __init__(self, range=False, loc=False, source=None, tokens=False, comment=False, tolerant=False, **options): + self.range = range + self.loc = loc + self.source = source + self.tokens = tokens + self.comment = comment + self.tolerant = tolerant + for k, v in options.items(): + setattr(self, k, v) + + +class Context(object): + def __init__(self, isModule=False, allowAwait=False, allowIn=True, allowStrictDirective=True, allowYield=True, firstCoverInitializedNameError=None, isAssignmentTarget=False, isBindingElement=False, inFunctionBody=False, inIteration=False, inSwitch=False, labelSet=None, strict=False): + self.isModule = isModule + self.allowAwait = allowAwait + self.allowIn = allowIn + self.allowStrictDirective = allowStrictDirective + self.allowYield = allowYield + self.firstCoverInitializedNameError = firstCoverInitializedNameError + self.isAssignmentTarget = isAssignmentTarget + self.isBindingElement = isBindingElement + self.inFunctionBody = inFunctionBody + self.inIteration = inIteration + self.inSwitch = inSwitch + self.labelSet = {} if labelSet is None else labelSet + self.strict = strict + + +class Marker(object): + def __init__(self, index=None, line=None, column=None): + self.index = index + self.line = line + self.column = column + + +class TokenEntry(Object): + def __init__(self, type=None, value=None, regex=None, range=None, loc=None): + self.type = type + self.value = value + self.regex = regex + self.range = range + self.loc = loc + + +class Parser(object): + def __init__(self, code, options={}, delegate=None): + self.config = Config(**options) + + self.delegate = delegate + + self.errorHandler = ErrorHandler() + self.errorHandler.tolerant = self.config.tolerant + self.scanner = Scanner(code, self.errorHandler) + self.scanner.trackComment = self.config.comment + + self.operatorPrecedence = { + '||': 1, + '&&': 2, + '|': 3, + '^': 4, + '&': 5, + '==': 6, + '!=': 6, + '===': 6, + '!==': 6, + '<': 7, + '>': 7, + '<=': 7, + '>=': 7, + 'instanceof': 7, + 'in': 7, + '<<': 8, + '>>': 8, + '>>>': 8, + '+': 9, + '-': 9, + '*': 11, + '/': 11, + '%': 11, + } + + self.lookahead = RawToken( + type=Token.EOF, + value='', + lineNumber=self.scanner.lineNumber, + lineStart=0, + start=0, + end=0 + ) + self.hasLineTerminator = False + + self.context = Context( + isModule=False, + allowAwait=False, + allowIn=True, + allowStrictDirective=True, + allowYield=True, + firstCoverInitializedNameError=None, + isAssignmentTarget=False, + isBindingElement=False, + inFunctionBody=False, + inIteration=False, + inSwitch=False, + labelSet={}, + strict=False + ) + self.tokens = [] + + self.startMarker = Marker( + index=0, + line=self.scanner.lineNumber, + column=0 + ) + self.lastMarker = Marker( + index=0, + line=self.scanner.lineNumber, + column=0 + ) + self.nextToken() + self.lastMarker = Marker( + index=self.scanner.index, + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart + ) + + def throwError(self, messageFormat, *args): + msg = format(messageFormat, *args) + index = self.lastMarker.index + line = self.lastMarker.line + column = self.lastMarker.column + 1 + raise self.errorHandler.createError(index, line, column, msg) + + def tolerateError(self, messageFormat, *args): + msg = format(messageFormat, *args) + index = self.lastMarker.index + line = self.scanner.lineNumber + column = self.lastMarker.column + 1 + self.errorHandler.tolerateError(index, line, column, msg) + + # Throw an exception because of the token. + + def unexpectedTokenError(self, token=None, message=None): + msg = message or Messages.UnexpectedToken + if token: + if not message: + typ = token.type + if typ is Token.EOF: + msg = Messages.UnexpectedEOS + elif typ is Token.Identifier: + msg = Messages.UnexpectedIdentifier + elif typ is Token.NumericLiteral: + msg = Messages.UnexpectedNumber + elif typ is Token.StringLiteral: + msg = Messages.UnexpectedString + elif typ is Token.Template: + msg = Messages.UnexpectedTemplate + elif typ is Token.Keyword: + if self.scanner.isFutureReservedWord(token.value): + msg = Messages.UnexpectedReserved + elif self.context.strict and self.scanner.isStrictModeReservedWord(token.value): + msg = Messages.StrictReservedWord + else: + msg = Messages.UnexpectedToken + value = token.value + else: + value = 'ILLEGAL' + + msg = msg.replace('%0', unicode(value), 1) + + if token and isinstance(token.lineNumber, int): + index = token.start + line = token.lineNumber + lastMarkerLineStart = self.lastMarker.index - self.lastMarker.column + column = token.start - lastMarkerLineStart + 1 + return self.errorHandler.createError(index, line, column, msg) + else: + index = self.lastMarker.index + line = self.lastMarker.line + column = self.lastMarker.column + 1 + return self.errorHandler.createError(index, line, column, msg) + + def throwUnexpectedToken(self, token=None, message=None): + raise self.unexpectedTokenError(token, message) + + def tolerateUnexpectedToken(self, token=None, message=None): + self.errorHandler.tolerate(self.unexpectedTokenError(token, message)) + + def collectComments(self): + if not self.config.comment: + self.scanner.scanComments() + else: + comments = self.scanner.scanComments() + if comments: + for e in comments: + if e.multiLine: + node = Node.BlockComment(self.scanner.source[e.slice[0]:e.slice[1]]) + else: + node = Node.LineComment(self.scanner.source[e.slice[0]:e.slice[1]]) + if self.config.range: + node.range = e.range + if self.config.loc: + node.loc = e.loc + if self.delegate: + metadata = SourceLocation( + start=Position( + line=e.loc.start.line, + column=e.loc.start.column, + offset=e.range[0], + ), + end=Position( + line=e.loc.end.line, + column=e.loc.end.column, + offset=e.range[1], + ) + ) + new_node = self.delegate(node, metadata) + if new_node is not None: + node = new_node + + # From internal representation to an external structure + + def getTokenRaw(self, token): + return self.scanner.source[token.start:token.end] + + def convertToken(self, token): + t = TokenEntry( + type=TokenName[token.type], + value=self.getTokenRaw(token), + ) + if self.config.range: + t.range = [token.start, token.end] + if self.config.loc: + t.loc = SourceLocation( + start=Position( + line=self.startMarker.line, + column=self.startMarker.column, + ), + end=Position( + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart, + ), + ) + if token.type is Token.RegularExpression: + t.regex = RegExp( + pattern=token.pattern, + flags=token.flags, + ) + + return t + + def nextToken(self): + token = self.lookahead + + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + + self.collectComments() + + if self.scanner.index != self.startMarker.index: + self.startMarker.index = self.scanner.index + self.startMarker.line = self.scanner.lineNumber + self.startMarker.column = self.scanner.index - self.scanner.lineStart + + next = self.scanner.lex() + self.hasLineTerminator = token.lineNumber != next.lineNumber + + if next and self.context.strict and next.type is Token.Identifier: + if self.scanner.isStrictModeReservedWord(next.value): + next.type = Token.Keyword + self.lookahead = next + + if self.config.tokens and next.type is not Token.EOF: + self.tokens.append(self.convertToken(next)) + + return token + + def nextRegexToken(self): + self.collectComments() + + token = self.scanner.scanRegExp() + if self.config.tokens: + # Pop the previous token, '/' or '/=' + # self is added from the lookahead token. + self.tokens.pop() + + self.tokens.append(self.convertToken(token)) + + # Prime the next lookahead. + self.lookahead = token + self.nextToken() + + return token + + def createNode(self): + return Marker( + index=self.startMarker.index, + line=self.startMarker.line, + column=self.startMarker.column, + ) + + def startNode(self, token, lastLineStart=0): + column = token.start - token.lineStart + line = token.lineNumber + if column < 0: + column += lastLineStart + line -= 1 + + return Marker( + index=token.start, + line=line, + column=column, + ) + + def finalize(self, marker, node): + if self.config.range: + node.range = [marker.index, self.lastMarker.index] + + if self.config.loc: + node.loc = SourceLocation( + start=Position( + line=marker.line, + column=marker.column, + ), + end=Position( + line=self.lastMarker.line, + column=self.lastMarker.column, + ), + ) + if self.config.source: + node.loc.source = self.config.source + + if self.delegate: + metadata = SourceLocation( + start=Position( + line=marker.line, + column=marker.column, + offset=marker.index, + ), + end=Position( + line=self.lastMarker.line, + column=self.lastMarker.column, + offset=self.lastMarker.index, + ) + ) + new_node = self.delegate(node, metadata) + if new_node is not None: + node = new_node + + return node + + # Expect the next token to match the specified punctuator. + # If not, an exception will be thrown. + + def expect(self, value): + token = self.nextToken() + if token.type is not Token.Punctuator or token.value != value: + self.throwUnexpectedToken(token) + + # Quietly expect a comma when in tolerant mode, otherwise delegates to expect(). + + def expectCommaSeparator(self): + if self.config.tolerant: + token = self.lookahead + if token.type is Token.Punctuator and token.value == ',': + self.nextToken() + elif token.type is Token.Punctuator and token.value == ';': + self.nextToken() + self.tolerateUnexpectedToken(token) + else: + self.tolerateUnexpectedToken(token, Messages.UnexpectedToken) + else: + self.expect(',') + + # Expect the next token to match the specified keyword. + # If not, an exception will be thrown. + + def expectKeyword(self, keyword): + token = self.nextToken() + if token.type is not Token.Keyword or token.value != keyword: + self.throwUnexpectedToken(token) + + # Return true if the next token matches the specified punctuator. + + def match(self, *value): + return self.lookahead.type is Token.Punctuator and self.lookahead.value in value + + # Return true if the next token matches the specified keyword + + def matchKeyword(self, *keyword): + return self.lookahead.type is Token.Keyword and self.lookahead.value in keyword + + # Return true if the next token matches the specified contextual keyword + # (where an identifier is sometimes a keyword depending on the context) + + def matchContextualKeyword(self, *keyword): + return self.lookahead.type is Token.Identifier and self.lookahead.value in keyword + + # Return true if the next token is an assignment operator + + def matchAssign(self): + if self.lookahead.type is not Token.Punctuator: + return False + + op = self.lookahead.value + return op in ('=', '*=', '**=', '/=', '%=', '+=', '-=', '<<=', '>>=', '>>>=', '&=', '^=', '|=') + + # Cover grammar support. + # + # When an assignment expression position starts with an left parenthesis, the determination of the type + # of the syntax is to be deferred arbitrarily long until the end of the parentheses pair (plus a lookahead) + # or the first comma. This situation also defers the determination of all the expressions nested in the pair. + # + # There are three productions that can be parsed in a parentheses pair that needs to be determined + # after the outermost pair is closed. They are: + # + # 1. AssignmentExpression + # 2. BindingElements + # 3. AssignmentTargets + # + # In order to avoid exponential backtracking, we use two flags to denote if the production can be + # binding element or assignment target. + # + # The three productions have the relationship: + # + # BindingElements ⊆ AssignmentTargets ⊆ AssignmentExpression + # + # with a single exception that CoverInitializedName when used directly in an Expression, generates + # an early error. Therefore, we need the third state, firstCoverInitializedNameError, to track the + # first usage of CoverInitializedName and report it when we reached the end of the parentheses pair. + # + # isolateCoverGrammar function runs the given parser function with a new cover grammar context, and it does not + # effect the current flags. This means the production the parser parses is only used as an expression. Therefore + # the CoverInitializedName check is conducted. + # + # inheritCoverGrammar function runs the given parse function with a new cover grammar context, and it propagates + # the flags outside of the parser. This means the production the parser parses is used as a part of a potential + # pattern. The CoverInitializedName check is deferred. + + def isolateCoverGrammar(self, parseFunction): + previousIsBindingElement = self.context.isBindingElement + previousIsAssignmentTarget = self.context.isAssignmentTarget + previousFirstCoverInitializedNameError = self.context.firstCoverInitializedNameError + + self.context.isBindingElement = True + self.context.isAssignmentTarget = True + self.context.firstCoverInitializedNameError = None + + result = parseFunction() + if self.context.firstCoverInitializedNameError is not None: + self.throwUnexpectedToken(self.context.firstCoverInitializedNameError) + + self.context.isBindingElement = previousIsBindingElement + self.context.isAssignmentTarget = previousIsAssignmentTarget + self.context.firstCoverInitializedNameError = previousFirstCoverInitializedNameError + + return result + + def inheritCoverGrammar(self, parseFunction): + previousIsBindingElement = self.context.isBindingElement + previousIsAssignmentTarget = self.context.isAssignmentTarget + previousFirstCoverInitializedNameError = self.context.firstCoverInitializedNameError + + self.context.isBindingElement = True + self.context.isAssignmentTarget = True + self.context.firstCoverInitializedNameError = None + + result = parseFunction() + + self.context.isBindingElement = self.context.isBindingElement and previousIsBindingElement + self.context.isAssignmentTarget = self.context.isAssignmentTarget and previousIsAssignmentTarget + self.context.firstCoverInitializedNameError = previousFirstCoverInitializedNameError or self.context.firstCoverInitializedNameError + + return result + + def consumeSemicolon(self): + if self.match(';'): + self.nextToken() + elif not self.hasLineTerminator: + if self.lookahead.type is not Token.EOF and not self.match('}'): + self.throwUnexpectedToken(self.lookahead) + self.lastMarker.index = self.startMarker.index + self.lastMarker.line = self.startMarker.line + self.lastMarker.column = self.startMarker.column + + # https://tc39.github.io/ecma262/#sec-primary-expression + + def parsePrimaryExpression(self): + node = self.createNode() + + typ = self.lookahead.type + if typ is Token.Identifier: + if (self.context.isModule or self.context.allowAwait) and self.lookahead.value == 'await': + self.tolerateUnexpectedToken(self.lookahead) + expr = self.parseFunctionExpression() if self.matchAsyncFunction() else self.finalize(node, Node.Identifier(self.nextToken().value)) + + elif typ in ( + Token.NumericLiteral, + Token.StringLiteral, + ): + if self.context.strict and self.lookahead.octal: + self.tolerateUnexpectedToken(self.lookahead, Messages.StrictOctalLiteral) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + token = self.nextToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.Literal(token.value, raw)) + + elif typ is Token.BooleanLiteral: + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + token = self.nextToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.Literal(token.value == 'true', raw)) + + elif typ is Token.NullLiteral: + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + token = self.nextToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.Literal(None, raw)) + + elif typ is Token.Template: + expr = self.parseTemplateLiteral() + + elif typ is Token.Punctuator: + value = self.lookahead.value + if value == '(': + self.context.isBindingElement = False + expr = self.inheritCoverGrammar(self.parseGroupExpression) + elif value == '[': + expr = self.inheritCoverGrammar(self.parseArrayInitializer) + elif value == '{': + expr = self.inheritCoverGrammar(self.parseObjectInitializer) + elif value in ('/', '/='): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + self.scanner.index = self.startMarker.index + token = self.nextRegexToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.RegexLiteral(token.regex, raw, token.pattern, token.flags)) + else: + expr = self.throwUnexpectedToken(self.nextToken()) + + elif typ is Token.Keyword: + if not self.context.strict and self.context.allowYield and self.matchKeyword('yield'): + expr = self.parseIdentifierName() + elif not self.context.strict and self.matchKeyword('let'): + expr = self.finalize(node, Node.Identifier(self.nextToken().value)) + else: + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + if self.matchKeyword('function'): + expr = self.parseFunctionExpression() + elif self.matchKeyword('this'): + self.nextToken() + expr = self.finalize(node, Node.ThisExpression()) + elif self.matchKeyword('class'): + expr = self.parseClassExpression() + elif self.matchImportCall(): + expr = self.parseImportCall() + else: + expr = self.throwUnexpectedToken(self.nextToken()) + + else: + expr = self.throwUnexpectedToken(self.nextToken()) + + return expr + + # https://tc39.github.io/ecma262/#sec-array-initializer + + def parseSpreadElement(self): + node = self.createNode() + self.expect('...') + arg = self.inheritCoverGrammar(self.parseAssignmentExpression) + return self.finalize(node, Node.SpreadElement(arg)) + + def parseArrayInitializer(self): + node = self.createNode() + elements = [] + + self.expect('[') + while not self.match(']'): + if self.match(','): + self.nextToken() + elements.append(None) + elif self.match('...'): + element = self.parseSpreadElement() + if not self.match(']'): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + self.expect(',') + elements.append(element) + else: + elements.append(self.inheritCoverGrammar(self.parseAssignmentExpression)) + if not self.match(']'): + self.expect(',') + self.expect(']') + + return self.finalize(node, Node.ArrayExpression(elements)) + + # https://tc39.github.io/ecma262/#sec-object-initializer + + def parsePropertyMethod(self, params): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = params.simple + body = self.isolateCoverGrammar(self.parseFunctionSourceElements) + if self.context.strict and params.firstRestricted: + self.tolerateUnexpectedToken(params.firstRestricted, params.message) + if self.context.strict and params.stricted: + self.tolerateUnexpectedToken(params.stricted, params.message) + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + + return body + + def parsePropertyMethodFunction(self): + isGenerator = False + node = self.createNode() + + previousAllowYield = self.context.allowYield + self.context.allowYield = True + params = self.parseFormalParameters() + method = self.parsePropertyMethod(params) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, params.params, method, isGenerator)) + + def parsePropertyMethodAsyncFunction(self): + node = self.createNode() + + previousAllowYield = self.context.allowYield + previousAwait = self.context.allowAwait + self.context.allowYield = False + self.context.allowAwait = True + params = self.parseFormalParameters() + method = self.parsePropertyMethod(params) + self.context.allowYield = previousAllowYield + self.context.allowAwait = previousAwait + + return self.finalize(node, Node.AsyncFunctionExpression(None, params.params, method)) + + def parseObjectPropertyKey(self): + node = self.createNode() + token = self.nextToken() + + typ = token.type + if typ in ( + Token.StringLiteral, + Token.NumericLiteral, + ): + if self.context.strict and token.octal: + self.tolerateUnexpectedToken(token, Messages.StrictOctalLiteral) + raw = self.getTokenRaw(token) + key = self.finalize(node, Node.Literal(token.value, raw)) + + elif typ in ( + Token.Identifier, + Token.BooleanLiteral, + Token.NullLiteral, + Token.Keyword, + ): + key = self.finalize(node, Node.Identifier(token.value)) + + elif typ is Token.Punctuator: + if token.value == '[': + key = self.isolateCoverGrammar(self.parseAssignmentExpression) + self.expect(']') + else: + key = self.throwUnexpectedToken(token) + + else: + key = self.throwUnexpectedToken(token) + + return key + + def isPropertyKey(self, key, value): + return ( + (key.type is Syntax.Identifier and key.name == value) or + (key.type is Syntax.Literal and key.value == value) + ) + + def parseObjectProperty(self, hasProto): + node = self.createNode() + token = self.lookahead + + key = None + value = None + + computed = False + method = False + shorthand = False + isAsync = False + + if token.type is Token.Identifier: + id = token.value + self.nextToken() + computed = self.match('[') + isAsync = not self.hasLineTerminator and (id == 'async') and not (self.match(':', '(', '*', ',')) + key = self.parseObjectPropertyKey() if isAsync else self.finalize(node, Node.Identifier(id)) + elif self.match('*'): + self.nextToken() + else: + computed = self.match('[') + key = self.parseObjectPropertyKey() + + lookaheadPropertyKey = self.qualifiedPropertyName(self.lookahead) + if token.type is Token.Identifier and not isAsync and token.value == 'get' and lookaheadPropertyKey: + kind = 'get' + computed = self.match('[') + key = self.parseObjectPropertyKey() + self.context.allowYield = False + value = self.parseGetterMethod() + + elif token.type is Token.Identifier and not isAsync and token.value == 'set' and lookaheadPropertyKey: + kind = 'set' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseSetterMethod() + + elif token.type is Token.Punctuator and token.value == '*' and lookaheadPropertyKey: + kind = 'init' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseGeneratorMethod() + method = True + + else: + if not key: + self.throwUnexpectedToken(self.lookahead) + + kind = 'init' + if self.match(':') and not isAsync: + if not computed and self.isPropertyKey(key, '__proto__'): + if hasProto.value: + self.tolerateError(Messages.DuplicateProtoProperty) + hasProto.value = True + self.nextToken() + value = self.inheritCoverGrammar(self.parseAssignmentExpression) + + elif self.match('('): + value = self.parsePropertyMethodAsyncFunction() if isAsync else self.parsePropertyMethodFunction() + method = True + + elif token.type is Token.Identifier: + id = self.finalize(node, Node.Identifier(token.value)) + if self.match('='): + self.context.firstCoverInitializedNameError = self.lookahead + self.nextToken() + shorthand = True + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + value = self.finalize(node, Node.AssignmentPattern(id, init)) + else: + shorthand = True + value = id + else: + self.throwUnexpectedToken(self.nextToken()) + + return self.finalize(node, Node.Property(kind, key, computed, value, method, shorthand)) + + def parseObjectInitializer(self): + node = self.createNode() + + self.expect('{') + properties = [] + hasProto = Value(False) + while not self.match('}'): + properties.append(self.parseSpreadElement() if self.match('...') else self.parseObjectProperty(hasProto)) + if not self.match('}'): + self.expectCommaSeparator() + self.expect('}') + + return self.finalize(node, Node.ObjectExpression(properties)) + + # https://tc39.github.io/ecma262/#sec-template-literals + + def parseTemplateHead(self): + assert self.lookahead.head, 'Template literal must start with a template head' + + node = self.createNode() + token = self.nextToken() + raw = token.value + cooked = token.cooked + + return self.finalize(node, Node.TemplateElement(raw, cooked, token.tail)) + + def parseTemplateElement(self): + if self.lookahead.type is not Token.Template: + self.throwUnexpectedToken() + + node = self.createNode() + token = self.nextToken() + raw = token.value + cooked = token.cooked + + return self.finalize(node, Node.TemplateElement(raw, cooked, token.tail)) + + def parseTemplateLiteral(self): + node = self.createNode() + + expressions = [] + quasis = [] + + quasi = self.parseTemplateHead() + quasis.append(quasi) + while not quasi.tail: + expressions.append(self.parseExpression()) + quasi = self.parseTemplateElement() + quasis.append(quasi) + + return self.finalize(node, Node.TemplateLiteral(quasis, expressions)) + + # https://tc39.github.io/ecma262/#sec-grouping-operator + + def reinterpretExpressionAsPattern(self, expr): + typ = expr.type + if typ in ( + Syntax.Identifier, + Syntax.MemberExpression, + Syntax.RestElement, + Syntax.AssignmentPattern, + ): + pass + elif typ is Syntax.SpreadElement: + expr.type = Syntax.RestElement + self.reinterpretExpressionAsPattern(expr.argument) + elif typ is Syntax.ArrayExpression: + expr.type = Syntax.ArrayPattern + for elem in expr.elements: + if elem is not None: + self.reinterpretExpressionAsPattern(elem) + elif typ is Syntax.ObjectExpression: + expr.type = Syntax.ObjectPattern + for prop in expr.properties: + self.reinterpretExpressionAsPattern(prop if prop.type is Syntax.SpreadElement else prop.value) + elif typ is Syntax.AssignmentExpression: + expr.type = Syntax.AssignmentPattern + del expr.operator + self.reinterpretExpressionAsPattern(expr.left) + else: + # Allow other node type for tolerant parsing. + pass + + def parseGroupExpression(self): + self.expect('(') + if self.match(')'): + self.nextToken() + if not self.match('=>'): + self.expect('=>') + expr = Node.ArrowParameterPlaceHolder([]) + else: + startToken = self.lookahead + params = [] + if self.match('...'): + expr = self.parseRestElement(params) + self.expect(')') + if not self.match('=>'): + self.expect('=>') + expr = Node.ArrowParameterPlaceHolder([expr]) + else: + arrow = False + self.context.isBindingElement = True + expr = self.inheritCoverGrammar(self.parseAssignmentExpression) + + if self.match(','): + expressions = [] + + self.context.isAssignmentTarget = False + expressions.append(expr) + while self.lookahead.type is not Token.EOF: + if not self.match(','): + break + self.nextToken() + if self.match(')'): + self.nextToken() + for expression in expressions: + self.reinterpretExpressionAsPattern(expression) + arrow = True + expr = Node.ArrowParameterPlaceHolder(expressions) + elif self.match('...'): + if not self.context.isBindingElement: + self.throwUnexpectedToken(self.lookahead) + expressions.append(self.parseRestElement(params)) + self.expect(')') + if not self.match('=>'): + self.expect('=>') + self.context.isBindingElement = False + for expression in expressions: + self.reinterpretExpressionAsPattern(expression) + arrow = True + expr = Node.ArrowParameterPlaceHolder(expressions) + else: + expressions.append(self.inheritCoverGrammar(self.parseAssignmentExpression)) + if arrow: + break + if not arrow: + expr = self.finalize(self.startNode(startToken), Node.SequenceExpression(expressions)) + + if not arrow: + self.expect(')') + if self.match('=>'): + if expr.type is Syntax.Identifier and expr.name == 'yield': + arrow = True + expr = Node.ArrowParameterPlaceHolder([expr]) + if not arrow: + if not self.context.isBindingElement: + self.throwUnexpectedToken(self.lookahead) + + if expr.type is Syntax.SequenceExpression: + for expression in expr.expressions: + self.reinterpretExpressionAsPattern(expression) + else: + self.reinterpretExpressionAsPattern(expr) + + if expr.type is Syntax.SequenceExpression: + parameters = expr.expressions + else: + parameters = [expr] + expr = Node.ArrowParameterPlaceHolder(parameters) + self.context.isBindingElement = False + + return expr + + # https://tc39.github.io/ecma262/#sec-left-hand-side-expressions + + def parseArguments(self): + self.expect('(') + args = [] + if not self.match(')'): + while True: + if self.match('...'): + expr = self.parseSpreadElement() + else: + expr = self.isolateCoverGrammar(self.parseAssignmentExpression) + args.append(expr) + if self.match(')'): + break + self.expectCommaSeparator() + if self.match(')'): + break + self.expect(')') + + return args + + def isIdentifierName(self, token): + return ( + token.type is Token.Identifier or + token.type is Token.Keyword or + token.type is Token.BooleanLiteral or + token.type is Token.NullLiteral + ) + + def parseIdentifierName(self): + node = self.createNode() + token = self.nextToken() + if not self.isIdentifierName(token): + self.throwUnexpectedToken(token) + return self.finalize(node, Node.Identifier(token.value)) + + def parseNewExpression(self): + node = self.createNode() + + id = self.parseIdentifierName() + assert id.name == 'new', 'New expression must start with `new`' + + if self.match('.'): + self.nextToken() + if self.lookahead.type is Token.Identifier and self.context.inFunctionBody and self.lookahead.value == 'target': + property = self.parseIdentifierName() + expr = Node.MetaProperty(id, property) + else: + self.throwUnexpectedToken(self.lookahead) + elif self.matchKeyword('import'): + self.throwUnexpectedToken(self.lookahead) + else: + callee = self.isolateCoverGrammar(self.parseLeftHandSideExpression) + args = self.parseArguments() if self.match('(') else [] + expr = Node.NewExpression(callee, args) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + return self.finalize(node, expr) + + def parseAsyncArgument(self): + arg = self.parseAssignmentExpression() + self.context.firstCoverInitializedNameError = None + return arg + + def parseAsyncArguments(self): + self.expect('(') + args = [] + if not self.match(')'): + while True: + if self.match('...'): + expr = self.parseSpreadElement() + else: + expr = self.isolateCoverGrammar(self.parseAsyncArgument) + args.append(expr) + if self.match(')'): + break + self.expectCommaSeparator() + if self.match(')'): + break + self.expect(')') + + return args + + def matchImportCall(self): + match = self.matchKeyword('import') + if match: + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.scanner.lex() + self.scanner.restoreState(state) + match = (next.type is Token.Punctuator) and (next.value == '(') + + return match + + def parseImportCall(self): + node = self.createNode() + self.expectKeyword('import') + return self.finalize(node, Node.Import()) + + def parseLeftHandSideExpressionAllowCall(self): + startToken = self.lookahead + maybeAsync = self.matchContextualKeyword('async') + + previousAllowIn = self.context.allowIn + self.context.allowIn = True + + if self.matchKeyword('super') and self.context.inFunctionBody: + expr = self.createNode() + self.nextToken() + expr = self.finalize(expr, Node.Super()) + if not self.match('(') and not self.match('.') and not self.match('['): + self.throwUnexpectedToken(self.lookahead) + else: + expr = self.inheritCoverGrammar(self.parseNewExpression if self.matchKeyword('new') else self.parsePrimaryExpression) + + while True: + if self.match('.'): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('.') + property = self.parseIdentifierName() + expr = self.finalize(self.startNode(startToken), Node.StaticMemberExpression(expr, property)) + + elif self.match('('): + asyncArrow = maybeAsync and (startToken.lineNumber == self.lookahead.lineNumber) + self.context.isBindingElement = False + self.context.isAssignmentTarget = False + if asyncArrow: + args = self.parseAsyncArguments() + else: + args = self.parseArguments() + if expr.type is Syntax.Import and len(args) != 1: + self.tolerateError(Messages.BadImportCallArity) + expr = self.finalize(self.startNode(startToken), Node.CallExpression(expr, args)) + if asyncArrow and self.match('=>'): + for arg in args: + self.reinterpretExpressionAsPattern(arg) + expr = Node.AsyncArrowParameterPlaceHolder(args) + elif self.match('['): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('[') + property = self.isolateCoverGrammar(self.parseExpression) + self.expect(']') + expr = self.finalize(self.startNode(startToken), Node.ComputedMemberExpression(expr, property)) + + elif self.lookahead.type is Token.Template and self.lookahead.head: + quasi = self.parseTemplateLiteral() + expr = self.finalize(self.startNode(startToken), Node.TaggedTemplateExpression(expr, quasi)) + + else: + break + + self.context.allowIn = previousAllowIn + + return expr + + def parseSuper(self): + node = self.createNode() + + self.expectKeyword('super') + if not self.match('[') and not self.match('.'): + self.throwUnexpectedToken(self.lookahead) + + return self.finalize(node, Node.Super()) + + def parseLeftHandSideExpression(self): + assert self.context.allowIn, 'callee of new expression always allow in keyword.' + + node = self.startNode(self.lookahead) + if self.matchKeyword('super') and self.context.inFunctionBody: + expr = self.parseSuper() + else: + expr = self.inheritCoverGrammar(self.parseNewExpression if self.matchKeyword('new') else self.parsePrimaryExpression) + + while True: + if self.match('['): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('[') + property = self.isolateCoverGrammar(self.parseExpression) + self.expect(']') + expr = self.finalize(node, Node.ComputedMemberExpression(expr, property)) + + elif self.match('.'): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('.') + property = self.parseIdentifierName() + expr = self.finalize(node, Node.StaticMemberExpression(expr, property)) + + elif self.lookahead.type is Token.Template and self.lookahead.head: + quasi = self.parseTemplateLiteral() + expr = self.finalize(node, Node.TaggedTemplateExpression(expr, quasi)) + + else: + break + + return expr + + # https://tc39.github.io/ecma262/#sec-update-expressions + + def parseUpdateExpression(self): + startToken = self.lookahead + + if self.match('++', '--'): + node = self.startNode(startToken) + token = self.nextToken() + expr = self.inheritCoverGrammar(self.parseUnaryExpression) + if self.context.strict and expr.type is Syntax.Identifier and self.scanner.isRestrictedWord(expr.name): + self.tolerateError(Messages.StrictLHSPrefix) + if not self.context.isAssignmentTarget: + self.tolerateError(Messages.InvalidLHSInAssignment) + prefix = True + expr = self.finalize(node, Node.UpdateExpression(token.value, expr, prefix)) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + else: + expr = self.inheritCoverGrammar(self.parseLeftHandSideExpressionAllowCall) + if not self.hasLineTerminator and self.lookahead.type is Token.Punctuator: + if self.match('++', '--'): + if self.context.strict and expr.type is Syntax.Identifier and self.scanner.isRestrictedWord(expr.name): + self.tolerateError(Messages.StrictLHSPostfix) + if not self.context.isAssignmentTarget: + self.tolerateError(Messages.InvalidLHSInAssignment) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + operator = self.nextToken().value + prefix = False + expr = self.finalize(self.startNode(startToken), Node.UpdateExpression(operator, expr, prefix)) + + return expr + + # https://tc39.github.io/ecma262/#sec-unary-operators + + def parseAwaitExpression(self): + node = self.createNode() + self.nextToken() + argument = self.parseUnaryExpression() + return self.finalize(node, Node.AwaitExpression(argument)) + + def parseUnaryExpression(self): + if ( + self.match('+', '-', '~', '!') or + self.matchKeyword('delete', 'void', 'typeof') + ): + node = self.startNode(self.lookahead) + token = self.nextToken() + expr = self.inheritCoverGrammar(self.parseUnaryExpression) + expr = self.finalize(node, Node.UnaryExpression(token.value, expr)) + if self.context.strict and expr.operator == 'delete' and expr.argument.type is Syntax.Identifier: + self.tolerateError(Messages.StrictDelete) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + elif self.context.allowAwait and self.matchContextualKeyword('await'): + expr = self.parseAwaitExpression() + else: + expr = self.parseUpdateExpression() + + return expr + + def parseExponentiationExpression(self): + startToken = self.lookahead + + expr = self.inheritCoverGrammar(self.parseUnaryExpression) + if expr.type is not Syntax.UnaryExpression and self.match('**'): + self.nextToken() + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + left = expr + right = self.isolateCoverGrammar(self.parseExponentiationExpression) + expr = self.finalize(self.startNode(startToken), Node.BinaryExpression('**', left, right)) + + return expr + + # https://tc39.github.io/ecma262/#sec-exp-operator + # https://tc39.github.io/ecma262/#sec-multiplicative-operators + # https://tc39.github.io/ecma262/#sec-additive-operators + # https://tc39.github.io/ecma262/#sec-bitwise-shift-operators + # https://tc39.github.io/ecma262/#sec-relational-operators + # https://tc39.github.io/ecma262/#sec-equality-operators + # https://tc39.github.io/ecma262/#sec-binary-bitwise-operators + # https://tc39.github.io/ecma262/#sec-binary-logical-operators + + def binaryPrecedence(self, token): + op = token.value + if token.type is Token.Punctuator: + precedence = self.operatorPrecedence.get(op, 0) + elif token.type is Token.Keyword: + precedence = 7 if (op == 'instanceof' or (self.context.allowIn and op == 'in')) else 0 + else: + precedence = 0 + return precedence + + def parseBinaryExpression(self): + startToken = self.lookahead + + expr = self.inheritCoverGrammar(self.parseExponentiationExpression) + + token = self.lookahead + prec = self.binaryPrecedence(token) + if prec > 0: + self.nextToken() + + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + markers = [startToken, self.lookahead] + left = expr + right = self.isolateCoverGrammar(self.parseExponentiationExpression) + + stack = [left, token.value, right] + precedences = [prec] + while True: + prec = self.binaryPrecedence(self.lookahead) + if prec <= 0: + break + + # Reduce: make a binary expression from the three topmost entries. + while len(stack) > 2 and prec <= precedences[-1]: + right = stack.pop() + operator = stack.pop() + precedences.pop() + left = stack.pop() + markers.pop() + node = self.startNode(markers[-1]) + stack.append(self.finalize(node, Node.BinaryExpression(operator, left, right))) + + # Shift. + stack.append(self.nextToken().value) + precedences.append(prec) + markers.append(self.lookahead) + stack.append(self.isolateCoverGrammar(self.parseExponentiationExpression)) + + # Final reduce to clean-up the stack. + i = len(stack) - 1 + expr = stack[i] + + lastMarker = markers.pop() + while i > 1: + marker = markers.pop() + lastLineStart = lastMarker.lineStart if lastMarker else 0 + node = self.startNode(marker, lastLineStart) + operator = stack[i - 1] + expr = self.finalize(node, Node.BinaryExpression(operator, stack[i - 2], expr)) + i -= 2 + lastMarker = marker + + return expr + + # https://tc39.github.io/ecma262/#sec-conditional-operator + + def parseConditionalExpression(self): + startToken = self.lookahead + + expr = self.inheritCoverGrammar(self.parseBinaryExpression) + if self.match('?'): + self.nextToken() + + previousAllowIn = self.context.allowIn + self.context.allowIn = True + consequent = self.isolateCoverGrammar(self.parseAssignmentExpression) + self.context.allowIn = previousAllowIn + + self.expect(':') + alternate = self.isolateCoverGrammar(self.parseAssignmentExpression) + + expr = self.finalize(self.startNode(startToken), Node.ConditionalExpression(expr, consequent, alternate)) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + return expr + + # https://tc39.github.io/ecma262/#sec-assignment-operators + + def checkPatternParam(self, options, param): + typ = param.type + if typ is Syntax.Identifier: + self.validateParam(options, param, param.name) + elif typ is Syntax.RestElement: + self.checkPatternParam(options, param.argument) + elif typ is Syntax.AssignmentPattern: + self.checkPatternParam(options, param.left) + elif typ is Syntax.ArrayPattern: + for element in param.elements: + if element is not None: + self.checkPatternParam(options, element) + elif typ is Syntax.ObjectPattern: + for prop in param.properties: + self.checkPatternParam(options, prop if prop.type is Syntax.RestElement else prop.value) + + options.simple = options.simple and isinstance(param, Node.Identifier) + + def reinterpretAsCoverFormalsList(self, expr): + params = [expr] + + asyncArrow = False + typ = expr.type + if typ is Syntax.Identifier: + pass + elif typ is Syntax.ArrowParameterPlaceHolder: + params = expr.params + asyncArrow = expr.isAsync + else: + return None + + options = Params( + simple=True, + paramSet={}, + ) + + for param in params: + if param.type is Syntax.AssignmentPattern: + if param.right.type is Syntax.YieldExpression: + if param.right.argument: + self.throwUnexpectedToken(self.lookahead) + param.right.type = Syntax.Identifier + param.right.name = 'yield' + del param.right.argument + del param.right.delegate + elif asyncArrow and param.type is Syntax.Identifier and param.name == 'await': + self.throwUnexpectedToken(self.lookahead) + self.checkPatternParam(options, param) + + if self.context.strict or not self.context.allowYield: + for param in params: + if param.type is Syntax.YieldExpression: + self.throwUnexpectedToken(self.lookahead) + + if options.message is Messages.StrictParamDupe: + token = options.stricted if self.context.strict else options.firstRestricted + self.throwUnexpectedToken(token, options.message) + + return Params( + simple=options.simple, + params=params, + stricted=options.stricted, + firstRestricted=options.firstRestricted, + message=options.message + ) + + def parseAssignmentExpression(self): + if not self.context.allowYield and self.matchKeyword('yield'): + expr = self.parseYieldExpression() + else: + startToken = self.lookahead + token = startToken + expr = self.parseConditionalExpression() + + if token.type is Token.Identifier and (token.lineNumber == self.lookahead.lineNumber) and token.value == 'async': + if self.lookahead.type is Token.Identifier or self.matchKeyword('yield'): + arg = self.parsePrimaryExpression() + self.reinterpretExpressionAsPattern(arg) + expr = Node.AsyncArrowParameterPlaceHolder([arg]) + + if expr.type is Syntax.ArrowParameterPlaceHolder or self.match('=>'): + + # https://tc39.github.io/ecma262/#sec-arrow-function-definitions + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + isAsync = expr.isAsync + list = self.reinterpretAsCoverFormalsList(expr) + + if list: + if self.hasLineTerminator: + self.tolerateUnexpectedToken(self.lookahead) + self.context.firstCoverInitializedNameError = None + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = list.simple + + previousAllowYield = self.context.allowYield + previousAwait = self.context.allowAwait + self.context.allowYield = True + self.context.allowAwait = isAsync + + node = self.startNode(startToken) + self.expect('=>') + if self.match('{'): + previousAllowIn = self.context.allowIn + self.context.allowIn = True + body = self.parseFunctionSourceElements() + self.context.allowIn = previousAllowIn + else: + body = self.isolateCoverGrammar(self.parseAssignmentExpression) + expression = body.type is not Syntax.BlockStatement + + if self.context.strict and list.firstRestricted: + self.throwUnexpectedToken(list.firstRestricted, list.message) + if self.context.strict and list.stricted: + self.tolerateUnexpectedToken(list.stricted, list.message) + if isAsync: + expr = self.finalize(node, Node.AsyncArrowFunctionExpression(list.params, body, expression)) + else: + expr = self.finalize(node, Node.ArrowFunctionExpression(list.params, body, expression)) + + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + self.context.allowYield = previousAllowYield + self.context.allowAwait = previousAwait + else: + if self.matchAssign(): + if not self.context.isAssignmentTarget: + self.tolerateError(Messages.InvalidLHSInAssignment) + + if self.context.strict and expr.type is Syntax.Identifier: + id = expr + if self.scanner.isRestrictedWord(id.name): + self.tolerateUnexpectedToken(token, Messages.StrictLHSAssignment) + if self.scanner.isStrictModeReservedWord(id.name): + self.tolerateUnexpectedToken(token, Messages.StrictReservedWord) + + if not self.match('='): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + else: + self.reinterpretExpressionAsPattern(expr) + + token = self.nextToken() + operator = token.value + right = self.isolateCoverGrammar(self.parseAssignmentExpression) + expr = self.finalize(self.startNode(startToken), Node.AssignmentExpression(operator, expr, right)) + self.context.firstCoverInitializedNameError = None + + return expr + + # https://tc39.github.io/ecma262/#sec-comma-operator + + def parseExpression(self): + startToken = self.lookahead + expr = self.isolateCoverGrammar(self.parseAssignmentExpression) + + if self.match(','): + expressions = [] + expressions.append(expr) + while self.lookahead.type is not Token.EOF: + if not self.match(','): + break + self.nextToken() + expressions.append(self.isolateCoverGrammar(self.parseAssignmentExpression)) + + expr = self.finalize(self.startNode(startToken), Node.SequenceExpression(expressions)) + + return expr + + # https://tc39.github.io/ecma262/#sec-block + + def parseStatementListItem(self): + self.context.isAssignmentTarget = True + self.context.isBindingElement = True + if self.lookahead.type is Token.Keyword: + value = self.lookahead.value + if value == 'export': + if not self.context.isModule: + self.tolerateUnexpectedToken(self.lookahead, Messages.IllegalExportDeclaration) + statement = self.parseExportDeclaration() + elif value == 'import': + if self.matchImportCall(): + statement = self.parseExpressionStatement() + else: + if not self.context.isModule: + self.tolerateUnexpectedToken(self.lookahead, Messages.IllegalImportDeclaration) + statement = self.parseImportDeclaration() + elif value == 'const': + statement = self.parseLexicalDeclaration(Params(inFor=False)) + elif value == 'function': + statement = self.parseFunctionDeclaration() + elif value == 'class': + statement = self.parseClassDeclaration() + elif value == 'let': + statement = self.parseLexicalDeclaration(Params(inFor=False)) if self.isLexicalDeclaration() else self.parseStatement() + else: + statement = self.parseStatement() + else: + statement = self.parseStatement() + + return statement + + def parseBlock(self): + node = self.createNode() + + self.expect('{') + block = [] + while True: + if self.match('}'): + break + block.append(self.parseStatementListItem()) + self.expect('}') + + return self.finalize(node, Node.BlockStatement(block)) + + # https://tc39.github.io/ecma262/#sec-let-and-const-declarations + + def parseLexicalBinding(self, kind, options): + node = self.createNode() + params = [] + id = self.parsePattern(params, kind) + + if self.context.strict and id.type is Syntax.Identifier: + if self.scanner.isRestrictedWord(id.name): + self.tolerateError(Messages.StrictVarName) + + init = None + if kind == 'const': + if not self.matchKeyword('in') and not self.matchContextualKeyword('of'): + if self.match('='): + self.nextToken() + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + else: + self.throwError(Messages.DeclarationMissingInitializer, 'const') + elif (not options.inFor and id.type is not Syntax.Identifier) or self.match('='): + self.expect('=') + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + + return self.finalize(node, Node.VariableDeclarator(id, init)) + + def parseBindingList(self, kind, options): + lst = [self.parseLexicalBinding(kind, options)] + + while self.match(','): + self.nextToken() + lst.append(self.parseLexicalBinding(kind, options)) + + return lst + + def isLexicalDeclaration(self): + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.scanner.lex() + self.scanner.restoreState(state) + + return ( + (next.type is Token.Identifier) or + (next.type is Token.Punctuator and next.value == '[') or + (next.type is Token.Punctuator and next.value == '{') or + (next.type is Token.Keyword and next.value == 'let') or + (next.type is Token.Keyword and next.value == 'yield') + ) + + def parseLexicalDeclaration(self, options): + node = self.createNode() + kind = self.nextToken().value + assert kind == 'let' or kind == 'const', 'Lexical declaration must be either or const' + + declarations = self.parseBindingList(kind, options) + self.consumeSemicolon() + + return self.finalize(node, Node.VariableDeclaration(declarations, kind)) + + # https://tc39.github.io/ecma262/#sec-destructuring-binding-patterns + + def parseBindingRestElement(self, params, kind=None): + node = self.createNode() + + self.expect('...') + arg = self.parsePattern(params, kind) + + return self.finalize(node, Node.RestElement(arg)) + + def parseArrayPattern(self, params, kind=None): + node = self.createNode() + + self.expect('[') + elements = [] + while not self.match(']'): + if self.match(','): + self.nextToken() + elements.append(None) + else: + if self.match('...'): + elements.append(self.parseBindingRestElement(params, kind)) + break + else: + elements.append(self.parsePatternWithDefault(params, kind)) + if not self.match(']'): + self.expect(',') + self.expect(']') + + return self.finalize(node, Node.ArrayPattern(elements)) + + def parsePropertyPattern(self, params, kind=None): + node = self.createNode() + + computed = False + shorthand = False + method = False + + key = None + + if self.lookahead.type is Token.Identifier: + keyToken = self.lookahead + key = self.parseVariableIdentifier() + init = self.finalize(node, Node.Identifier(keyToken.value)) + if self.match('='): + params.append(keyToken) + shorthand = True + self.nextToken() + expr = self.parseAssignmentExpression() + value = self.finalize(self.startNode(keyToken), Node.AssignmentPattern(init, expr)) + elif not self.match(':'): + params.append(keyToken) + shorthand = True + value = init + else: + self.expect(':') + value = self.parsePatternWithDefault(params, kind) + else: + computed = self.match('[') + key = self.parseObjectPropertyKey() + self.expect(':') + value = self.parsePatternWithDefault(params, kind) + + return self.finalize(node, Node.Property('init', key, computed, value, method, shorthand)) + + def parseRestProperty(self, params, kind): + node = self.createNode() + self.expect('...') + arg = self.parsePattern(params) + if self.match('='): + self.throwError(Messages.DefaultRestProperty) + if not self.match('}'): + self.throwError(Messages.PropertyAfterRestProperty) + return self.finalize(node, Node.RestElement(arg)) + + def parseObjectPattern(self, params, kind=None): + node = self.createNode() + properties = [] + + self.expect('{') + while not self.match('}'): + properties.append(self.parseRestProperty(params, kind) if self.match('...') else self.parsePropertyPattern(params, kind)) + if not self.match('}'): + self.expect(',') + self.expect('}') + + return self.finalize(node, Node.ObjectPattern(properties)) + + def parsePattern(self, params, kind=None): + if self.match('['): + pattern = self.parseArrayPattern(params, kind) + elif self.match('{'): + pattern = self.parseObjectPattern(params, kind) + else: + if self.matchKeyword('let') and (kind in ('const', 'let')): + self.tolerateUnexpectedToken(self.lookahead, Messages.LetInLexicalBinding) + params.append(self.lookahead) + pattern = self.parseVariableIdentifier(kind) + + return pattern + + def parsePatternWithDefault(self, params, kind=None): + startToken = self.lookahead + + pattern = self.parsePattern(params, kind) + if self.match('='): + self.nextToken() + previousAllowYield = self.context.allowYield + self.context.allowYield = True + right = self.isolateCoverGrammar(self.parseAssignmentExpression) + self.context.allowYield = previousAllowYield + pattern = self.finalize(self.startNode(startToken), Node.AssignmentPattern(pattern, right)) + + return pattern + + # https://tc39.github.io/ecma262/#sec-variable-statement + + def parseVariableIdentifier(self, kind=None): + node = self.createNode() + + token = self.nextToken() + if token.type is Token.Keyword and token.value == 'yield': + if self.context.strict: + self.tolerateUnexpectedToken(token, Messages.StrictReservedWord) + elif not self.context.allowYield: + self.throwUnexpectedToken(token) + elif token.type is not Token.Identifier: + if self.context.strict and token.type is Token.Keyword and self.scanner.isStrictModeReservedWord(token.value): + self.tolerateUnexpectedToken(token, Messages.StrictReservedWord) + else: + if self.context.strict or token.value != 'let' or kind != 'var': + self.throwUnexpectedToken(token) + elif (self.context.isModule or self.context.allowAwait) and token.type is Token.Identifier and token.value == 'await': + self.tolerateUnexpectedToken(token) + + return self.finalize(node, Node.Identifier(token.value)) + + def parseVariableDeclaration(self, options): + node = self.createNode() + + params = [] + id = self.parsePattern(params, 'var') + + if self.context.strict and id.type is Syntax.Identifier: + if self.scanner.isRestrictedWord(id.name): + self.tolerateError(Messages.StrictVarName) + + init = None + if self.match('='): + self.nextToken() + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + elif id.type is not Syntax.Identifier and not options.inFor: + self.expect('=') + + return self.finalize(node, Node.VariableDeclarator(id, init)) + + def parseVariableDeclarationList(self, options): + opt = Params(inFor=options.inFor) + + lst = [] + lst.append(self.parseVariableDeclaration(opt)) + while self.match(','): + self.nextToken() + lst.append(self.parseVariableDeclaration(opt)) + + return lst + + def parseVariableStatement(self): + node = self.createNode() + self.expectKeyword('var') + declarations = self.parseVariableDeclarationList(Params(inFor=False)) + self.consumeSemicolon() + + return self.finalize(node, Node.VariableDeclaration(declarations, 'var')) + + # https://tc39.github.io/ecma262/#sec-empty-statement + + def parseEmptyStatement(self): + node = self.createNode() + self.expect(';') + return self.finalize(node, Node.EmptyStatement()) + + # https://tc39.github.io/ecma262/#sec-expression-statement + + def parseExpressionStatement(self): + node = self.createNode() + expr = self.parseExpression() + self.consumeSemicolon() + return self.finalize(node, Node.ExpressionStatement(expr)) + + # https://tc39.github.io/ecma262/#sec-if-statement + + def parseIfClause(self): + if self.context.strict and self.matchKeyword('function'): + self.tolerateError(Messages.StrictFunction) + return self.parseStatement() + + def parseIfStatement(self): + node = self.createNode() + alternate = None + + self.expectKeyword('if') + self.expect('(') + test = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + consequent = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + consequent = self.parseIfClause() + if self.matchKeyword('else'): + self.nextToken() + alternate = self.parseIfClause() + + return self.finalize(node, Node.IfStatement(test, consequent, alternate)) + + # https://tc39.github.io/ecma262/#sec-do-while-statement + + def parseDoWhileStatement(self): + node = self.createNode() + self.expectKeyword('do') + + previousInIteration = self.context.inIteration + self.context.inIteration = True + body = self.parseStatement() + self.context.inIteration = previousInIteration + + self.expectKeyword('while') + self.expect('(') + test = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + else: + self.expect(')') + if self.match(';'): + self.nextToken() + + return self.finalize(node, Node.DoWhileStatement(body, test)) + + # https://tc39.github.io/ecma262/#sec-while-statement + + def parseWhileStatement(self): + node = self.createNode() + + self.expectKeyword('while') + self.expect('(') + test = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + body = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + + previousInIteration = self.context.inIteration + self.context.inIteration = True + body = self.parseStatement() + self.context.inIteration = previousInIteration + + return self.finalize(node, Node.WhileStatement(test, body)) + + # https://tc39.github.io/ecma262/#sec-for-statement + # https://tc39.github.io/ecma262/#sec-for-in-and-for-of-statements + + def parseForStatement(self): + init = None + test = None + update = None + forIn = True + left = None + right = None + + node = self.createNode() + self.expectKeyword('for') + self.expect('(') + + if self.match(';'): + self.nextToken() + else: + if self.matchKeyword('var'): + init = self.createNode() + self.nextToken() + + previousAllowIn = self.context.allowIn + self.context.allowIn = False + declarations = self.parseVariableDeclarationList(Params(inFor=True)) + self.context.allowIn = previousAllowIn + + if len(declarations) == 1 and self.matchKeyword('in'): + decl = declarations[0] + if decl.init and (decl.id.type is Syntax.ArrayPattern or decl.id.type is Syntax.ObjectPattern or self.context.strict): + self.tolerateError(Messages.ForInOfLoopInitializer, 'for-in') + init = self.finalize(init, Node.VariableDeclaration(declarations, 'var')) + self.nextToken() + left = init + right = self.parseExpression() + init = None + elif len(declarations) == 1 and declarations[0].init is None and self.matchContextualKeyword('of'): + init = self.finalize(init, Node.VariableDeclaration(declarations, 'var')) + self.nextToken() + left = init + right = self.parseAssignmentExpression() + init = None + forIn = False + else: + init = self.finalize(init, Node.VariableDeclaration(declarations, 'var')) + self.expect(';') + elif self.matchKeyword('const', 'let'): + init = self.createNode() + kind = self.nextToken().value + + if not self.context.strict and self.lookahead.value == 'in': + init = self.finalize(init, Node.Identifier(kind)) + self.nextToken() + left = init + right = self.parseExpression() + init = None + else: + previousAllowIn = self.context.allowIn + self.context.allowIn = False + declarations = self.parseBindingList(kind, Params(inFor=True)) + self.context.allowIn = previousAllowIn + + if len(declarations) == 1 and declarations[0].init is None and self.matchKeyword('in'): + init = self.finalize(init, Node.VariableDeclaration(declarations, kind)) + self.nextToken() + left = init + right = self.parseExpression() + init = None + elif len(declarations) == 1 and declarations[0].init is None and self.matchContextualKeyword('of'): + init = self.finalize(init, Node.VariableDeclaration(declarations, kind)) + self.nextToken() + left = init + right = self.parseAssignmentExpression() + init = None + forIn = False + else: + self.consumeSemicolon() + init = self.finalize(init, Node.VariableDeclaration(declarations, kind)) + else: + initStartToken = self.lookahead + previousAllowIn = self.context.allowIn + self.context.allowIn = False + init = self.inheritCoverGrammar(self.parseAssignmentExpression) + self.context.allowIn = previousAllowIn + + if self.matchKeyword('in'): + if not self.context.isAssignmentTarget or init.type is Syntax.AssignmentExpression: + self.tolerateError(Messages.InvalidLHSInForIn) + + self.nextToken() + self.reinterpretExpressionAsPattern(init) + left = init + right = self.parseExpression() + init = None + elif self.matchContextualKeyword('of'): + if not self.context.isAssignmentTarget or init.type is Syntax.AssignmentExpression: + self.tolerateError(Messages.InvalidLHSInForLoop) + + self.nextToken() + self.reinterpretExpressionAsPattern(init) + left = init + right = self.parseAssignmentExpression() + init = None + forIn = False + else: + if self.match(','): + initSeq = [init] + while self.match(','): + self.nextToken() + initSeq.append(self.isolateCoverGrammar(self.parseAssignmentExpression)) + init = self.finalize(self.startNode(initStartToken), Node.SequenceExpression(initSeq)) + self.expect(';') + + if left is None: + if not self.match(';'): + test = self.parseExpression() + self.expect(';') + if not self.match(')'): + update = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + body = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + + previousInIteration = self.context.inIteration + self.context.inIteration = True + body = self.isolateCoverGrammar(self.parseStatement) + self.context.inIteration = previousInIteration + + if left is None: + return self.finalize(node, Node.ForStatement(init, test, update, body)) + + if forIn: + return self.finalize(node, Node.ForInStatement(left, right, body)) + + return self.finalize(node, Node.ForOfStatement(left, right, body)) + + # https://tc39.github.io/ecma262/#sec-continue-statement + + def parseContinueStatement(self): + node = self.createNode() + self.expectKeyword('continue') + + label = None + if self.lookahead.type is Token.Identifier and not self.hasLineTerminator: + id = self.parseVariableIdentifier() + label = id + + key = '$' + id.name + if key not in self.context.labelSet: + self.throwError(Messages.UnknownLabel, id.name) + + self.consumeSemicolon() + if label is None and not self.context.inIteration: + self.throwError(Messages.IllegalContinue) + + return self.finalize(node, Node.ContinueStatement(label)) + + # https://tc39.github.io/ecma262/#sec-break-statement + + def parseBreakStatement(self): + node = self.createNode() + self.expectKeyword('break') + + label = None + if self.lookahead.type is Token.Identifier and not self.hasLineTerminator: + id = self.parseVariableIdentifier() + + key = '$' + id.name + if key not in self.context.labelSet: + self.throwError(Messages.UnknownLabel, id.name) + label = id + + self.consumeSemicolon() + if label is None and not self.context.inIteration and not self.context.inSwitch: + self.throwError(Messages.IllegalBreak) + + return self.finalize(node, Node.BreakStatement(label)) + + # https://tc39.github.io/ecma262/#sec-return-statement + + def parseReturnStatement(self): + if not self.context.inFunctionBody: + self.tolerateError(Messages.IllegalReturn) + + node = self.createNode() + self.expectKeyword('return') + + hasArgument = ( + ( + not self.match(';') and not self.match('}') and + not self.hasLineTerminator and self.lookahead.type is not Token.EOF + ) or + self.lookahead.type is Token.StringLiteral or + self.lookahead.type is Token.Template + ) + argument = self.parseExpression() if hasArgument else None + self.consumeSemicolon() + + return self.finalize(node, Node.ReturnStatement(argument)) + + # https://tc39.github.io/ecma262/#sec-with-statement + + def parseWithStatement(self): + if self.context.strict: + self.tolerateError(Messages.StrictModeWith) + + node = self.createNode() + + self.expectKeyword('with') + self.expect('(') + object = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + body = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + body = self.parseStatement() + + return self.finalize(node, Node.WithStatement(object, body)) + + # https://tc39.github.io/ecma262/#sec-switch-statement + + def parseSwitchCase(self): + node = self.createNode() + + if self.matchKeyword('default'): + self.nextToken() + test = None + else: + self.expectKeyword('case') + test = self.parseExpression() + self.expect(':') + + consequent = [] + while True: + if self.match('}') or self.matchKeyword('default', 'case'): + break + consequent.append(self.parseStatementListItem()) + + return self.finalize(node, Node.SwitchCase(test, consequent)) + + def parseSwitchStatement(self): + node = self.createNode() + self.expectKeyword('switch') + + self.expect('(') + discriminant = self.parseExpression() + self.expect(')') + + previousInSwitch = self.context.inSwitch + self.context.inSwitch = True + + cases = [] + defaultFound = False + self.expect('{') + while True: + if self.match('}'): + break + clause = self.parseSwitchCase() + if clause.test is None: + if defaultFound: + self.throwError(Messages.MultipleDefaultsInSwitch) + defaultFound = True + cases.append(clause) + self.expect('}') + + self.context.inSwitch = previousInSwitch + + return self.finalize(node, Node.SwitchStatement(discriminant, cases)) + + # https://tc39.github.io/ecma262/#sec-labelled-statements + + def parseLabelledStatement(self): + node = self.createNode() + expr = self.parseExpression() + + if expr.type is Syntax.Identifier and self.match(':'): + self.nextToken() + + id = expr + key = '$' + id.name + if key in self.context.labelSet: + self.throwError(Messages.Redeclaration, 'Label', id.name) + + self.context.labelSet[key] = True + if self.matchKeyword('class'): + self.tolerateUnexpectedToken(self.lookahead) + body = self.parseClassDeclaration() + elif self.matchKeyword('function'): + token = self.lookahead + declaration = self.parseFunctionDeclaration() + if self.context.strict: + self.tolerateUnexpectedToken(token, Messages.StrictFunction) + elif declaration.generator: + self.tolerateUnexpectedToken(token, Messages.GeneratorInLegacyContext) + body = declaration + else: + body = self.parseStatement() + del self.context.labelSet[key] + + statement = Node.LabeledStatement(id, body) + else: + self.consumeSemicolon() + statement = Node.ExpressionStatement(expr) + + return self.finalize(node, statement) + + # https://tc39.github.io/ecma262/#sec-throw-statement + + def parseThrowStatement(self): + node = self.createNode() + self.expectKeyword('throw') + + if self.hasLineTerminator: + self.throwError(Messages.NewlineAfterThrow) + + argument = self.parseExpression() + self.consumeSemicolon() + + return self.finalize(node, Node.ThrowStatement(argument)) + + # https://tc39.github.io/ecma262/#sec-try-statement + + def parseCatchClause(self): + node = self.createNode() + + self.expectKeyword('catch') + + self.expect('(') + if self.match(')'): + self.throwUnexpectedToken(self.lookahead) + + params = [] + param = self.parsePattern(params) + paramMap = {} + for p in params: + key = '$' + p.value + if key in paramMap: + self.tolerateError(Messages.DuplicateBinding, p.value) + paramMap[key] = True + + if self.context.strict and param.type is Syntax.Identifier: + if self.scanner.isRestrictedWord(param.name): + self.tolerateError(Messages.StrictCatchVariable) + + self.expect(')') + body = self.parseBlock() + + return self.finalize(node, Node.CatchClause(param, body)) + + def parseFinallyClause(self): + self.expectKeyword('finally') + return self.parseBlock() + + def parseTryStatement(self): + node = self.createNode() + self.expectKeyword('try') + + block = self.parseBlock() + handler = self.parseCatchClause() if self.matchKeyword('catch') else None + finalizer = self.parseFinallyClause() if self.matchKeyword('finally') else None + + if not handler and not finalizer: + self.throwError(Messages.NoCatchOrFinally) + + return self.finalize(node, Node.TryStatement(block, handler, finalizer)) + + # https://tc39.github.io/ecma262/#sec-debugger-statement + + def parseDebuggerStatement(self): + node = self.createNode() + self.expectKeyword('debugger') + self.consumeSemicolon() + return self.finalize(node, Node.DebuggerStatement()) + + # https://tc39.github.io/ecma262/#sec-ecmascript-language-statements-and-declarations + + def parseStatement(self): + typ = self.lookahead.type + if typ in ( + Token.BooleanLiteral, + Token.NullLiteral, + Token.NumericLiteral, + Token.StringLiteral, + Token.Template, + Token.RegularExpression, + ): + statement = self.parseExpressionStatement() + + elif typ is Token.Punctuator: + value = self.lookahead.value + if value == '{': + statement = self.parseBlock() + elif value == '(': + statement = self.parseExpressionStatement() + elif value == ';': + statement = self.parseEmptyStatement() + else: + statement = self.parseExpressionStatement() + + elif typ is Token.Identifier: + statement = self.parseFunctionDeclaration() if self.matchAsyncFunction() else self.parseLabelledStatement() + + elif typ is Token.Keyword: + value = self.lookahead.value + if value == 'break': + statement = self.parseBreakStatement() + elif value == 'continue': + statement = self.parseContinueStatement() + elif value == 'debugger': + statement = self.parseDebuggerStatement() + elif value == 'do': + statement = self.parseDoWhileStatement() + elif value == 'for': + statement = self.parseForStatement() + elif value == 'function': + statement = self.parseFunctionDeclaration() + elif value == 'if': + statement = self.parseIfStatement() + elif value == 'return': + statement = self.parseReturnStatement() + elif value == 'switch': + statement = self.parseSwitchStatement() + elif value == 'throw': + statement = self.parseThrowStatement() + elif value == 'try': + statement = self.parseTryStatement() + elif value == 'var': + statement = self.parseVariableStatement() + elif value == 'while': + statement = self.parseWhileStatement() + elif value == 'with': + statement = self.parseWithStatement() + else: + statement = self.parseExpressionStatement() + + else: + statement = self.throwUnexpectedToken(self.lookahead) + + return statement + + # https://tc39.github.io/ecma262/#sec-function-definitions + + def parseFunctionSourceElements(self): + node = self.createNode() + + self.expect('{') + body = self.parseDirectivePrologues() + + previousLabelSet = self.context.labelSet + previousInIteration = self.context.inIteration + previousInSwitch = self.context.inSwitch + previousInFunctionBody = self.context.inFunctionBody + + self.context.labelSet = {} + self.context.inIteration = False + self.context.inSwitch = False + self.context.inFunctionBody = True + + while self.lookahead.type is not Token.EOF: + if self.match('}'): + break + body.append(self.parseStatementListItem()) + + self.expect('}') + + self.context.labelSet = previousLabelSet + self.context.inIteration = previousInIteration + self.context.inSwitch = previousInSwitch + self.context.inFunctionBody = previousInFunctionBody + + return self.finalize(node, Node.BlockStatement(body)) + + def validateParam(self, options, param, name): + key = '$' + name + if self.context.strict: + if self.scanner.isRestrictedWord(name): + options.stricted = param + options.message = Messages.StrictParamName + if key in options.paramSet: + options.stricted = param + options.message = Messages.StrictParamDupe + elif not options.firstRestricted: + if self.scanner.isRestrictedWord(name): + options.firstRestricted = param + options.message = Messages.StrictParamName + elif self.scanner.isStrictModeReservedWord(name): + options.firstRestricted = param + options.message = Messages.StrictReservedWord + elif key in options.paramSet: + options.stricted = param + options.message = Messages.StrictParamDupe + + options.paramSet[key] = True + + def parseRestElement(self, params): + node = self.createNode() + + self.expect('...') + arg = self.parsePattern(params) + if self.match('='): + self.throwError(Messages.DefaultRestParameter) + if not self.match(')'): + self.throwError(Messages.ParameterAfterRestParameter) + + return self.finalize(node, Node.RestElement(arg)) + + def parseFormalParameter(self, options): + params = [] + param = self.parseRestElement(params) if self.match('...') else self.parsePatternWithDefault(params) + for p in params: + self.validateParam(options, p, p.value) + options.simple = options.simple and isinstance(param, Node.Identifier) + options.params.append(param) + + def parseFormalParameters(self, firstRestricted=None): + options = Params( + simple=True, + params=[], + firstRestricted=firstRestricted + ) + + self.expect('(') + if not self.match(')'): + options.paramSet = {} + while self.lookahead.type is not Token.EOF: + self.parseFormalParameter(options) + if self.match(')'): + break + self.expect(',') + if self.match(')'): + break + self.expect(')') + + return Params( + simple=options.simple, + params=options.params, + stricted=options.stricted, + firstRestricted=options.firstRestricted, + message=options.message + ) + + def matchAsyncFunction(self): + match = self.matchContextualKeyword('async') + if match: + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.scanner.lex() + self.scanner.restoreState(state) + + match = (state.lineNumber == next.lineNumber) and (next.type is Token.Keyword) and (next.value == 'function') + + return match + + def parseFunctionDeclaration(self, identifierIsOptional=False): + node = self.createNode() + + isAsync = self.matchContextualKeyword('async') + if isAsync: + self.nextToken() + + self.expectKeyword('function') + + isGenerator = False if isAsync else self.match('*') + if isGenerator: + self.nextToken() + + id = None + firstRestricted = None + + if not identifierIsOptional or not self.match('('): + token = self.lookahead + id = self.parseVariableIdentifier() + if self.context.strict: + if self.scanner.isRestrictedWord(token.value): + self.tolerateUnexpectedToken(token, Messages.StrictFunctionName) + else: + if self.scanner.isRestrictedWord(token.value): + firstRestricted = token + message = Messages.StrictFunctionName + elif self.scanner.isStrictModeReservedWord(token.value): + firstRestricted = token + message = Messages.StrictReservedWord + + previousAllowAwait = self.context.allowAwait + previousAllowYield = self.context.allowYield + self.context.allowAwait = isAsync + self.context.allowYield = not isGenerator + + formalParameters = self.parseFormalParameters(firstRestricted) + params = formalParameters.params + stricted = formalParameters.stricted + firstRestricted = formalParameters.firstRestricted + if formalParameters.message: + message = formalParameters.message + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = formalParameters.simple + body = self.parseFunctionSourceElements() + if self.context.strict and firstRestricted: + self.throwUnexpectedToken(firstRestricted, message) + if self.context.strict and stricted: + self.tolerateUnexpectedToken(stricted, message) + + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + self.context.allowAwait = previousAllowAwait + self.context.allowYield = previousAllowYield + + if isAsync: + return self.finalize(node, Node.AsyncFunctionDeclaration(id, params, body)) + + return self.finalize(node, Node.FunctionDeclaration(id, params, body, isGenerator)) + + def parseFunctionExpression(self): + node = self.createNode() + + isAsync = self.matchContextualKeyword('async') + if isAsync: + self.nextToken() + + self.expectKeyword('function') + + isGenerator = False if isAsync else self.match('*') + if isGenerator: + self.nextToken() + + id = None + firstRestricted = None + + previousAllowAwait = self.context.allowAwait + previousAllowYield = self.context.allowYield + self.context.allowAwait = isAsync + self.context.allowYield = not isGenerator + + if not self.match('('): + token = self.lookahead + id = self.parseIdentifierName() if not self.context.strict and not isGenerator and self.matchKeyword('yield') else self.parseVariableIdentifier() + if self.context.strict: + if self.scanner.isRestrictedWord(token.value): + self.tolerateUnexpectedToken(token, Messages.StrictFunctionName) + else: + if self.scanner.isRestrictedWord(token.value): + firstRestricted = token + message = Messages.StrictFunctionName + elif self.scanner.isStrictModeReservedWord(token.value): + firstRestricted = token + message = Messages.StrictReservedWord + + formalParameters = self.parseFormalParameters(firstRestricted) + params = formalParameters.params + stricted = formalParameters.stricted + firstRestricted = formalParameters.firstRestricted + if formalParameters.message: + message = formalParameters.message + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = formalParameters.simple + body = self.parseFunctionSourceElements() + if self.context.strict and firstRestricted: + self.throwUnexpectedToken(firstRestricted, message) + if self.context.strict and stricted: + self.tolerateUnexpectedToken(stricted, message) + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + self.context.allowAwait = previousAllowAwait + self.context.allowYield = previousAllowYield + + if isAsync: + return self.finalize(node, Node.AsyncFunctionExpression(id, params, body)) + + return self.finalize(node, Node.FunctionExpression(id, params, body, isGenerator)) + + # https://tc39.github.io/ecma262/#sec-directive-prologues-and-the-use-strict-directive + + def parseDirective(self): + token = self.lookahead + + node = self.createNode() + expr = self.parseExpression() + directive = self.getTokenRaw(token)[1:-1] if expr.type is Syntax.Literal else None + self.consumeSemicolon() + + return self.finalize(node, Node.Directive(expr, directive) if directive else Node.ExpressionStatement(expr)) + + def parseDirectivePrologues(self): + firstRestricted = None + + body = [] + while True: + token = self.lookahead + if token.type is not Token.StringLiteral: + break + + statement = self.parseDirective() + body.append(statement) + directive = statement.directive + if not isinstance(directive, basestring): + break + + if directive == 'use strict': + self.context.strict = True + if firstRestricted: + self.tolerateUnexpectedToken(firstRestricted, Messages.StrictOctalLiteral) + if not self.context.allowStrictDirective: + self.tolerateUnexpectedToken(token, Messages.IllegalLanguageModeDirective) + else: + if not firstRestricted and token.octal: + firstRestricted = token + + return body + + # https://tc39.github.io/ecma262/#sec-method-definitions + + def qualifiedPropertyName(self, token): + typ = token.type + if typ in ( + Token.Identifier, + Token.StringLiteral, + Token.BooleanLiteral, + Token.NullLiteral, + Token.NumericLiteral, + Token.Keyword, + ): + return True + elif typ is Token.Punctuator: + return token.value == '[' + return False + + def parseGetterMethod(self): + node = self.createNode() + + isGenerator = False + previousAllowYield = self.context.allowYield + self.context.allowYield = not isGenerator + formalParameters = self.parseFormalParameters() + if len(formalParameters.params) > 0: + self.tolerateError(Messages.BadGetterArity) + method = self.parsePropertyMethod(formalParameters) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, formalParameters.params, method, isGenerator)) + + def parseSetterMethod(self): + node = self.createNode() + + isGenerator = False + previousAllowYield = self.context.allowYield + self.context.allowYield = not isGenerator + formalParameters = self.parseFormalParameters() + if len(formalParameters.params) != 1: + self.tolerateError(Messages.BadSetterArity) + elif isinstance(formalParameters.params[0], Node.RestElement): + self.tolerateError(Messages.BadSetterRestParameter) + method = self.parsePropertyMethod(formalParameters) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, formalParameters.params, method, isGenerator)) + + def parseGeneratorMethod(self): + node = self.createNode() + + isGenerator = True + previousAllowYield = self.context.allowYield + + self.context.allowYield = True + params = self.parseFormalParameters() + self.context.allowYield = False + method = self.parsePropertyMethod(params) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, params.params, method, isGenerator)) + + # https://tc39.github.io/ecma262/#sec-generator-function-definitions + + def isStartOfExpression(self): + start = True + + value = self.lookahead.value + typ = self.lookahead.type + if typ is Token.Punctuator: + start = value in ('[', '(', '{', '+', '-', '!', '~', '++', '--', '/', '/=') # regular expression literal ) + + elif typ is Token.Keyword: + start = value in ('class', 'delete', 'function', 'let', 'new', 'super', 'this', 'typeof', 'void', 'yield') + + return start + + def parseYieldExpression(self): + node = self.createNode() + self.expectKeyword('yield') + + argument = None + delegate = False + if not self.hasLineTerminator: + previousAllowYield = self.context.allowYield + self.context.allowYield = False + delegate = self.match('*') + if delegate: + self.nextToken() + argument = self.parseAssignmentExpression() + elif self.isStartOfExpression(): + argument = self.parseAssignmentExpression() + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.YieldExpression(argument, delegate)) + + # https://tc39.github.io/ecma262/#sec-class-definitions + + def parseClassElement(self, hasConstructor): + token = self.lookahead + node = self.createNode() + + kind = '' + key = None + value = None + computed = False + isStatic = False + isAsync = False + + if self.match('*'): + self.nextToken() + + else: + computed = self.match('[') + key = self.parseObjectPropertyKey() + id = key + if id.name == 'static' and (self.qualifiedPropertyName(self.lookahead) or self.match('*')): + token = self.lookahead + isStatic = True + computed = self.match('[') + if self.match('*'): + self.nextToken() + else: + key = self.parseObjectPropertyKey() + if token.type is Token.Identifier and not self.hasLineTerminator and token.value == 'async': + punctuator = self.lookahead.value + if punctuator != ':' and punctuator != '(' and punctuator != '*': + isAsync = True + token = self.lookahead + key = self.parseObjectPropertyKey() + if token.type is Token.Identifier and token.value == 'constructor': + self.tolerateUnexpectedToken(token, Messages.ConstructorIsAsync) + + lookaheadPropertyKey = self.qualifiedPropertyName(self.lookahead) + if token.type is Token.Identifier: + if token.value == 'get' and lookaheadPropertyKey: + kind = 'get' + computed = self.match('[') + key = self.parseObjectPropertyKey() + self.context.allowYield = False + value = self.parseGetterMethod() + elif token.value == 'set' and lookaheadPropertyKey: + kind = 'set' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseSetterMethod() + elif self.config.classProperties and not self.match('('): + kind = 'init' + id = self.finalize(node, Node.Identifier(token.value)) + if self.match('='): + self.nextToken() + value = self.parseAssignmentExpression() + + elif token.type is Token.Punctuator and token.value == '*' and lookaheadPropertyKey: + kind = 'method' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseGeneratorMethod() + + if not kind and key and self.match('('): + kind = 'method' + value = self.parsePropertyMethodAsyncFunction() if isAsync else self.parsePropertyMethodFunction() + + if not kind: + self.throwUnexpectedToken(self.lookahead) + + if not computed: + if isStatic and self.isPropertyKey(key, 'prototype'): + self.throwUnexpectedToken(token, Messages.StaticPrototype) + if not isStatic and self.isPropertyKey(key, 'constructor'): + if kind != 'method' or (value and value.generator): + self.throwUnexpectedToken(token, Messages.ConstructorSpecialMethod) + if hasConstructor.value: + self.throwUnexpectedToken(token, Messages.DuplicateConstructor) + else: + hasConstructor.value = True + kind = 'constructor' + + if kind in ('constructor', 'method', 'get', 'set'): + return self.finalize(node, Node.MethodDefinition(key, computed, value, kind, isStatic)) + + else: + return self.finalize(node, Node.FieldDefinition(key, computed, value, kind, isStatic)) + + def parseClassElementList(self): + body = [] + hasConstructor = Value(False) + + self.expect('{') + while not self.match('}'): + if self.match(';'): + self.nextToken() + else: + body.append(self.parseClassElement(hasConstructor)) + self.expect('}') + + return body + + def parseClassBody(self): + node = self.createNode() + elementList = self.parseClassElementList() + + return self.finalize(node, Node.ClassBody(elementList)) + + def parseClassDeclaration(self, identifierIsOptional=False): + node = self.createNode() + + previousStrict = self.context.strict + self.context.strict = True + self.expectKeyword('class') + + id = None if identifierIsOptional and self.lookahead.type is not Token.Identifier else self.parseVariableIdentifier() + superClass = None + if self.matchKeyword('extends'): + self.nextToken() + superClass = self.isolateCoverGrammar(self.parseLeftHandSideExpressionAllowCall) + classBody = self.parseClassBody() + self.context.strict = previousStrict + + return self.finalize(node, Node.ClassDeclaration(id, superClass, classBody)) + + def parseClassExpression(self): + node = self.createNode() + + previousStrict = self.context.strict + self.context.strict = True + self.expectKeyword('class') + id = self.parseVariableIdentifier() if self.lookahead.type is Token.Identifier else None + superClass = None + if self.matchKeyword('extends'): + self.nextToken() + superClass = self.isolateCoverGrammar(self.parseLeftHandSideExpressionAllowCall) + classBody = self.parseClassBody() + self.context.strict = previousStrict + + return self.finalize(node, Node.ClassExpression(id, superClass, classBody)) + + # https://tc39.github.io/ecma262/#sec-scripts + # https://tc39.github.io/ecma262/#sec-modules + + def parseModule(self): + self.context.strict = True + self.context.isModule = True + self.scanner.isModule = True + node = self.createNode() + body = self.parseDirectivePrologues() + while self.lookahead.type is not Token.EOF: + body.append(self.parseStatementListItem()) + return self.finalize(node, Node.Module(body)) + + def parseScript(self): + node = self.createNode() + body = self.parseDirectivePrologues() + while self.lookahead.type is not Token.EOF: + body.append(self.parseStatementListItem()) + return self.finalize(node, Node.Script(body)) + + # https://tc39.github.io/ecma262/#sec-imports + + def parseModuleSpecifier(self): + node = self.createNode() + + if self.lookahead.type is not Token.StringLiteral: + self.throwError(Messages.InvalidModuleSpecifier) + + token = self.nextToken() + raw = self.getTokenRaw(token) + return self.finalize(node, Node.Literal(token.value, raw)) + + # import {} ... + def parseImportSpecifier(self): + node = self.createNode() + + if self.lookahead.type is Token.Identifier: + imported = self.parseVariableIdentifier() + local = imported + if self.matchContextualKeyword('as'): + self.nextToken() + local = self.parseVariableIdentifier() + else: + imported = self.parseIdentifierName() + local = imported + if self.matchContextualKeyword('as'): + self.nextToken() + local = self.parseVariableIdentifier() + else: + self.throwUnexpectedToken(self.nextToken()) + + return self.finalize(node, Node.ImportSpecifier(local, imported)) + + # {foo, bar as bas + def parseNamedImports(self): + self.expect('{') + specifiers = [] + while not self.match('}'): + specifiers.append(self.parseImportSpecifier()) + if not self.match('}'): + self.expect(',') + self.expect('}') + + return specifiers + + # import ... + def parseImportDefaultSpecifier(self): + node = self.createNode() + local = self.parseIdentifierName() + return self.finalize(node, Node.ImportDefaultSpecifier(local)) + + # import <* as foo> ... + def parseImportNamespaceSpecifier(self): + node = self.createNode() + + self.expect('*') + if not self.matchContextualKeyword('as'): + self.throwError(Messages.NoAsAfterImportNamespace) + self.nextToken() + local = self.parseIdentifierName() + + return self.finalize(node, Node.ImportNamespaceSpecifier(local)) + + def parseImportDeclaration(self): + if self.context.inFunctionBody: + self.throwError(Messages.IllegalImportDeclaration) + + node = self.createNode() + self.expectKeyword('import') + + specifiers = [] + if self.lookahead.type is Token.StringLiteral: + # import 'foo' + src = self.parseModuleSpecifier() + else: + if self.match('{'): + # import {bar + specifiers.extend(self.parseNamedImports()) + elif self.match('*'): + # import * as foo + specifiers.append(self.parseImportNamespaceSpecifier()) + elif self.isIdentifierName(self.lookahead) and not self.matchKeyword('default'): + # import foo + specifiers.append(self.parseImportDefaultSpecifier()) + if self.match(','): + self.nextToken() + if self.match('*'): + # import foo, * as foo + specifiers.append(self.parseImportNamespaceSpecifier()) + elif self.match('{'): + # import foo, {bar + specifiers.extend(self.parseNamedImports()) + else: + self.throwUnexpectedToken(self.lookahead) + else: + self.throwUnexpectedToken(self.nextToken()) + + if not self.matchContextualKeyword('from'): + message = Messages.UnexpectedToken if self.lookahead.value else Messages.MissingFromClause + self.throwError(message, self.lookahead.value) + self.nextToken() + src = self.parseModuleSpecifier() + self.consumeSemicolon() + + return self.finalize(node, Node.ImportDeclaration(specifiers, src)) + + # https://tc39.github.io/ecma262/#sec-exports + + def parseExportSpecifier(self): + node = self.createNode() + + local = self.parseIdentifierName() + exported = local + if self.matchContextualKeyword('as'): + self.nextToken() + exported = self.parseIdentifierName() + + return self.finalize(node, Node.ExportSpecifier(local, exported)) + + def parseExportDefaultSpecifier(self): + node = self.createNode() + local = self.parseIdentifierName() + return self.finalize(node, Node.ExportDefaultSpecifier(local)) + + def parseExportDeclaration(self): + if self.context.inFunctionBody: + self.throwError(Messages.IllegalExportDeclaration) + + node = self.createNode() + self.expectKeyword('export') + + if self.matchKeyword('default'): + # export default ... + self.nextToken() + if self.matchKeyword('function'): + # export default function foo (: + # export default function (: + declaration = self.parseFunctionDeclaration(True) + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + elif self.matchKeyword('class'): + # export default class foo { + declaration = self.parseClassDeclaration(True) + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + elif self.matchContextualKeyword('async'): + # export default async function f (: + # export default async function (: + # export default async x => x + declaration = self.parseFunctionDeclaration(True) if self.matchAsyncFunction() else self.parseAssignmentExpression() + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + else: + if self.matchContextualKeyword('from'): + self.throwError(Messages.UnexpectedToken, self.lookahead.value) + # export default {} + # export default [] + # export default (1 + 2) + if self.match('{'): + declaration = self.parseObjectInitializer() + elif self.match('['): + declaration = self.parseArrayInitializer() + else: + declaration = self.parseAssignmentExpression() + self.consumeSemicolon() + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + + elif self.match('*'): + # export * from 'foo' + self.nextToken() + if not self.matchContextualKeyword('from'): + message = Messages.UnexpectedToken if self.lookahead.value else Messages.MissingFromClause + self.throwError(message, self.lookahead.value) + self.nextToken() + src = self.parseModuleSpecifier() + self.consumeSemicolon() + exportDeclaration = self.finalize(node, Node.ExportAllDeclaration(src)) + + elif self.lookahead.type is Token.Keyword: + # export var f = 1 + value = self.lookahead.value + if value in ( + 'let', + 'const', + ): + declaration = self.parseLexicalDeclaration(Params(inFor=False)) + elif value in ( + 'var', + 'class', + 'function', + ): + declaration = self.parseStatementListItem() + else: + self.throwUnexpectedToken(self.lookahead) + exportDeclaration = self.finalize(node, Node.ExportNamedDeclaration(declaration, [], None)) + + elif self.matchAsyncFunction(): + declaration = self.parseFunctionDeclaration() + exportDeclaration = self.finalize(node, Node.ExportNamedDeclaration(declaration, [], None)) + + else: + specifiers = [] + source = None + isExportFromIdentifier = False + + expectSpecifiers = True + if self.lookahead.type is Token.Identifier: + specifiers.append(self.parseExportDefaultSpecifier()) + if self.match(','): + self.nextToken() + else: + expectSpecifiers = False + + if expectSpecifiers: + self.expect('{') + while not self.match('}'): + isExportFromIdentifier = isExportFromIdentifier or self.matchKeyword('default') + specifiers.append(self.parseExportSpecifier()) + if not self.match('}'): + self.expect(',') + self.expect('}') + + if self.matchContextualKeyword('from'): + # export {default} from 'foo' + # export {foo} from 'foo' + self.nextToken() + source = self.parseModuleSpecifier() + self.consumeSemicolon() + elif isExportFromIdentifier: + # export {default}; # missing fromClause + message = Messages.UnexpectedToken if self.lookahead.value else Messages.MissingFromClause + self.throwError(message, self.lookahead.value) + else: + # export {foo} + self.consumeSemicolon() + exportDeclaration = self.finalize(node, Node.ExportNamedDeclaration(None, specifiers, source)) + + return exportDeclaration diff --git a/third_party/python/esprima/esprima/scanner.py b/third_party/python/esprima/esprima/scanner.py new file mode 100644 index 000000000000..53502a51d37e --- /dev/null +++ b/third_party/python/esprima/esprima/scanner.py @@ -0,0 +1,1189 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +import re + +from .objects import Object +from .compat import xrange, unicode, uchr, uord +from .character import Character, HEX_CONV, OCTAL_CONV +from .messages import Messages +from .token import Token + + +def hexValue(ch): + return HEX_CONV[ch] + + +def octalValue(ch): + return OCTAL_CONV[ch] + + +class RegExp(Object): + def __init__(self, pattern=None, flags=None): + self.pattern = pattern + self.flags = flags + + +class Position(Object): + def __init__(self, line=None, column=None, offset=None): + self.line = line + self.column = column + self.offset = offset + + +class SourceLocation(Object): + def __init__(self, start=None, end=None, source=None): + self.start = start + self.end = end + self.source = source + + +class Comment(Object): + def __init__(self, multiLine=None, slice=None, range=None, loc=None): + self.multiLine = multiLine + self.slice = slice + self.range = range + self.loc = loc + + +class RawToken(Object): + def __init__(self, type=None, value=None, pattern=None, flags=None, regex=None, octal=None, cooked=None, head=None, tail=None, lineNumber=None, lineStart=None, start=None, end=None): + self.type = type + self.value = value + self.pattern = pattern + self.flags = flags + self.regex = regex + self.octal = octal + self.cooked = cooked + self.head = head + self.tail = tail + self.lineNumber = lineNumber + self.lineStart = lineStart + self.start = start + self.end = end + + +class ScannerState(Object): + def __init__(self, index=None, lineNumber=None, lineStart=None): + self.index = index + self.lineNumber = lineNumber + self.lineStart = lineStart + + +class Octal(object): + def __init__(self, octal, code): + self.octal = octal + self.code = code + + +class Scanner(object): + def __init__(self, code, handler): + self.source = unicode(code) + '\x00' + self.errorHandler = handler + self.trackComment = False + self.isModule = False + + self.length = len(code) + self.index = 0 + self.lineNumber = 1 if self.length > 0 else 0 + self.lineStart = 0 + self.curlyStack = [] + + def saveState(self): + return ScannerState( + index=self.index, + lineNumber=self.lineNumber, + lineStart=self.lineStart + ) + + def restoreState(self, state): + self.index = state.index + self.lineNumber = state.lineNumber + self.lineStart = state.lineStart + + def eof(self): + return self.index >= self.length + + def throwUnexpectedToken(self, message=Messages.UnexpectedTokenIllegal): + return self.errorHandler.throwError(self.index, self.lineNumber, + self.index - self.lineStart + 1, message) + + def tolerateUnexpectedToken(self, message=Messages.UnexpectedTokenIllegal): + self.errorHandler.tolerateError(self.index, self.lineNumber, + self.index - self.lineStart + 1, message) + + # https://tc39.github.io/ecma262/#sec-comments + + def skipSingleLineComment(self, offset): + comments = [] + + if self.trackComment: + start = self.index - offset + loc = SourceLocation( + start=Position( + line=self.lineNumber, + column=self.index - self.lineStart - offset + ), + end=Position() + ) + + while not self.eof(): + ch = self.source[self.index] + self.index += 1 + if Character.isLineTerminator(ch): + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart - 1 + ) + entry = Comment( + multiLine=False, + slice=[start + offset, self.index - 1], + range=[start, self.index - 1], + loc=loc + ) + comments.append(entry) + + if ch == '\r' and self.source[self.index] == '\n': + self.index += 1 + + self.lineNumber += 1 + self.lineStart = self.index + return comments + + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart + ) + entry = Comment( + multiLine=False, + slice=[start + offset, self.index], + range=[start, self.index], + loc=loc + ) + comments.append(entry) + + return comments + + def skipMultiLineComment(self): + comments = [] + + if self.trackComment: + comments = [] + start = self.index - 2 + loc = SourceLocation( + start=Position( + line=self.lineNumber, + column=self.index - self.lineStart - 2 + ), + end=Position() + ) + + while not self.eof(): + ch = self.source[self.index] + if Character.isLineTerminator(ch): + if ch == '\r' and self.source[self.index + 1] == '\n': + self.index += 1 + + self.lineNumber += 1 + self.index += 1 + self.lineStart = self.index + elif ch == '*': + # Block comment ends with '*/'. + if self.source[self.index + 1] == '/': + self.index += 2 + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart + ) + entry = Comment( + multiLine=True, + slice=[start + 2, self.index - 2], + range=[start, self.index], + loc=loc + ) + comments.append(entry) + + return comments + + self.index += 1 + else: + self.index += 1 + + # Ran off the end of the file - the whole thing is a comment + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart + ) + entry = Comment( + multiLine=True, + slice=[start + 2, self.index], + range=[start, self.index], + loc=loc + ) + comments.append(entry) + + self.tolerateUnexpectedToken() + return comments + + def scanComments(self): + comments = [] + + start = self.index == 0 + while not self.eof(): + ch = self.source[self.index] + + if Character.isWhiteSpace(ch): + self.index += 1 + elif Character.isLineTerminator(ch): + self.index += 1 + if ch == '\r' and self.source[self.index] == '\n': + self.index += 1 + + self.lineNumber += 1 + self.lineStart = self.index + start = True + elif ch == '/': # U+002F is '/' + ch = self.source[self.index + 1] + if ch == '/': + self.index += 2 + comment = self.skipSingleLineComment(2) + if self.trackComment: + comments.extend(comment) + + start = True + elif ch == '*': # U+002A is '*' + self.index += 2 + comment = self.skipMultiLineComment() + if self.trackComment: + comments.extend(comment) + + else: + break + + elif start and ch == '-': # U+002D is '-' + # U+003E is '>' + if self.source[self.index + 1:self.index + 3] == '->': + # '-->' is a single-line comment + self.index += 3 + comment = self.skipSingleLineComment(3) + if self.trackComment: + comments.extend(comment) + + else: + break + + elif ch == '<' and not self.isModule: # U+003C is '<' + if self.source[self.index + 1:self.index + 4] == '!--': + self.index += 4 # `