From 33f1eee4d572bb1dc9b2a5520e20eb9e58c785c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tarek=20Ziad=C3=A9?= Date: Tue, 28 Apr 2020 17:07:14 +0000 Subject: [PATCH] Bug 1633437 - Support for test metadata r=acreskey This patch adds support for tests metadata. A test script parser is added as well as a new "doc" flavor that can be used to display the script info in the command line. This parser will be the basis for building automated docs and scripts verifications if we want to do this. Differential Revision: https://phabricator.services.mozilla.com/D72800 --- build/virtualenv_packages.txt | 1 + python/mozperftest/mozperftest/argparser.py | 8 + .../mozperftest/mozperftest/mach_commands.py | 38 + python/mozperftest/mozperftest/scriptinfo.py | 83 + .../mozperftest/mozperftest/tests/example.js | 19 - .../tests/samples/perftest_example.js | 45 + .../mozperftest/mozperftest/tests/support.py | 5 + .../mozperftest/tests/test_browsertime.py | 6 +- .../mozperftest/tests/test_consoleoutput.py | 4 +- .../mozperftest/tests/test_mach_commands.py | 24 +- .../mozperftest/tests/test_perfherder.py | 4 +- .../mozperftest/tests/test_scriptinfo.py | 20 + python/mozperftest/mozperftest/utils.py | 19 + testing/performance/hooks_applink.py | 21 +- testing/performance/perftest_applink.js | 29 +- third_party/python/esprima/PKG-INFO | 143 + third_party/python/esprima/README | 117 + .../python/esprima/esprima/__init__.py | 29 + .../python/esprima/esprima/__main__.py | 105 + .../python/esprima/esprima/character.py | 125 + .../python/esprima/esprima/comment_handler.py | 176 + third_party/python/esprima/esprima/compat.py | 72 + .../python/esprima/esprima/error_handler.py | 74 + third_party/python/esprima/esprima/esprima.py | 125 + .../python/esprima/esprima/jsx_nodes.py | 100 + .../python/esprima/esprima/jsx_parser.py | 584 ++++ .../python/esprima/esprima/jsx_syntax.py | 38 + .../python/esprima/esprima/messages.py | 90 + third_party/python/esprima/esprima/nodes.py | 620 ++++ third_party/python/esprima/esprima/objects.py | 46 + third_party/python/esprima/esprima/parser.py | 3104 +++++++++++++++++ third_party/python/esprima/esprima/scanner.py | 1189 +++++++ third_party/python/esprima/esprima/syntax.py | 100 + third_party/python/esprima/esprima/token.py | 50 + .../python/esprima/esprima/tokenizer.py | 193 + third_party/python/esprima/esprima/utils.py | 40 + third_party/python/esprima/esprima/visitor.py | 288 ++ .../python/esprima/esprima/xhtml_entities.py | 281 ++ third_party/python/esprima/setup.cfg | 4 + third_party/python/esprima/setup.py | 55 + third_party/python/requirements.in | 1 + third_party/python/requirements.txt | 3 + 42 files changed, 8037 insertions(+), 41 deletions(-) create mode 100644 python/mozperftest/mozperftest/scriptinfo.py delete mode 100644 python/mozperftest/mozperftest/tests/example.js create mode 100644 python/mozperftest/mozperftest/tests/samples/perftest_example.js create mode 100644 python/mozperftest/mozperftest/tests/test_scriptinfo.py create mode 100644 third_party/python/esprima/PKG-INFO create mode 100644 third_party/python/esprima/README create mode 100644 third_party/python/esprima/esprima/__init__.py create mode 100644 third_party/python/esprima/esprima/__main__.py create mode 100644 third_party/python/esprima/esprima/character.py create mode 100644 third_party/python/esprima/esprima/comment_handler.py create mode 100644 third_party/python/esprima/esprima/compat.py create mode 100644 third_party/python/esprima/esprima/error_handler.py create mode 100644 third_party/python/esprima/esprima/esprima.py create mode 100644 third_party/python/esprima/esprima/jsx_nodes.py create mode 100644 third_party/python/esprima/esprima/jsx_parser.py create mode 100644 third_party/python/esprima/esprima/jsx_syntax.py create mode 100644 third_party/python/esprima/esprima/messages.py create mode 100644 third_party/python/esprima/esprima/nodes.py create mode 100644 third_party/python/esprima/esprima/objects.py create mode 100644 third_party/python/esprima/esprima/parser.py create mode 100644 third_party/python/esprima/esprima/scanner.py create mode 100644 third_party/python/esprima/esprima/syntax.py create mode 100644 third_party/python/esprima/esprima/token.py create mode 100644 third_party/python/esprima/esprima/tokenizer.py create mode 100644 third_party/python/esprima/esprima/utils.py create mode 100644 third_party/python/esprima/esprima/visitor.py create mode 100644 third_party/python/esprima/esprima/xhtml_entities.py create mode 100644 third_party/python/esprima/setup.cfg create mode 100644 third_party/python/esprima/setup.py diff --git a/build/virtualenv_packages.txt b/build/virtualenv_packages.txt index dabca237ab45..7fab1dd9080c 100644 --- a/build/virtualenv_packages.txt +++ b/build/virtualenv_packages.txt @@ -23,6 +23,7 @@ mozilla.pth:third_party/python/distro mozilla.pth:third_party/python/dlmanager mozilla.pth:third_party/python/ecdsa/src mozilla.pth:third_party/python/enum34 +mozilla.pth:third_party/python/esprima mozilla.pth:third_party/python/fluent.migrate mozilla.pth:third_party/python/fluent.syntax mozilla.pth:third_party/python/funcsigs diff --git a/python/mozperftest/mozperftest/argparser.py b/python/mozperftest/mozperftest/argparser.py index bdc8cfac89bf..188a169b75a4 100644 --- a/python/mozperftest/mozperftest/argparser.py +++ b/python/mozperftest/mozperftest/argparser.py @@ -19,10 +19,18 @@ from mozperftest.system import get_layers as system_layers from mozperftest.browser import get_layers as browser_layers from mozperftest.metrics import get_layers as metrics_layers +FLAVORS = ["script", "doc"] + class Options: general_args = { + "--flavor": { + "choices": FLAVORS, + "metavar": "{{{}}}".format(", ".join(FLAVORS)), + "default": None, + "help": "Only run tests of this flavor.", + }, "tests": { "nargs": "*", "metavar": "TEST", diff --git a/python/mozperftest/mozperftest/mach_commands.py b/python/mozperftest/mozperftest/mach_commands.py index 6829181ef84b..7cda839761f3 100644 --- a/python/mozperftest/mozperftest/mach_commands.py +++ b/python/mozperftest/mozperftest/mach_commands.py @@ -1,6 +1,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import random from functools import partial from mach.decorators import CommandProvider, Command from mozbuild.base import MachCommandBase, MachCommandConditions as conditions @@ -14,6 +16,25 @@ def get_perftest_parser(): @CommandProvider class Perftest(MachCommandBase): + def _build_test_list(self, tests, randomized=False): + res = [] + for test in tests: + if os.path.isfile(test): + tests.append(test) + elif os.path.isdir(test): + for root, dirs, files in os.walk(test): + for file in files: + if not file.startswith("perftest"): + continue + res.append(os.path.join(root, file)) + if not randomized: + res.sort() + else: + # random shuffling is used to make sure + # we don't always run tests in the same order + random.shuffle(res) + return res + @Command( "perftest", category="testing", @@ -24,6 +45,23 @@ class Perftest(MachCommandBase): def run_perftest( self, flavor="script", test_objects=None, resolve_tests=True, **kwargs ): + + MachCommandBase._activate_virtualenv(self) + kwargs["tests"] = self._build_test_list( + kwargs["tests"], randomized=flavor != "doc" + ) + + if flavor == "doc": + from mozperftest.utils import install_package + + install_package(self.virtualenv_manager, "esprima") + + from mozperftest.scriptinfo import ScriptInfo + + for test in kwargs["tests"]: + print(ScriptInfo(test)) + return + from mozperftest import MachEnvironment, Metadata kwargs["test_objects"] = test_objects diff --git a/python/mozperftest/mozperftest/scriptinfo.py b/python/mozperftest/mozperftest/scriptinfo.py new file mode 100644 index 000000000000..1f3837c34f14 --- /dev/null +++ b/python/mozperftest/mozperftest/scriptinfo.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from collections import defaultdict +import re +import os +import textwrap + +# This import will fail if esprima is not installed. +# The package is vendored in python/third_party and also available at PyPI +# The mach perftest command will make sure it's installed when --flavor doc +# is being used. +import esprima + + +_INFO = """\ +%(filename)s +%(filename_underline)s + +%(description)s + +Owner: %(owner)s +Usage: +%(usage)s + +Description: +%(long_description)s +""" + + +class MetadataDict(defaultdict): + def __missing__(self, key): + return "N/A" + + +class ScriptInfo(MetadataDict): + def __init__(self, script): + super(ScriptInfo, self).__init__() + filename = os.path.basename(script) + self["filename"] = script, filename + self["filename_underline"] = None, "-" * len(filename) + self.script = script + with open(script) as f: + self.parsed = esprima.parseScript(f.read()) + + # looking for the exports statement + for stmt in self.parsed.body: + if ( + stmt.type != "ExpressionStatement" + or stmt.expression.left is None + or stmt.expression.left.property.name != "exports" + or stmt.expression.right is None + or stmt.expression.right.properties is None + ): + continue + + # now scanning the properties + for prop in stmt.expression.right.properties: + if prop.value.type == "Identifier": + value = prop.value.name + elif prop.value.type == "Literal": + value = prop.value.value + elif prop.value.type == "TemplateLiteral": + # ugly + value = prop.value.quasis[0].value.cooked.replace("\n", " ") + value = re.sub("\s+", " ", value).strip() + elif prop.value.type == "ArrayExpression": + value = [e.value for e in prop.value.elements] + else: + raise ValueError(prop.value.type) + # line wrapping + if isinstance(value, str): + repr = "\n".join(textwrap.wrap(value, break_on_hyphens=False)) + elif isinstance(value, list): + repr = ", ".join(value) + + self[prop.key.name] = value, repr + + def __str__(self): + reprs = dict((k, v[1]) for k, v in self.items()) + d = MetadataDict() + d.update(reprs) + return _INFO % d diff --git a/python/mozperftest/mozperftest/tests/example.js b/python/mozperftest/mozperftest/tests/example.js deleted file mode 100644 index b7e98218954d..000000000000 --- a/python/mozperftest/mozperftest/tests/example.js +++ /dev/null @@ -1,19 +0,0 @@ -async function setUp(context) { - context.log.info("setUp example!"); -} - -async function test(context, commands) { - context.log.info("Test with setUp/tearDown example!"); - await commands.measure.start("https://www.sitespeed.io/"); - await commands.measure.start("https://www.mozilla.org/en-US/"); -} - -async function tearDown(context) { - context.log.info("tearDown example!"); -} - -module.exports = { // eslint-disable-line - setUp, - tearDown, - test, -}; diff --git a/python/mozperftest/mozperftest/tests/samples/perftest_example.js b/python/mozperftest/mozperftest/tests/samples/perftest_example.js new file mode 100644 index 000000000000..34d876ac82fe --- /dev/null +++ b/python/mozperftest/mozperftest/tests/samples/perftest_example.js @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +/* eslint-env node */ +"use strict"; + +var someVar; + +async function setUp(context) { + context.log.info("setUp example!"); +} + +async function test(context, commands) { + context.log.info("Test with setUp/tearDown example!"); + await commands.measure.start("https://www.sitespeed.io/"); + await commands.measure.start("https://www.mozilla.org/en-US/"); +} + +async function tearDown(context) { + context.log.info("tearDown example!"); +} + + +module.exports = { + setUp, + tearDown, + test, + owner: "Performance Team", + description: "Measures cold process applink time", + long_description: ` + This test launches the appropriate android app, simulating an app link + workflow. The application is launched with the intent action + android.intent.action.VIEW loading a trivially simple website. The reported + metric is the time from process start to navigationStart, reported as processLaunchToNavStart + `, + usage: ` + ./mach perftest testing/performance/perftest_applink.js \ + --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk \ + --hooks testing/performance/hooks_applink.py \ + --android-app-name org.mozilla.fennec_aurora \ + --perfherder-metrics processLaunchToNavStart + `, + supported_browser: ["Fenix nightly", "Geckoview_example", "Fennec"], + platform: ["Android"], +}; diff --git a/python/mozperftest/mozperftest/tests/support.py b/python/mozperftest/mozperftest/tests/support.py index 326de703ef6a..01b2f7ae1779 100644 --- a/python/mozperftest/mozperftest/tests/support.py +++ b/python/mozperftest/mozperftest/tests/support.py @@ -8,6 +8,11 @@ from mozperftest.metadata import Metadata from mozperftest.environment import MachEnvironment +HERE = os.path.dirname(__file__) +EXAMPLE_TESTS_DIR = os.path.join(HERE, "samples") +EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js") + + @contextlib.contextmanager def temp_file(name="temp", content=None): tempdir = tempfile.mkdtemp() diff --git a/python/mozperftest/mozperftest/tests/test_browsertime.py b/python/mozperftest/mozperftest/tests/test_browsertime.py index 282cb6889c60..7d0752209cc9 100644 --- a/python/mozperftest/mozperftest/tests/test_browsertime.py +++ b/python/mozperftest/mozperftest/tests/test_browsertime.py @@ -4,7 +4,7 @@ import mozunit import mock import shutil -from mozperftest.tests.support import get_running_env +from mozperftest.tests.support import get_running_env, EXAMPLE_TEST from mozperftest.environment import BROWSER from mozperftest.browser.browsertime import add_options from mozperftest.utils import silence @@ -28,7 +28,7 @@ def fetch(self, url): def test_browser(): mach_cmd, metadata, env = get_running_env() browser = env.layers[BROWSER] - env.set_arg("tests", [os.path.join(HERE, "example.js")]) + env.set_arg("tests", [EXAMPLE_TEST]) try: with browser as b, silence(): @@ -38,7 +38,7 @@ def test_browser(): assert mach_cmd.run_process.call_count == 1 # XXX more checks - assert mach_cmd.run_process.call_args[0][-1][-1] == os.path.join(HERE, "example.js") + assert mach_cmd.run_process.call_args[0][-1][-1] == EXAMPLE_TEST def test_add_options(): diff --git a/python/mozperftest/mozperftest/tests/test_consoleoutput.py b/python/mozperftest/mozperftest/tests/test_consoleoutput.py index 4b255383d62d..195c959d19c6 100644 --- a/python/mozperftest/mozperftest/tests/test_consoleoutput.py +++ b/python/mozperftest/mozperftest/tests/test_consoleoutput.py @@ -2,7 +2,7 @@ import os import mozunit -from mozperftest.tests.support import get_running_env, temp_dir +from mozperftest.tests.support import EXAMPLE_TEST, get_running_env, temp_dir from mozperftest.environment import METRICS from mozperftest.utils import silence @@ -26,7 +26,7 @@ def test_console_output(): mach_cmd.run_process = _run_process metrics = env.layers[METRICS] - env.set_arg("tests", [os.path.join(HERE, "example.js")]) + env.set_arg("tests", [EXAMPLE_TEST]) metadata.set_result(os.path.join(HERE, "browsertime-results")) with metrics as console, silence(): diff --git a/python/mozperftest/mozperftest/tests/test_mach_commands.py b/python/mozperftest/mozperftest/tests/test_mach_commands.py index a2d96f513b09..05e68fc1d91b 100644 --- a/python/mozperftest/mozperftest/tests/test_mach_commands.py +++ b/python/mozperftest/mozperftest/tests/test_mach_commands.py @@ -7,6 +7,7 @@ import os import mock import tempfile import shutil +from contextlib import contextmanager from mach.registrar import Registrar @@ -16,6 +17,7 @@ Registrar.commands_by_category = {"testing": set()} from mozperftest.environment import MachEnvironment from mozperftest.mach_commands import Perftest +from mozperftest.tests.support import EXAMPLE_TESTS_DIR class _TestMachEnvironment(MachEnvironment): @@ -29,8 +31,8 @@ class _TestMachEnvironment(MachEnvironment): pass -@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment) -def test_command(): +@contextmanager +def _get_perftest(): from mozbuild.base import MozbuildObject config = MozbuildObject.from_environment() @@ -43,11 +45,25 @@ def test_command(): state_dir = tempfile.mkdtemp() try: - test = Perftest(context()) - test.run_perftest() + yield Perftest(context()) finally: shutil.rmtree(context.state_dir) +@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment) +@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv") +def test_command(mocked_func): + with _get_perftest() as test: + test.run_perftest(tests=[EXAMPLE_TESTS_DIR]) + # XXX add assertions + + +@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment) +@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv") +def test_doc_flavor(mocked_func): + with _get_perftest() as test: + test.run_perftest(tests=[EXAMPLE_TESTS_DIR], flavor="doc") + + if __name__ == "__main__": mozunit.main() diff --git a/python/mozperftest/mozperftest/tests/test_perfherder.py b/python/mozperftest/mozperftest/tests/test_perfherder.py index f1cf86b7445b..6de6355c3e27 100644 --- a/python/mozperftest/mozperftest/tests/test_perfherder.py +++ b/python/mozperftest/mozperftest/tests/test_perfherder.py @@ -3,7 +3,7 @@ import os import mozunit import json -from mozperftest.tests.support import get_running_env, temp_file +from mozperftest.tests.support import get_running_env, temp_file, EXAMPLE_TEST from mozperftest.environment import METRICS from mozperftest.utils import silence @@ -22,7 +22,7 @@ def test_metrics(): mach_cmd.run_process = _run_process metrics = env.layers[METRICS] - env.set_arg("tests", [os.path.join(HERE, "example.js")]) + env.set_arg("tests", [EXAMPLE_TEST]) metadata.set_result(os.path.join(HERE, "browsertime-results")) with temp_file() as output: diff --git a/python/mozperftest/mozperftest/tests/test_scriptinfo.py b/python/mozperftest/mozperftest/tests/test_scriptinfo.py new file mode 100644 index 000000000000..65b22f6562b2 --- /dev/null +++ b/python/mozperftest/mozperftest/tests/test_scriptinfo.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import mozunit + +from mozperftest.scriptinfo import ScriptInfo +from mozperftest.tests.support import EXAMPLE_TEST + + +def test_scriptinfo(): + info = ScriptInfo(EXAMPLE_TEST) + assert info["author"] == "N/A" + + display = str(info) + assert "appropriate android app" in display + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mozperftest/mozperftest/utils.py b/python/mozperftest/mozperftest/utils.py index 9c45a37645ce..8991803a2ac9 100644 --- a/python/mozperftest/mozperftest/utils.py +++ b/python/mozperftest/mozperftest/utils.py @@ -4,6 +4,8 @@ import logging import contextlib import sys +import os + from six import StringIO @@ -54,3 +56,20 @@ class MachLogger: def error(self, msg, name="mozperftest", **kwargs): self._logger(logging.ERROR, name, kwargs, msg) + + +def install_package(virtualenv_manager, package): + from pip._internal.req.constructors import install_req_from_line + + req = install_req_from_line(package) + req.check_if_exists(use_user_site=False) + # already installed, check if it's in our venv + if req.satisfied_by is not None: + venv_site_lib = os.path.abspath( + os.path.join(virtualenv_manager.bin_path, "..", "lib") + ) + site_packages = os.path.abspath(req.satisfied_by.location) + if site_packages.startswith(venv_site_lib): + # already installed in this venv, we can skip + return + virtualenv_manager._run_pip(["install", package]) diff --git a/testing/performance/hooks_applink.py b/testing/performance/hooks_applink.py index 823ec73fd8d9..171c220a1e7c 100644 --- a/testing/performance/hooks_applink.py +++ b/testing/performance/hooks_applink.py @@ -5,23 +5,28 @@ from mozperftest.browser.browsertime import add_options url = "'https://www.example.com'" -common_options = [("processStartTime", "true"), - ("firefox.disableBrowsertimeExtension", "true"), - ("firefox.android.intentArgument", "'-a'"), - ("firefox.android.intentArgument", "'android.intent.action.VIEW'"), - ("firefox.android.intentArgument", "'-d'"), - ("firefox.android.intentArgument", url)] +common_options = [ + ("processStartTime", "true"), + ("firefox.disableBrowsertimeExtension", "true"), + ("firefox.android.intentArgument", "'-a'"), + ("firefox.android.intentArgument", "'android.intent.action.VIEW'"), + ("firefox.android.intentArgument", "'-d'"), + ("firefox.android.intentArgument", url), +] app_options = { "org.mozilla.geckoview_example": [ - ("firefox.android.activity", "'org.mozilla.geckoview_example.GeckoViewActivity'") + ( + "firefox.android.activity", + "'org.mozilla.geckoview_example.GeckoViewActivity'", + ) ], "org.mozilla.fennec_aurora": [ ("firefox.android.activity", "'org.mozilla.fenix.IntentReceiverActivity'") ], "org.mozilla.firefox": [ ("firefox.android.activity", "'org.mozilla.gecko.BrowserApp'") - ] + ], } diff --git a/testing/performance/perftest_applink.js b/testing/performance/perftest_applink.js index f372466c0065..d05f7b362f7a 100644 --- a/testing/performance/perftest_applink.js +++ b/testing/performance/perftest_applink.js @@ -1,8 +1,10 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. /* eslint-env node */ +"use strict"; -module.exports = async function(context, commands) { - "use strict"; - +async function test(context, commands) { // This violates all sorts of abstraction boundaries, but I don't see supported APIs for "just // waiting" nor for allowing navigation scripts to produce measurements. await commands.measure.start(); @@ -19,4 +21,25 @@ module.exports = async function(context, commands) { console.log("processLaunchToNavStart: " + processLaunchToNavStart); return true; +} + +module.exports = { + test, + owner: "Performance Team", + description: "Measures cold process applink time", + long_description: ` + This test launches the appropriate android app, simulating an app link + workflow. The application is launched with the intent action + android.intent.action.VIEW loading a trivially simple website. The reported + metric is the time from process start to navigationStart, reported as processLaunchToNavStart + `, + usage: ` + ./mach perftest testing/performance/perftest_applink.js \ + --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk \ + --hooks testing/performance/hooks_applink.py \ + --android-app-name org.mozilla.fennec_aurora \ + --perfherder-metrics processLaunchToNavStart + `, + supported_browser: ["Fenix nightly", "Geckoview_example", "Fennec"], + platform: ["Android"], }; diff --git a/third_party/python/esprima/PKG-INFO b/third_party/python/esprima/PKG-INFO new file mode 100644 index 000000000000..c2fee6ace6f6 --- /dev/null +++ b/third_party/python/esprima/PKG-INFO @@ -0,0 +1,143 @@ +Metadata-Version: 1.1 +Name: esprima +Version: 4.0.1 +Summary: ECMAScript parsing infrastructure for multipurpose analysis in Python +Home-page: https://github.com/Kronuz/esprima-python +Author: German M. Bravo (Kronuz) +Author-email: german.mb@gmail.com +License: BSD License +Description: |Donate| |PyPI Version| |PyPI License| |PyPI Format| |PyPI Status| + + **Esprima** (`esprima.org `__, BSD license) is a + high performance, standard-compliant + `ECMAScript `__ + parser officially written in ECMAScript (also popularly known as + `JavaScript `__) and ported to + Python. Esprima is created and maintained by `Ariya + Hidayat `__, with the help of `many + contributors `__. + + Python port is a line-by-line manual translation and was created and is + maintained by `German Mendez Bravo + (Kronuz) `__. + + Features + ~~~~~~~~ + + - Full support for ECMAScript 2017 (`ECMA-262 8th + Edition `__) + - Sensible `syntax tree + format `__ as + standardized by `ESTree project `__ + - Experimental support for `JSX `__, a + syntax extension for `React `__ + - Optional tracking of syntax node location (index-based and + line-column) + - `Heavily tested `__ (~1500 `unit + tests `__ + with `full code + coverage `__) + + Installation + ~~~~~~~~~~~~ + + .. code:: shell + + pip install esprima + + API + ~~~ + + Esprima can be used to perform `lexical + analysis `__ + (tokenization) or `syntactic + analysis `__ (parsing) of a + JavaScript program. + + A simple example: + + .. code:: javascript + + >>> import esprima + >>> program = 'const answer = 42' + + >>> esprima.tokenize(program) + [{ + type: "Keyword", + value: "const" + }, { + type: "Identifier", + value: "answer" + }, { + type: "Punctuator", + value: "=" + }, { + type: "Numeric", + value: "42" + }] + + >>> esprima.parseScript(program) + { + body: [ + { + kind: "const", + declarations: [ + { + init: { + raw: "42", + type: "Literal", + value: 42 + }, + type: "VariableDeclarator", + id: { + type: "Identifier", + name: "answer" + } + } + ], + type: "VariableDeclaration" + } + ], + type: "Program", + sourceType: "script" + } + + For more information, please read the `complete + documentation `__. + + .. |Donate| image:: https://img.shields.io/badge/Donate-PayPal-green.svg + :target: https://www.paypal.me/Kronuz/25 + .. |PyPI Version| image:: https://img.shields.io/pypi/v/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI License| image:: https://img.shields.io/pypi/l/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Wheel| image:: https://img.shields.io/pypi/wheel/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Format| image:: https://img.shields.io/pypi/format/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Implementation| image:: https://img.shields.io/pypi/implementation/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Status| image:: https://img.shields.io/pypi/status/esprima.svg + :target: https://pypi.python.org/pypi/esprima + .. |PyPI Downloads| image:: https://img.shields.io/pypi/dm/esprima.svg + :target: https://pypi.python.org/pypi/esprima +Keywords: esprima ecmascript javascript parser ast +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Topic :: Software Development :: Code Generators +Classifier: Topic :: Software Development :: Compilers +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: General +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 diff --git a/third_party/python/esprima/README b/third_party/python/esprima/README new file mode 100644 index 000000000000..442fbc7b11fa --- /dev/null +++ b/third_party/python/esprima/README @@ -0,0 +1,117 @@ +|Donate| |PyPI Version| |PyPI License| |PyPI Format| |PyPI Status| + +**Esprima** (`esprima.org `__, BSD license) is a +high performance, standard-compliant +`ECMAScript `__ +parser officially written in ECMAScript (also popularly known as +`JavaScript `__) and ported to +Python. Esprima is created and maintained by `Ariya +Hidayat `__, with the help of `many +contributors `__. + +Python port is a line-by-line manual translation and was created and is +maintained by `German Mendez Bravo +(Kronuz) `__. + +Features +~~~~~~~~ + +- Full support for ECMAScript 2017 (`ECMA-262 8th + Edition `__) +- Sensible `syntax tree + format `__ as + standardized by `ESTree project `__ +- Experimental support for `JSX `__, a + syntax extension for `React `__ +- Optional tracking of syntax node location (index-based and + line-column) +- `Heavily tested `__ (~1500 `unit + tests `__ + with `full code + coverage `__) + +Installation +~~~~~~~~~~~~ + +.. code:: shell + + pip install esprima + +API +~~~ + +Esprima can be used to perform `lexical +analysis `__ +(tokenization) or `syntactic +analysis `__ (parsing) of a +JavaScript program. + +A simple example: + +.. code:: javascript + + >>> import esprima + >>> program = 'const answer = 42' + + >>> esprima.tokenize(program) + [{ + type: "Keyword", + value: "const" + }, { + type: "Identifier", + value: "answer" + }, { + type: "Punctuator", + value: "=" + }, { + type: "Numeric", + value: "42" + }] + + >>> esprima.parseScript(program) + { + body: [ + { + kind: "const", + declarations: [ + { + init: { + raw: "42", + type: "Literal", + value: 42 + }, + type: "VariableDeclarator", + id: { + type: "Identifier", + name: "answer" + } + } + ], + type: "VariableDeclaration" + } + ], + type: "Program", + sourceType: "script" + } + +For more information, please read the `complete +documentation `__. + +.. |Donate| image:: https://img.shields.io/badge/Donate-PayPal-green.svg + :target: https://www.paypal.me/Kronuz/25 +.. |PyPI Version| image:: https://img.shields.io/pypi/v/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI License| image:: https://img.shields.io/pypi/l/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Wheel| image:: https://img.shields.io/pypi/wheel/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Format| image:: https://img.shields.io/pypi/format/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Implementation| image:: https://img.shields.io/pypi/implementation/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Status| image:: https://img.shields.io/pypi/status/esprima.svg + :target: https://pypi.python.org/pypi/esprima +.. |PyPI Downloads| image:: https://img.shields.io/pypi/dm/esprima.svg + :target: https://pypi.python.org/pypi/esprima diff --git a/third_party/python/esprima/esprima/__init__.py b/third_party/python/esprima/esprima/__init__.py new file mode 100644 index 000000000000..0dcdf99e5ec2 --- /dev/null +++ b/third_party/python/esprima/esprima/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import + +version = '4.0.1' +__version__ = (4, 0, 1) + +from .esprima import * # NOQA diff --git a/third_party/python/esprima/esprima/__main__.py b/third_party/python/esprima/esprima/__main__.py new file mode 100644 index 000000000000..92f2aa2ec596 --- /dev/null +++ b/third_party/python/esprima/esprima/__main__.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals, print_function, division + +import sys + +from .esprima import parse, tokenize, Error, toDict +from . import version + + +def main(): + import json + import time + import optparse + + usage = "usage: %prog [options] [file.js]" + parser = optparse.OptionParser(usage=usage, version=version) + parser.add_option("--comment", dest="comment", + action="store_true", default=False, + help="Gather all line and block comments in an array") + parser.add_option("--attachComment", dest="attachComment", + action="store_true", default=False, + help="Attach comments to nodes") + parser.add_option("--loc", dest="loc", default=False, + action="store_true", + help="Include line-column location info for each syntax node") + parser.add_option("--range", dest="range", default=False, + action="store_true", + help="Include index-based range for each syntax node") + parser.add_option("--raw", dest="raw", default=False, + action="store_true", + help="Display the raw value of literals") + parser.add_option("--tokens", dest="tokens", default=False, + action="store_true", + help="List all tokens in an array") + parser.add_option("--tolerant", dest="tolerant", default=False, + action="store_true", + help="Tolerate errors on a best-effort basis (experimental)") + parser.add_option("--tokenize", dest="tokenize", default=False, + action="store_true", + help="Only tokenize, do not parse.") + parser.add_option("--module", dest="sourceType", default='string', + action="store_const", const='module', + help="Tolerate errors on a best-effort basis (experimental)") + parser.set_defaults(jsx=True, classProperties=True) + opts, args = parser.parse_args() + + if len(args) == 1: + with open(args[0], 'rb') as f: + code = f.read().decode('utf-8') + elif sys.stdin.isatty(): + parser.print_help() + return 64 + else: + code = sys.stdin.read().decode('utf-8') + + options = opts.__dict__ + do_tokenize = options.pop('tokenize') + + t = time.time() + try: + if do_tokenize: + del options['sourceType'] + del options['tokens'] + del options['raw'] + del options['jsx'] + res = toDict(tokenize(code, options=options)) + else: + res = toDict(parse(code, options=options)) + except Error as e: + res = e.toDict() + dt = time.time() - t + 0.000000001 + + print(json.dumps(res, indent=4)) + print() + print('Parsed everyting in', round(dt, 5), 'seconds.') + print('Thats %d characters per second' % (len(code) // dt)) + + return 0 + + +if __name__ == '__main__': + retval = main() + sys.exit(retval) diff --git a/third_party/python/esprima/esprima/character.py b/third_party/python/esprima/esprima/character.py new file mode 100644 index 000000000000..a650a714a949 --- /dev/null +++ b/third_party/python/esprima/esprima/character.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +import sys + +import unicodedata +from collections import defaultdict + +from .compat import uchr, xrange + +# http://stackoverflow.com/questions/14245893/efficiently-list-all-characters-in-a-given-unicode-category +U_CATEGORIES = defaultdict(list) +for c in map(uchr, xrange(sys.maxunicode + 1)): + U_CATEGORIES[unicodedata.category(c)].append(c) +UNICODE_LETTER = set( + U_CATEGORIES['Lu'] + U_CATEGORIES['Ll'] + + U_CATEGORIES['Lt'] + U_CATEGORIES['Lm'] + + U_CATEGORIES['Lo'] + U_CATEGORIES['Nl'] +) +UNICODE_OTHER_ID_START = set(( + # Other_ID_Start + '\u1885', '\u1886', '\u2118', '\u212E', '\u309B', '\u309C', + # New in Unicode 8.0 + '\u08B3', '\u0AF9', '\u13F8', '\u9FCD', '\uAB60', '\U00010CC0', '\U000108E0', '\U0002B820', + # New in Unicode 9.0 + '\u1C80', '\U000104DB', '\U0001E922', + '\U0001EE00', '\U0001EE06', '\U0001EE0A', +)) +UNICODE_OTHER_ID_CONTINUE = set(( + # Other_ID_Continue + '\xB7', '\u0387', '\u1369', '\u136A', '\u136B', '\u136C', + '\u136D', '\u136E', '\u136F', '\u1370', '\u1371', '\u19DA', + # New in Unicode 8.0 + '\u08E3', '\uA69E', '\U00011730', + # New in Unicode 9.0 + '\u08D4', '\u1DFB', '\uA8C5', '\U00011450', + '\U0001EE03', '\U0001EE0B', +)) +UNICODE_COMBINING_MARK = set(U_CATEGORIES['Mn'] + U_CATEGORIES['Mc']) +UNICODE_DIGIT = set(U_CATEGORIES['Nd']) +UNICODE_CONNECTOR_PUNCTUATION = set(U_CATEGORIES['Pc']) +IDENTIFIER_START = UNICODE_LETTER.union(UNICODE_OTHER_ID_START).union(set(('$', '_', '\\'))) +IDENTIFIER_PART = IDENTIFIER_START.union(UNICODE_COMBINING_MARK).union(UNICODE_DIGIT).union(UNICODE_CONNECTOR_PUNCTUATION).union(set(('\u200D', '\u200C'))).union(UNICODE_OTHER_ID_CONTINUE) + +WHITE_SPACE = set(( + '\x09', '\x0B', '\x0C', '\x20', '\xA0', + '\u1680', '\u180E', '\u2000', '\u2001', '\u2002', + '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', + '\u2008', '\u2009', '\u200A', '\u202F', '\u205F', + '\u3000', '\uFEFF', +)) +LINE_TERMINATOR = set(('\x0A', '\x0D', '\u2028', '\u2029')) + +DECIMAL_CONV = dict((c, n) for n, c in enumerate('0123456789')) +OCTAL_CONV = dict((c, n) for n, c in enumerate('01234567')) +HEX_CONV = dict((c, n) for n, c in enumerate('0123456789abcdef')) +for n, c in enumerate('ABCDEF', 10): + HEX_CONV[c] = n +DECIMAL_DIGIT = set(DECIMAL_CONV.keys()) +OCTAL_DIGIT = set(OCTAL_CONV.keys()) +HEX_DIGIT = set(HEX_CONV.keys()) + + +class Character: + @staticmethod + def fromCodePoint(code): + return uchr(code) + + # https://tc39.github.io/ecma262/#sec-white-space + + @staticmethod + def isWhiteSpace(ch): + return ch in WHITE_SPACE + + # https://tc39.github.io/ecma262/#sec-line-terminators + + @staticmethod + def isLineTerminator(ch): + return ch in LINE_TERMINATOR + + # https://tc39.github.io/ecma262/#sec-names-and-keywords + + @staticmethod + def isIdentifierStart(ch): + return ch in IDENTIFIER_START + + @staticmethod + def isIdentifierPart(ch): + return ch in IDENTIFIER_PART + + # https://tc39.github.io/ecma262/#sec-literals-numeric-literals + + @staticmethod + def isDecimalDigit(ch): + return ch in DECIMAL_DIGIT + + @staticmethod + def isHexDigit(ch): + return ch in HEX_DIGIT + + @staticmethod + def isOctalDigit(ch): + return ch in OCTAL_DIGIT diff --git a/third_party/python/esprima/esprima/comment_handler.py b/third_party/python/esprima/esprima/comment_handler.py new file mode 100644 index 000000000000..09a37a5fd2f0 --- /dev/null +++ b/third_party/python/esprima/esprima/comment_handler.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, self.list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, self.list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# self.SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES +# LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# self.SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .objects import Object +from .nodes import Node +from .syntax import Syntax + + +class Comment(Node): + def __init__(self, type, value, range=None, loc=None): + self.type = type + self.value = value + self.range = range + self.loc = loc + + +class Entry(Object): + def __init__(self, comment, start): + self.comment = comment + self.start = start + + +class NodeInfo(Object): + def __init__(self, node, start): + self.node = node + self.start = start + + +class CommentHandler(object): + def __init__(self): + self.attach = False + self.comments = [] + self.stack = [] + self.leading = [] + self.trailing = [] + + def insertInnerComments(self, node, metadata): + # innnerComments for properties empty block + # `function a(:/** comments **\/}` + if node.type is Syntax.BlockStatement and not node.body: + innerComments = [] + for i, entry in enumerate(self.leading): + if metadata.end.offset >= entry.start: + innerComments.append(entry.comment) + self.leading[i] = None + self.trailing[i] = None + if innerComments: + node.innerComments = innerComments + self.leading = [v for v in self.leading if v is not None] + self.trailing = [v for v in self.trailing if v is not None] + + def findTrailingComments(self, metadata): + trailingComments = [] + + if self.trailing: + for i, entry in enumerate(self.trailing): + if entry.start >= metadata.end.offset: + trailingComments.append(entry.comment) + if trailingComments: + self.trailing = [] + return trailingComments + + last = self.stack and self.stack[-1] + if last and last.node.trailingComments: + firstComment = last.node.trailingComments[0] + if firstComment and firstComment.range[0] >= metadata.end.offset: + trailingComments = last.node.trailingComments + del last.node.trailingComments + return trailingComments + + def findLeadingComments(self, metadata): + leadingComments = [] + + target = None + while self.stack: + entry = self.stack and self.stack[-1] + if entry and entry.start >= metadata.start.offset: + target = entry.node + self.stack.pop() + else: + break + + if target: + if target.leadingComments: + for i, comment in enumerate(target.leadingComments): + if comment.range[1] <= metadata.start.offset: + leadingComments.append(comment) + target.leadingComments[i] = None + if leadingComments: + target.leadingComments = [v for v in target.leadingComments if v is not None] + if not target.leadingComments: + del target.leadingComments + return leadingComments + + for i, entry in enumerate(self.leading): + if entry.start <= metadata.start.offset: + leadingComments.append(entry.comment) + self.leading[i] = None + if leadingComments: + self.leading = [v for v in self.leading if v is not None] + + return leadingComments + + def visitNode(self, node, metadata): + if node.type is Syntax.Program and node.body: + return + + self.insertInnerComments(node, metadata) + trailingComments = self.findTrailingComments(metadata) + leadingComments = self.findLeadingComments(metadata) + if leadingComments: + node.leadingComments = leadingComments + if trailingComments: + node.trailingComments = trailingComments + + self.stack.append(NodeInfo( + node=node, + start=metadata.start.offset + )) + + def visitComment(self, node, metadata): + type = 'Line' if node.type[0] == 'L' else 'Block' + comment = Comment( + type=type, + value=node.value + ) + if node.range: + comment.range = node.range + if node.loc: + comment.loc = node.loc + self.comments.append(comment) + + if self.attach: + entry = Entry( + comment=Comment( + type=type, + value=node.value, + range=[metadata.start.offset, metadata.end.offset] + ), + start=metadata.start.offset + ) + if node.loc: + entry.comment.loc = node.loc + node.type = type + self.leading.append(entry) + self.trailing.append(entry) + + def visit(self, node, metadata): + if node.type == 'LineComment': + self.visitComment(node, metadata) + elif node.type == 'BlockComment': + self.visitComment(node, metadata) + elif self.attach: + self.visitNode(node, metadata) diff --git a/third_party/python/esprima/esprima/compat.py b/third_party/python/esprima/esprima/compat.py new file mode 100644 index 000000000000..79543255e377 --- /dev/null +++ b/third_party/python/esprima/esprima/compat.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +import sys + +PY3 = sys.version_info >= (3, 0) + +if PY3: + # Python 3: + basestring = str + long = int + xrange = range + unicode = str + uchr = chr + + def uord(ch): + return ord(ch[0]) + +else: + basestring = basestring + long = long + xrange = xrange + unicode = unicode + + try: + # Python 2 UCS4: + unichr(0x10000) + uchr = unichr + + def uord(ch): + return ord(ch[0]) + + except ValueError: + # Python 2 UCS2: + def uchr(code): + # UTF-16 Encoding + if code <= 0xFFFF: + return unichr(code) + cu1 = ((code - 0x10000) >> 10) + 0xD800 + cu2 = ((code - 0x10000) & 1023) + 0xDC00 + return unichr(cu1) + unichr(cu2) + + def uord(ch): + cp = ord(ch[0]) + if cp >= 0xD800 and cp <= 0xDBFF: + second = ord(ch[1]) + if second >= 0xDC00 and second <= 0xDFFF: + first = cp + cp = (first - 0xD800) * 0x400 + second - 0xDC00 + 0x10000 + return cp diff --git a/third_party/python/esprima/esprima/error_handler.py b/third_party/python/esprima/esprima/error_handler.py new file mode 100644 index 000000000000..9b0f5cb843f3 --- /dev/null +++ b/third_party/python/esprima/esprima/error_handler.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import unicode_literals + +from .compat import unicode + + +class Error(Exception): + def __init__(self, message, name=None, index=None, lineNumber=None, column=None, description=None): + super(Error, self).__init__(message) + self.message = message + self.name = name + self.index = index + self.lineNumber = lineNumber + self.column = column + # self.description = description + + def toString(self): + return '%s: %s' % (self.__class__.__name__, self) + + def toDict(self): + d = dict((unicode(k), v) for k, v in self.__dict__.items() if v is not None) + d['message'] = self.toString() + return d + + +class ErrorHandler: + def __init__(self): + self.errors = [] + self.tolerant = False + + def recordError(self, error): + self.errors.append(error.toDict()) + + def tolerate(self, error): + if self.tolerant: + self.recordError(error) + else: + raise error + + def createError(self, index, line, col, description): + msg = 'Line %s: %s' % (line, description) + return Error(msg, index=index, lineNumber=line, column=col, description=description) + + def throwError(self, index, line, col, description): + raise self.createError(index, line, col, description) + + def tolerateError(self, index, line, col, description): + error = self.createError(index, line, col, description) + if self.tolerant: + self.recordError(error) + else: + raise error diff --git a/third_party/python/esprima/esprima/esprima.py b/third_party/python/esprima/esprima/esprima.py new file mode 100644 index 000000000000..faea0c2dda06 --- /dev/null +++ b/third_party/python/esprima/esprima/esprima.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .comment_handler import CommentHandler +from .error_handler import Error +from .jsx_parser import JSXParser +from .jsx_syntax import JSXSyntax +from .objects import Array, toDict +from .parser import Parser +from .syntax import Syntax +from .tokenizer import Tokenizer +from .visitor import NodeVisitor +from . import nodes +from . import jsx_nodes + + +__all__ = ['Syntax', 'JSXSyntax', 'Error', 'NodeVisitor', 'nodes', 'jsx_nodes', + 'parse', 'parseModule', 'parseScript', 'tokenize', 'toDict'] + + +def parse(code, options=None, delegate=None, **kwargs): + options = {} if options is None else options.copy() + options.update(kwargs) + + # ESNext presset: + if options.get('esnext', False): + options['jsx'] = True + options['classProperties'] = True + + commentHandler = None + + def proxyDelegate(node, metadata): + if delegate: + new_node = delegate(node, metadata) + if new_node is not None: + node = new_node + if commentHandler: + commentHandler.visit(node, metadata) + return node + + parserDelegate = None if delegate is None else proxyDelegate + collectComment = options.get('comment', False) + attachComment = options.get('attachComment', False) + if collectComment or attachComment: + commentHandler = CommentHandler() + commentHandler.attach = attachComment + options['comment'] = True + parserDelegate = proxyDelegate + + isModule = options.get('sourceType', 'script') == 'module' + + if options.get('jsx', False): + parser = JSXParser(code, options=options, delegate=parserDelegate) + else: + parser = Parser(code, options=options, delegate=parserDelegate) + + ast = parser.parseModule() if isModule else parser.parseScript() + + if collectComment and commentHandler: + ast.comments = commentHandler.comments + + if parser.config.tokens: + ast.tokens = parser.tokens + + if parser.config.tolerant: + ast.errors = parser.errorHandler.errors + + return ast + + +def parseModule(code, options=None, delegate=None, **kwargs): + kwargs['sourceType'] = 'module' + return parse(code, options, delegate, **kwargs) + + +def parseScript(code, options=None, delegate=None, **kwargs): + kwargs['sourceType'] = 'script' + return parse(code, options, delegate, **kwargs) + + +def tokenize(code, options=None, delegate=None, **kwargs): + options = {} if options is None else options.copy() + options.update(kwargs) + + tokenizer = Tokenizer(code, options) + + tokens = Array() + + try: + while True: + token = tokenizer.getNextToken() + if not token: + break + if delegate: + token = delegate(token) + tokens.append(token) + except Error as e: + tokenizer.errorHandler.tolerate(e) + + if tokenizer.errorHandler.tolerant: + tokens.errors = tokenizer.errors() + + return tokens diff --git a/third_party/python/esprima/esprima/jsx_nodes.py b/third_party/python/esprima/esprima/jsx_nodes.py new file mode 100644 index 000000000000..f195653ab581 --- /dev/null +++ b/third_party/python/esprima/esprima/jsx_nodes.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .nodes import Node +from .jsx_syntax import JSXSyntax + + +class JSXClosingElement(Node): + def __init__(self, name): + self.type = JSXSyntax.JSXClosingElement + self.name = name + + +class JSXElement(Node): + def __init__(self, openingElement, children, closingElement): + self.type = JSXSyntax.JSXElement + self.openingElement = openingElement + self.children = children + self.closingElement = closingElement + + +class JSXEmptyExpression(Node): + def __init__(self): + self.type = JSXSyntax.JSXEmptyExpression + + +class JSXExpressionContainer(Node): + def __init__(self, expression): + self.type = JSXSyntax.JSXExpressionContainer + self.expression = expression + + +class JSXIdentifier(Node): + def __init__(self, name): + self.type = JSXSyntax.JSXIdentifier + self.name = name + + +class JSXMemberExpression(Node): + def __init__(self, object, property): + self.type = JSXSyntax.JSXMemberExpression + self.object = object + self.property = property + + +class JSXAttribute(Node): + def __init__(self, name, value): + self.type = JSXSyntax.JSXAttribute + self.name = name + self.value = value + + +class JSXNamespacedName(Node): + def __init__(self, namespace, name): + self.type = JSXSyntax.JSXNamespacedName + self.namespace = namespace + self.name = name + + +class JSXOpeningElement(Node): + def __init__(self, name, selfClosing, attributes): + self.type = JSXSyntax.JSXOpeningElement + self.name = name + self.selfClosing = selfClosing + self.attributes = attributes + + +class JSXSpreadAttribute(Node): + def __init__(self, argument): + self.type = JSXSyntax.JSXSpreadAttribute + self.argument = argument + + +class JSXText(Node): + def __init__(self, value, raw): + self.type = JSXSyntax.JSXText + self.value = value + self.raw = raw diff --git a/third_party/python/esprima/esprima/jsx_parser.py b/third_party/python/esprima/esprima/jsx_parser.py new file mode 100644 index 000000000000..ec71b9251df1 --- /dev/null +++ b/third_party/python/esprima/esprima/jsx_parser.py @@ -0,0 +1,584 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .compat import uchr +from .character import Character +from . import jsx_nodes as JSXNode +from .jsx_syntax import JSXSyntax +from . import nodes as Node +from .parser import Marker, Parser +from .token import Token, TokenName +from .xhtml_entities import XHTMLEntities + + +class MetaJSXElement(object): + def __init__(self, node=None, opening=None, closing=None, children=None): + self.node = node + self.opening = opening + self.closing = closing + self.children = children + + +class JSXToken(object): + Identifier = 100 + Text = 101 + + +class RawJSXToken(object): + def __init__(self, type=None, value=None, lineNumber=None, lineStart=None, start=None, end=None): + self.type = type + self.value = value + self.lineNumber = lineNumber + self.lineStart = lineStart + self.start = start + self.end = end + + +TokenName[JSXToken.Identifier] = "JSXIdentifier" +TokenName[JSXToken.Text] = "JSXText" + + +# Fully qualified element name, e.g. returns "svg:path" +def getQualifiedElementName(elementName): + typ = elementName.type + if typ is JSXSyntax.JSXIdentifier: + id = elementName + qualifiedName = id.name + elif typ is JSXSyntax.JSXNamespacedName: + ns = elementName + qualifiedName = getQualifiedElementName(ns.namespace) + ':' + getQualifiedElementName(ns.name) + elif typ is JSXSyntax.JSXMemberExpression: + expr = elementName + qualifiedName = getQualifiedElementName(expr.object) + '.' + getQualifiedElementName(expr.property) + + return qualifiedName + + +class JSXParser(Parser): + def __init__(self, code, options, delegate): + super(JSXParser, self).__init__(code, options, delegate) + + def parsePrimaryExpression(self): + return self.parseJSXRoot() if self.match('<') else super(JSXParser, self).parsePrimaryExpression() + + def startJSX(self): + # Unwind the scanner before the lookahead token. + self.scanner.index = self.startMarker.index + self.scanner.lineNumber = self.startMarker.line + self.scanner.lineStart = self.startMarker.index - self.startMarker.column + + def finishJSX(self): + # Prime the next lookahead. + self.nextToken() + + def reenterJSX(self): + self.startJSX() + self.expectJSX('}') + + # Pop the closing '}' added from the lookahead. + if self.config.tokens: + self.tokens.pop() + + def createJSXNode(self): + self.collectComments() + return Marker( + index=self.scanner.index, + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart + ) + + def createJSXChildNode(self): + return Marker( + index=self.scanner.index, + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart + ) + + def scanXHTMLEntity(self, quote): + result = '&' + + valid = True + terminated = False + numeric = False + hex = False + + while not self.scanner.eof() and valid and not terminated: + ch = self.scanner.source[self.scanner.index] + if ch == quote: + break + + terminated = (ch == ';') + result += ch + self.scanner.index += 1 + if not terminated: + length = len(result) + if length == 2: + # e.g. '{' + numeric = (ch == '#') + elif length == 3: + if numeric: + # e.g. 'A' + hex = ch == 'x' + valid = hex or Character.isDecimalDigit(ch) + numeric = numeric and not hex + else: + valid = valid and not (numeric and not Character.isDecimalDigit(ch)) + valid = valid and not (hex and not Character.isHexDigit(ch)) + + if valid and terminated and len(result) > 2: + # e.g. 'A' becomes just '#x41' + st = result[1:-1] + if numeric and len(st) > 1: + result = uchr(int(st[1:], 10)) + elif hex and len(st) > 2: + result = uchr(int(st[2:], 16)) + elif not numeric and not hex and st in XHTMLEntities: + result = XHTMLEntities[st] + + return result + + # Scan the next JSX token. This replaces Scanner#lex when in JSX mode. + + def lexJSX(self): + ch = self.scanner.source[self.scanner.index] + + # < > / : = { } + if ch in ('<', '>', '/', ':', '=', '{', '}'): + value = self.scanner.source[self.scanner.index] + self.scanner.index += 1 + return RawJSXToken( + type=Token.Punctuator, + value=value, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=self.scanner.index - 1, + end=self.scanner.index + ) + + # " ' + if ch in ('\'', '"'): + start = self.scanner.index + quote = self.scanner.source[self.scanner.index] + self.scanner.index += 1 + str = '' + while not self.scanner.eof(): + ch = self.scanner.source[self.scanner.index] + self.scanner.index += 1 + if ch == quote: + break + elif ch == '&': + str += self.scanXHTMLEntity(quote) + else: + str += ch + + return RawJSXToken( + type=Token.StringLiteral, + value=str, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + # ... or . + if ch == '.': + start = self.scanner.index + if self.scanner.source[start + 1:start + 3] == '..': + value = '...' + self.scanner.index += 3 + else: + value = '.' + self.scanner.index += 1 + return RawJSXToken( + type=Token.Punctuator, + value=value, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + # ` + if ch == '`': + # Only placeholder, since it will be rescanned as a real assignment expression. + return RawJSXToken( + type=Token.Template, + value='', + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=self.scanner.index, + end=self.scanner.index + ) + + # Identifer can not contain backslash (char code 92). + if Character.isIdentifierStart(ch) and ch != '\\': + start = self.scanner.index + self.scanner.index += 1 + while not self.scanner.eof(): + ch = self.scanner.source[self.scanner.index] + if Character.isIdentifierPart(ch) and ch != '\\': + self.scanner.index += 1 + elif ch == '-': + # Hyphen (char code 45) can be part of an identifier. + self.scanner.index += 1 + else: + break + + id = self.scanner.source[start:self.scanner.index] + return RawJSXToken( + type=JSXToken.Identifier, + value=id, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + return self.scanner.lex() + + def nextJSXToken(self): + self.collectComments() + + self.startMarker.index = self.scanner.index + self.startMarker.line = self.scanner.lineNumber + self.startMarker.column = self.scanner.index - self.scanner.lineStart + token = self.lexJSX() + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + + if self.config.tokens: + self.tokens.append(self.convertToken(token)) + + return token + + def nextJSXText(self): + self.startMarker.index = self.scanner.index + self.startMarker.line = self.scanner.lineNumber + self.startMarker.column = self.scanner.index - self.scanner.lineStart + + start = self.scanner.index + + text = '' + while not self.scanner.eof(): + ch = self.scanner.source[self.scanner.index] + if ch in ('{', '<'): + break + + self.scanner.index += 1 + text += ch + if Character.isLineTerminator(ch): + self.scanner.lineNumber += 1 + if ch == '\r' and self.scanner.source[self.scanner.index] == '\n': + self.scanner.index += 1 + + self.scanner.lineStart = self.scanner.index + + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + + token = RawJSXToken( + type=JSXToken.Text, + value=text, + lineNumber=self.scanner.lineNumber, + lineStart=self.scanner.lineStart, + start=start, + end=self.scanner.index + ) + + if text and self.config.tokens: + self.tokens.append(self.convertToken(token)) + + return token + + def peekJSXToken(self): + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.lexJSX() + self.scanner.restoreState(state) + + return next + + # Expect the next JSX token to match the specified punctuator. + # If not, an exception will be thrown. + + def expectJSX(self, value): + token = self.nextJSXToken() + if token.type is not Token.Punctuator or token.value != value: + self.throwUnexpectedToken(token) + + # Return True if the next JSX token matches the specified punctuator. + + def matchJSX(self, *value): + next = self.peekJSXToken() + return next.type is Token.Punctuator and next.value in value + + def parseJSXIdentifier(self): + node = self.createJSXNode() + token = self.nextJSXToken() + if token.type is not JSXToken.Identifier: + self.throwUnexpectedToken(token) + + return self.finalize(node, JSXNode.JSXIdentifier(token.value)) + + def parseJSXElementName(self): + node = self.createJSXNode() + elementName = self.parseJSXIdentifier() + + if self.matchJSX(':'): + namespace = elementName + self.expectJSX(':') + name = self.parseJSXIdentifier() + elementName = self.finalize(node, JSXNode.JSXNamespacedName(namespace, name)) + elif self.matchJSX('.'): + while self.matchJSX('.'): + object = elementName + self.expectJSX('.') + property = self.parseJSXIdentifier() + elementName = self.finalize(node, JSXNode.JSXMemberExpression(object, property)) + + return elementName + + def parseJSXAttributeName(self): + node = self.createJSXNode() + + identifier = self.parseJSXIdentifier() + if self.matchJSX(':'): + namespace = identifier + self.expectJSX(':') + name = self.parseJSXIdentifier() + attributeName = self.finalize(node, JSXNode.JSXNamespacedName(namespace, name)) + else: + attributeName = identifier + + return attributeName + + def parseJSXStringLiteralAttribute(self): + node = self.createJSXNode() + token = self.nextJSXToken() + if token.type is not Token.StringLiteral: + self.throwUnexpectedToken(token) + + raw = self.getTokenRaw(token) + return self.finalize(node, Node.Literal(token.value, raw)) + + def parseJSXExpressionAttribute(self): + node = self.createJSXNode() + + self.expectJSX('{') + self.finishJSX() + + if self.match('}'): + self.tolerateError('JSX attributes must only be assigned a non-empty expression') + + expression = self.parseAssignmentExpression() + self.reenterJSX() + + return self.finalize(node, JSXNode.JSXExpressionContainer(expression)) + + def parseJSXAttributeValue(self): + if self.matchJSX('{'): + return self.parseJSXExpressionAttribute() + if self.matchJSX('<'): + return self.parseJSXElement() + + return self.parseJSXStringLiteralAttribute() + + def parseJSXNameValueAttribute(self): + node = self.createJSXNode() + name = self.parseJSXAttributeName() + value = None + if self.matchJSX('='): + self.expectJSX('=') + value = self.parseJSXAttributeValue() + + return self.finalize(node, JSXNode.JSXAttribute(name, value)) + + def parseJSXSpreadAttribute(self): + node = self.createJSXNode() + self.expectJSX('{') + self.expectJSX('...') + + self.finishJSX() + argument = self.parseAssignmentExpression() + self.reenterJSX() + + return self.finalize(node, JSXNode.JSXSpreadAttribute(argument)) + + def parseJSXAttributes(self): + attributes = [] + + while not self.matchJSX('/', '>'): + attribute = self.parseJSXSpreadAttribute() if self.matchJSX('{') else self.parseJSXNameValueAttribute() + attributes.append(attribute) + + return attributes + + def parseJSXOpeningElement(self): + node = self.createJSXNode() + + self.expectJSX('<') + name = self.parseJSXElementName() + attributes = self.parseJSXAttributes() + selfClosing = self.matchJSX('/') + if selfClosing: + self.expectJSX('/') + + self.expectJSX('>') + + return self.finalize(node, JSXNode.JSXOpeningElement(name, selfClosing, attributes)) + + def parseJSXBoundaryElement(self): + node = self.createJSXNode() + + self.expectJSX('<') + if self.matchJSX('/'): + self.expectJSX('/') + elementName = self.parseJSXElementName() + self.expectJSX('>') + return self.finalize(node, JSXNode.JSXClosingElement(elementName)) + + name = self.parseJSXElementName() + attributes = self.parseJSXAttributes() + selfClosing = self.matchJSX('/') + if selfClosing: + self.expectJSX('/') + + self.expectJSX('>') + + return self.finalize(node, JSXNode.JSXOpeningElement(name, selfClosing, attributes)) + + def parseJSXEmptyExpression(self): + node = self.createJSXChildNode() + self.collectComments() + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + return self.finalize(node, JSXNode.JSXEmptyExpression()) + + def parseJSXExpressionContainer(self): + node = self.createJSXNode() + self.expectJSX('{') + + if self.matchJSX('}'): + expression = self.parseJSXEmptyExpression() + self.expectJSX('}') + else: + self.finishJSX() + expression = self.parseAssignmentExpression() + self.reenterJSX() + + return self.finalize(node, JSXNode.JSXExpressionContainer(expression)) + + def parseJSXChildren(self): + children = [] + + while not self.scanner.eof(): + node = self.createJSXChildNode() + token = self.nextJSXText() + if token.start < token.end: + raw = self.getTokenRaw(token) + child = self.finalize(node, JSXNode.JSXText(token.value, raw)) + children.append(child) + + if self.scanner.source[self.scanner.index] == '{': + container = self.parseJSXExpressionContainer() + children.append(container) + else: + break + + return children + + def parseComplexJSXElement(self, el): + stack = [] + + while not self.scanner.eof(): + el.children.extend(self.parseJSXChildren()) + node = self.createJSXChildNode() + element = self.parseJSXBoundaryElement() + if element.type is JSXSyntax.JSXOpeningElement: + opening = element + if opening.selfClosing: + child = self.finalize(node, JSXNode.JSXElement(opening, [], None)) + el.children.append(child) + else: + stack.append(el) + el = MetaJSXElement( + node=node, + opening=opening, + closing=None, + children=[], + ) + + if element.type is JSXSyntax.JSXClosingElement: + el.closing = element + open = getQualifiedElementName(el.opening.name) + close = getQualifiedElementName(el.closing.name) + if open != close: + self.tolerateError('Expected corresponding JSX closing tag for %0', open) + + if stack: + child = self.finalize(el.node, JSXNode.JSXElement(el.opening, el.children, el.closing)) + el = stack[-1] + el.children.append(child) + stack.pop() + else: + break + + return el + + def parseJSXElement(self): + node = self.createJSXNode() + + opening = self.parseJSXOpeningElement() + children = [] + closing = None + + if not opening.selfClosing: + el = self.parseComplexJSXElement(MetaJSXElement( + node=node, + opening=opening, + closing=closing, + children=children + )) + children = el.children + closing = el.closing + + return self.finalize(node, JSXNode.JSXElement(opening, children, closing)) + + def parseJSXRoot(self): + # Pop the opening '<' added from the lookahead. + if self.config.tokens: + self.tokens.pop() + + self.startJSX() + element = self.parseJSXElement() + self.finishJSX() + + return element + + def isStartOfExpression(self): + return super(JSXParser, self).isStartOfExpression() or self.match('<') diff --git a/third_party/python/esprima/esprima/jsx_syntax.py b/third_party/python/esprima/esprima/jsx_syntax.py new file mode 100644 index 000000000000..808cc8b02724 --- /dev/null +++ b/third_party/python/esprima/esprima/jsx_syntax.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import unicode_literals + + +class JSXSyntax: + JSXAttribute = "JSXAttribute" + JSXClosingElement = "JSXClosingElement" + JSXElement = "JSXElement" + JSXEmptyExpression = "JSXEmptyExpression" + JSXExpressionContainer = "JSXExpressionContainer" + JSXIdentifier = "JSXIdentifier" + JSXMemberExpression = "JSXMemberExpression" + JSXNamespacedName = "JSXNamespacedName" + JSXOpeningElement = "JSXOpeningElement" + JSXSpreadAttribute = "JSXSpreadAttribute" + JSXText = "JSXText" diff --git a/third_party/python/esprima/esprima/messages.py b/third_party/python/esprima/esprima/messages.py new file mode 100644 index 000000000000..bb6314e1ea6a --- /dev/null +++ b/third_party/python/esprima/esprima/messages.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import unicode_literals + + +# Error messages should be identical to V8. +class Messages: + ObjectPatternAsRestParameter = "Unexpected token {" + BadImportCallArity = "Unexpected token" + BadGetterArity = "Getter must not have any formal parameters" + BadSetterArity = "Setter must have exactly one formal parameter" + BadSetterRestParameter = "Setter function argument must not be a rest parameter" + ConstructorIsAsync = "Class constructor may not be an async method" + ConstructorSpecialMethod = "Class constructor may not be an accessor" + DeclarationMissingInitializer = "Missing initializer in %0 declaration" + DefaultRestParameter = "Unexpected token =" + DefaultRestProperty = "Unexpected token =" + DuplicateBinding = "Duplicate binding %0" + DuplicateConstructor = "A class may only have one constructor" + DuplicateProtoProperty = "Duplicate __proto__ fields are not allowed in object literals" + ForInOfLoopInitializer = "%0 loop variable declaration may not have an initializer" + GeneratorInLegacyContext = "Generator declarations are not allowed in legacy contexts" + IllegalBreak = "Illegal break statement" + IllegalContinue = "Illegal continue statement" + IllegalExportDeclaration = "Unexpected token" + IllegalImportDeclaration = "Unexpected token" + IllegalLanguageModeDirective = "Illegal 'use strict' directive in function with non-simple parameter list" + IllegalReturn = "Illegal return statement" + InvalidEscapedReservedWord = "Keyword must not contain escaped characters" + InvalidHexEscapeSequence = "Invalid hexadecimal escape sequence" + InvalidLHSInAssignment = "Invalid left-hand side in assignment" + InvalidLHSInForIn = "Invalid left-hand side in for-in" + InvalidLHSInForLoop = "Invalid left-hand side in for-loop" + InvalidModuleSpecifier = "Unexpected token" + InvalidRegExp = "Invalid regular expression" + LetInLexicalBinding = "let is disallowed as a lexically bound name" + MissingFromClause = "Unexpected token" + MultipleDefaultsInSwitch = "More than one default clause in switch statement" + NewlineAfterThrow = "Illegal newline after throw" + NoAsAfterImportNamespace = "Unexpected token" + NoCatchOrFinally = "Missing catch or finally after try" + ParameterAfterRestParameter = "Rest parameter must be last formal parameter" + PropertyAfterRestProperty = "Unexpected token" + Redeclaration = "%0 '%1' has already been declared" + StaticPrototype = "Classes may not have static property named prototype" + StrictCatchVariable = "Catch variable may not be eval or arguments in strict mode" + StrictDelete = "Delete of an unqualified identifier in strict mode." + StrictFunction = "In strict mode code, functions can only be declared at top level or inside a block" + StrictFunctionName = "Function name may not be eval or arguments in strict mode" + StrictLHSAssignment = "Assignment to eval or arguments is not allowed in strict mode" + StrictLHSPostfix = "Postfix increment/decrement may not have eval or arguments operand in strict mode" + StrictLHSPrefix = "Prefix increment/decrement may not have eval or arguments operand in strict mode" + StrictModeWith = "Strict mode code may not include a with statement" + StrictOctalLiteral = "Octal literals are not allowed in strict mode." + StrictParamDupe = "Strict mode function may not have duplicate parameter names" + StrictParamName = "Parameter name eval or arguments is not allowed in strict mode" + StrictReservedWord = "Use of future reserved word in strict mode" + StrictVarName = "Variable name may not be eval or arguments in strict mode" + TemplateOctalLiteral = "Octal literals are not allowed in template strings." + UnexpectedEOS = "Unexpected end of input" + UnexpectedIdentifier = "Unexpected identifier" + UnexpectedNumber = "Unexpected number" + UnexpectedReserved = "Unexpected reserved word" + UnexpectedString = "Unexpected string" + UnexpectedTemplate = "Unexpected quasi %0" + UnexpectedToken = "Unexpected token %0" + UnexpectedTokenIllegal = "Unexpected token ILLEGAL" + UnknownLabel = "Undefined label '%0'" + UnterminatedRegExp = "Invalid regular expression: missing /" diff --git a/third_party/python/esprima/esprima/nodes.py b/third_party/python/esprima/esprima/nodes.py new file mode 100644 index 000000000000..bbbbdb893b44 --- /dev/null +++ b/third_party/python/esprima/esprima/nodes.py @@ -0,0 +1,620 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .objects import Object +from .syntax import Syntax +from .scanner import RegExp + + +class Node(Object): + def __dir__(self): + return list(self.__dict__.keys()) + + def __iter__(self): + return self.__iter__ + + def keys(self): + return self.__dict__.keys() + + def items(self): + return self.__dict__.items() + + +class ArrayExpression(Node): + def __init__(self, elements): + self.type = Syntax.ArrayExpression + self.elements = elements + + +class ArrayPattern(Node): + def __init__(self, elements): + self.type = Syntax.ArrayPattern + self.elements = elements + + +class ArrowFunctionExpression(Node): + def __init__(self, params, body, expression): + self.type = Syntax.ArrowFunctionExpression + self.generator = False + self.isAsync = False + self.params = params + self.body = body + self.expression = expression + + +class AssignmentExpression(Node): + def __init__(self, operator, left, right): + self.type = Syntax.AssignmentExpression + self.operator = operator + self.left = left + self.right = right + + +class AssignmentPattern(Node): + def __init__(self, left, right): + self.type = Syntax.AssignmentPattern + self.left = left + self.right = right + + +class AsyncArrowFunctionExpression(Node): + def __init__(self, params, body, expression): + self.type = Syntax.ArrowFunctionExpression + self.generator = False + self.isAsync = True + self.params = params + self.body = body + self.expression = expression + + +class AsyncFunctionDeclaration(Node): + def __init__(self, id, params, body): + self.type = Syntax.FunctionDeclaration + self.generator = False + self.expression = False + self.isAsync = True + self.id = id + self.params = params + self.body = body + + +class AsyncFunctionExpression(Node): + def __init__(self, id, params, body): + self.type = Syntax.FunctionExpression + self.generator = False + self.expression = False + self.isAsync = True + self.id = id + self.params = params + self.body = body + + +class AwaitExpression(Node): + def __init__(self, argument): + self.type = Syntax.AwaitExpression + self.argument = argument + + +class BinaryExpression(Node): + def __init__(self, operator, left, right): + self.type = Syntax.LogicalExpression if operator in ('||', '&&') else Syntax.BinaryExpression + self.operator = operator + self.left = left + self.right = right + + +class BlockStatement(Node): + def __init__(self, body): + self.type = Syntax.BlockStatement + self.body = body + + +class BreakStatement(Node): + def __init__(self, label): + self.type = Syntax.BreakStatement + self.label = label + + +class CallExpression(Node): + def __init__(self, callee, args): + self.type = Syntax.CallExpression + self.callee = callee + self.arguments = args + + +class CatchClause(Node): + def __init__(self, param, body): + self.type = Syntax.CatchClause + self.param = param + self.body = body + + +class ClassBody(Node): + def __init__(self, body): + self.type = Syntax.ClassBody + self.body = body + + +class ClassDeclaration(Node): + def __init__(self, id, superClass, body): + self.type = Syntax.ClassDeclaration + self.id = id + self.superClass = superClass + self.body = body + + +class ClassExpression(Node): + def __init__(self, id, superClass, body): + self.type = Syntax.ClassExpression + self.id = id + self.superClass = superClass + self.body = body + + +class ComputedMemberExpression(Node): + def __init__(self, object, property): + self.type = Syntax.MemberExpression + self.computed = True + self.object = object + self.property = property + + +class ConditionalExpression(Node): + def __init__(self, test, consequent, alternate): + self.type = Syntax.ConditionalExpression + self.test = test + self.consequent = consequent + self.alternate = alternate + + +class ContinueStatement(Node): + def __init__(self, label): + self.type = Syntax.ContinueStatement + self.label = label + + +class DebuggerStatement(Node): + def __init__(self): + self.type = Syntax.DebuggerStatement + + +class Directive(Node): + def __init__(self, expression, directive): + self.type = Syntax.ExpressionStatement + self.expression = expression + self.directive = directive + + +class DoWhileStatement(Node): + def __init__(self, body, test): + self.type = Syntax.DoWhileStatement + self.body = body + self.test = test + + +class EmptyStatement(Node): + def __init__(self): + self.type = Syntax.EmptyStatement + + +class ExportAllDeclaration(Node): + def __init__(self, source): + self.type = Syntax.ExportAllDeclaration + self.source = source + + +class ExportDefaultDeclaration(Node): + def __init__(self, declaration): + self.type = Syntax.ExportDefaultDeclaration + self.declaration = declaration + + +class ExportNamedDeclaration(Node): + def __init__(self, declaration, specifiers, source): + self.type = Syntax.ExportNamedDeclaration + self.declaration = declaration + self.specifiers = specifiers + self.source = source + + +class ExportSpecifier(Node): + def __init__(self, local, exported): + self.type = Syntax.ExportSpecifier + self.exported = exported + self.local = local + + +class ExportDefaultSpecifier(Node): + def __init__(self, local): + self.type = Syntax.ExportDefaultSpecifier + self.local = local + + +class ExpressionStatement(Node): + def __init__(self, expression): + self.type = Syntax.ExpressionStatement + self.expression = expression + + +class ForInStatement(Node): + def __init__(self, left, right, body): + self.type = Syntax.ForInStatement + self.each = False + self.left = left + self.right = right + self.body = body + + +class ForOfStatement(Node): + def __init__(self, left, right, body): + self.type = Syntax.ForOfStatement + self.left = left + self.right = right + self.body = body + + +class ForStatement(Node): + def __init__(self, init, test, update, body): + self.type = Syntax.ForStatement + self.init = init + self.test = test + self.update = update + self.body = body + + +class FunctionDeclaration(Node): + def __init__(self, id, params, body, generator): + self.type = Syntax.FunctionDeclaration + self.expression = False + self.isAsync = False + self.id = id + self.params = params + self.body = body + self.generator = generator + + +class FunctionExpression(Node): + def __init__(self, id, params, body, generator): + self.type = Syntax.FunctionExpression + self.expression = False + self.isAsync = False + self.id = id + self.params = params + self.body = body + self.generator = generator + + +class Identifier(Node): + def __init__(self, name): + self.type = Syntax.Identifier + self.name = name + + +class IfStatement(Node): + def __init__(self, test, consequent, alternate): + self.type = Syntax.IfStatement + self.test = test + self.consequent = consequent + self.alternate = alternate + + +class Import(Node): + def __init__(self): + self.type = Syntax.Import + + +class ImportDeclaration(Node): + def __init__(self, specifiers, source): + self.type = Syntax.ImportDeclaration + self.specifiers = specifiers + self.source = source + + +class ImportDefaultSpecifier(Node): + def __init__(self, local): + self.type = Syntax.ImportDefaultSpecifier + self.local = local + + +class ImportNamespaceSpecifier(Node): + def __init__(self, local): + self.type = Syntax.ImportNamespaceSpecifier + self.local = local + + +class ImportSpecifier(Node): + def __init__(self, local, imported): + self.type = Syntax.ImportSpecifier + self.local = local + self.imported = imported + + +class LabeledStatement(Node): + def __init__(self, label, body): + self.type = Syntax.LabeledStatement + self.label = label + self.body = body + + +class Literal(Node): + def __init__(self, value, raw): + self.type = Syntax.Literal + self.value = value + self.raw = raw + + +class MetaProperty(Node): + def __init__(self, meta, property): + self.type = Syntax.MetaProperty + self.meta = meta + self.property = property + + +class MethodDefinition(Node): + def __init__(self, key, computed, value, kind, isStatic): + self.type = Syntax.MethodDefinition + self.key = key + self.computed = computed + self.value = value + self.kind = kind + self.static = isStatic + + +class FieldDefinition(Node): + def __init__(self, key, computed, value, kind, isStatic): + self.type = Syntax.FieldDefinition + self.key = key + self.computed = computed + self.value = value + self.kind = kind + self.static = isStatic + + +class Module(Node): + def __init__(self, body): + self.type = Syntax.Program + self.sourceType = 'module' + self.body = body + + +class NewExpression(Node): + def __init__(self, callee, args): + self.type = Syntax.NewExpression + self.callee = callee + self.arguments = args + + +class ObjectExpression(Node): + def __init__(self, properties): + self.type = Syntax.ObjectExpression + self.properties = properties + + +class ObjectPattern(Node): + def __init__(self, properties): + self.type = Syntax.ObjectPattern + self.properties = properties + + +class Property(Node): + def __init__(self, kind, key, computed, value, method, shorthand): + self.type = Syntax.Property + self.key = key + self.computed = computed + self.value = value + self.kind = kind + self.method = method + self.shorthand = shorthand + + +class RegexLiteral(Node): + def __init__(self, value, raw, pattern, flags): + self.type = Syntax.Literal + self.value = value + self.raw = raw + self.regex = RegExp( + pattern=pattern, + flags=flags, + ) + + +class RestElement(Node): + def __init__(self, argument): + self.type = Syntax.RestElement + self.argument = argument + + +class ReturnStatement(Node): + def __init__(self, argument): + self.type = Syntax.ReturnStatement + self.argument = argument + + +class Script(Node): + def __init__(self, body): + self.type = Syntax.Program + self.sourceType = 'script' + self.body = body + + +class SequenceExpression(Node): + def __init__(self, expressions): + self.type = Syntax.SequenceExpression + self.expressions = expressions + + +class SpreadElement(Node): + def __init__(self, argument): + self.type = Syntax.SpreadElement + self.argument = argument + + +class StaticMemberExpression(Node): + def __init__(self, object, property): + self.type = Syntax.MemberExpression + self.computed = False + self.object = object + self.property = property + + +class Super(Node): + def __init__(self): + self.type = Syntax.Super + + +class SwitchCase(Node): + def __init__(self, test, consequent): + self.type = Syntax.SwitchCase + self.test = test + self.consequent = consequent + + +class SwitchStatement(Node): + def __init__(self, discriminant, cases): + self.type = Syntax.SwitchStatement + self.discriminant = discriminant + self.cases = cases + + +class TaggedTemplateExpression(Node): + def __init__(self, tag, quasi): + self.type = Syntax.TaggedTemplateExpression + self.tag = tag + self.quasi = quasi + + +class TemplateElement(Node): + class Value(Object): + def __init__(self, raw, cooked): + self.raw = raw + self.cooked = cooked + + def __init__(self, raw, cooked, tail): + self.type = Syntax.TemplateElement + self.value = TemplateElement.Value(raw, cooked) + self.tail = tail + + +class TemplateLiteral(Node): + def __init__(self, quasis, expressions): + self.type = Syntax.TemplateLiteral + self.quasis = quasis + self.expressions = expressions + + +class ThisExpression(Node): + def __init__(self): + self.type = Syntax.ThisExpression + + +class ThrowStatement(Node): + def __init__(self, argument): + self.type = Syntax.ThrowStatement + self.argument = argument + + +class TryStatement(Node): + def __init__(self, block, handler, finalizer): + self.type = Syntax.TryStatement + self.block = block + self.handler = handler + self.finalizer = finalizer + + +class UnaryExpression(Node): + def __init__(self, operator, argument): + self.type = Syntax.UnaryExpression + self.prefix = True + self.operator = operator + self.argument = argument + + +class UpdateExpression(Node): + def __init__(self, operator, argument, prefix): + self.type = Syntax.UpdateExpression + self.operator = operator + self.argument = argument + self.prefix = prefix + + +class VariableDeclaration(Node): + def __init__(self, declarations, kind): + self.type = Syntax.VariableDeclaration + self.declarations = declarations + self.kind = kind + + +class VariableDeclarator(Node): + def __init__(self, id, init): + self.type = Syntax.VariableDeclarator + self.id = id + self.init = init + + +class WhileStatement(Node): + def __init__(self, test, body): + self.type = Syntax.WhileStatement + self.test = test + self.body = body + + +class WithStatement(Node): + def __init__(self, object, body): + self.type = Syntax.WithStatement + self.object = object + self.body = body + + +class YieldExpression(Node): + def __init__(self, argument, delegate): + self.type = Syntax.YieldExpression + self.argument = argument + self.delegate = delegate + + +class ArrowParameterPlaceHolder(Node): + def __init__(self, params): + self.type = Syntax.ArrowParameterPlaceHolder + self.params = params + self.isAsync = False + + +class AsyncArrowParameterPlaceHolder(Node): + def __init__(self, params): + self.type = Syntax.ArrowParameterPlaceHolder + self.params = params + self.isAsync = True + + +class BlockComment(Node): + def __init__(self, value): + self.type = Syntax.BlockComment + self.value = value + + +class LineComment(Node): + def __init__(self, value): + self.type = Syntax.LineComment + self.value = value diff --git a/third_party/python/esprima/esprima/objects.py b/third_party/python/esprima/esprima/objects.py new file mode 100644 index 000000000000..a8acca1b63a3 --- /dev/null +++ b/third_party/python/esprima/esprima/objects.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + + +def toDict(value): + from .visitor import ToDictVisitor + return ToDictVisitor().visit(value) + + +class Array(list): + pass + + +class Object(object): + def toDict(self): + from .visitor import ToDictVisitor + return ToDictVisitor().visit(self) + + def __repr__(self): + from .visitor import ReprVisitor + return ReprVisitor().visit(self) + + def __getattr__(self, name): + return None diff --git a/third_party/python/esprima/esprima/parser.py b/third_party/python/esprima/esprima/parser.py new file mode 100644 index 000000000000..2309e7b6fb8f --- /dev/null +++ b/third_party/python/esprima/esprima/parser.py @@ -0,0 +1,3104 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +from .objects import Object +from .compat import basestring, unicode +from .utils import format +from .error_handler import ErrorHandler +from .messages import Messages +from .scanner import RawToken, Scanner, SourceLocation, Position, RegExp +from .token import Token, TokenName +from .syntax import Syntax +from . import nodes as Node + + +class Value(object): + def __init__(self, value): + self.value = value + + +class Params(object): + def __init__(self, simple=None, message=None, stricted=None, firstRestricted=None, inFor=None, paramSet=None, params=None, get=None): + self.simple = simple + self.message = message + self.stricted = stricted + self.firstRestricted = firstRestricted + self.inFor = inFor + self.paramSet = paramSet + self.params = params + self.get = get + + +class Config(Object): + def __init__(self, range=False, loc=False, source=None, tokens=False, comment=False, tolerant=False, **options): + self.range = range + self.loc = loc + self.source = source + self.tokens = tokens + self.comment = comment + self.tolerant = tolerant + for k, v in options.items(): + setattr(self, k, v) + + +class Context(object): + def __init__(self, isModule=False, allowAwait=False, allowIn=True, allowStrictDirective=True, allowYield=True, firstCoverInitializedNameError=None, isAssignmentTarget=False, isBindingElement=False, inFunctionBody=False, inIteration=False, inSwitch=False, labelSet=None, strict=False): + self.isModule = isModule + self.allowAwait = allowAwait + self.allowIn = allowIn + self.allowStrictDirective = allowStrictDirective + self.allowYield = allowYield + self.firstCoverInitializedNameError = firstCoverInitializedNameError + self.isAssignmentTarget = isAssignmentTarget + self.isBindingElement = isBindingElement + self.inFunctionBody = inFunctionBody + self.inIteration = inIteration + self.inSwitch = inSwitch + self.labelSet = {} if labelSet is None else labelSet + self.strict = strict + + +class Marker(object): + def __init__(self, index=None, line=None, column=None): + self.index = index + self.line = line + self.column = column + + +class TokenEntry(Object): + def __init__(self, type=None, value=None, regex=None, range=None, loc=None): + self.type = type + self.value = value + self.regex = regex + self.range = range + self.loc = loc + + +class Parser(object): + def __init__(self, code, options={}, delegate=None): + self.config = Config(**options) + + self.delegate = delegate + + self.errorHandler = ErrorHandler() + self.errorHandler.tolerant = self.config.tolerant + self.scanner = Scanner(code, self.errorHandler) + self.scanner.trackComment = self.config.comment + + self.operatorPrecedence = { + '||': 1, + '&&': 2, + '|': 3, + '^': 4, + '&': 5, + '==': 6, + '!=': 6, + '===': 6, + '!==': 6, + '<': 7, + '>': 7, + '<=': 7, + '>=': 7, + 'instanceof': 7, + 'in': 7, + '<<': 8, + '>>': 8, + '>>>': 8, + '+': 9, + '-': 9, + '*': 11, + '/': 11, + '%': 11, + } + + self.lookahead = RawToken( + type=Token.EOF, + value='', + lineNumber=self.scanner.lineNumber, + lineStart=0, + start=0, + end=0 + ) + self.hasLineTerminator = False + + self.context = Context( + isModule=False, + allowAwait=False, + allowIn=True, + allowStrictDirective=True, + allowYield=True, + firstCoverInitializedNameError=None, + isAssignmentTarget=False, + isBindingElement=False, + inFunctionBody=False, + inIteration=False, + inSwitch=False, + labelSet={}, + strict=False + ) + self.tokens = [] + + self.startMarker = Marker( + index=0, + line=self.scanner.lineNumber, + column=0 + ) + self.lastMarker = Marker( + index=0, + line=self.scanner.lineNumber, + column=0 + ) + self.nextToken() + self.lastMarker = Marker( + index=self.scanner.index, + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart + ) + + def throwError(self, messageFormat, *args): + msg = format(messageFormat, *args) + index = self.lastMarker.index + line = self.lastMarker.line + column = self.lastMarker.column + 1 + raise self.errorHandler.createError(index, line, column, msg) + + def tolerateError(self, messageFormat, *args): + msg = format(messageFormat, *args) + index = self.lastMarker.index + line = self.scanner.lineNumber + column = self.lastMarker.column + 1 + self.errorHandler.tolerateError(index, line, column, msg) + + # Throw an exception because of the token. + + def unexpectedTokenError(self, token=None, message=None): + msg = message or Messages.UnexpectedToken + if token: + if not message: + typ = token.type + if typ is Token.EOF: + msg = Messages.UnexpectedEOS + elif typ is Token.Identifier: + msg = Messages.UnexpectedIdentifier + elif typ is Token.NumericLiteral: + msg = Messages.UnexpectedNumber + elif typ is Token.StringLiteral: + msg = Messages.UnexpectedString + elif typ is Token.Template: + msg = Messages.UnexpectedTemplate + elif typ is Token.Keyword: + if self.scanner.isFutureReservedWord(token.value): + msg = Messages.UnexpectedReserved + elif self.context.strict and self.scanner.isStrictModeReservedWord(token.value): + msg = Messages.StrictReservedWord + else: + msg = Messages.UnexpectedToken + value = token.value + else: + value = 'ILLEGAL' + + msg = msg.replace('%0', unicode(value), 1) + + if token and isinstance(token.lineNumber, int): + index = token.start + line = token.lineNumber + lastMarkerLineStart = self.lastMarker.index - self.lastMarker.column + column = token.start - lastMarkerLineStart + 1 + return self.errorHandler.createError(index, line, column, msg) + else: + index = self.lastMarker.index + line = self.lastMarker.line + column = self.lastMarker.column + 1 + return self.errorHandler.createError(index, line, column, msg) + + def throwUnexpectedToken(self, token=None, message=None): + raise self.unexpectedTokenError(token, message) + + def tolerateUnexpectedToken(self, token=None, message=None): + self.errorHandler.tolerate(self.unexpectedTokenError(token, message)) + + def collectComments(self): + if not self.config.comment: + self.scanner.scanComments() + else: + comments = self.scanner.scanComments() + if comments: + for e in comments: + if e.multiLine: + node = Node.BlockComment(self.scanner.source[e.slice[0]:e.slice[1]]) + else: + node = Node.LineComment(self.scanner.source[e.slice[0]:e.slice[1]]) + if self.config.range: + node.range = e.range + if self.config.loc: + node.loc = e.loc + if self.delegate: + metadata = SourceLocation( + start=Position( + line=e.loc.start.line, + column=e.loc.start.column, + offset=e.range[0], + ), + end=Position( + line=e.loc.end.line, + column=e.loc.end.column, + offset=e.range[1], + ) + ) + new_node = self.delegate(node, metadata) + if new_node is not None: + node = new_node + + # From internal representation to an external structure + + def getTokenRaw(self, token): + return self.scanner.source[token.start:token.end] + + def convertToken(self, token): + t = TokenEntry( + type=TokenName[token.type], + value=self.getTokenRaw(token), + ) + if self.config.range: + t.range = [token.start, token.end] + if self.config.loc: + t.loc = SourceLocation( + start=Position( + line=self.startMarker.line, + column=self.startMarker.column, + ), + end=Position( + line=self.scanner.lineNumber, + column=self.scanner.index - self.scanner.lineStart, + ), + ) + if token.type is Token.RegularExpression: + t.regex = RegExp( + pattern=token.pattern, + flags=token.flags, + ) + + return t + + def nextToken(self): + token = self.lookahead + + self.lastMarker.index = self.scanner.index + self.lastMarker.line = self.scanner.lineNumber + self.lastMarker.column = self.scanner.index - self.scanner.lineStart + + self.collectComments() + + if self.scanner.index != self.startMarker.index: + self.startMarker.index = self.scanner.index + self.startMarker.line = self.scanner.lineNumber + self.startMarker.column = self.scanner.index - self.scanner.lineStart + + next = self.scanner.lex() + self.hasLineTerminator = token.lineNumber != next.lineNumber + + if next and self.context.strict and next.type is Token.Identifier: + if self.scanner.isStrictModeReservedWord(next.value): + next.type = Token.Keyword + self.lookahead = next + + if self.config.tokens and next.type is not Token.EOF: + self.tokens.append(self.convertToken(next)) + + return token + + def nextRegexToken(self): + self.collectComments() + + token = self.scanner.scanRegExp() + if self.config.tokens: + # Pop the previous token, '/' or '/=' + # self is added from the lookahead token. + self.tokens.pop() + + self.tokens.append(self.convertToken(token)) + + # Prime the next lookahead. + self.lookahead = token + self.nextToken() + + return token + + def createNode(self): + return Marker( + index=self.startMarker.index, + line=self.startMarker.line, + column=self.startMarker.column, + ) + + def startNode(self, token, lastLineStart=0): + column = token.start - token.lineStart + line = token.lineNumber + if column < 0: + column += lastLineStart + line -= 1 + + return Marker( + index=token.start, + line=line, + column=column, + ) + + def finalize(self, marker, node): + if self.config.range: + node.range = [marker.index, self.lastMarker.index] + + if self.config.loc: + node.loc = SourceLocation( + start=Position( + line=marker.line, + column=marker.column, + ), + end=Position( + line=self.lastMarker.line, + column=self.lastMarker.column, + ), + ) + if self.config.source: + node.loc.source = self.config.source + + if self.delegate: + metadata = SourceLocation( + start=Position( + line=marker.line, + column=marker.column, + offset=marker.index, + ), + end=Position( + line=self.lastMarker.line, + column=self.lastMarker.column, + offset=self.lastMarker.index, + ) + ) + new_node = self.delegate(node, metadata) + if new_node is not None: + node = new_node + + return node + + # Expect the next token to match the specified punctuator. + # If not, an exception will be thrown. + + def expect(self, value): + token = self.nextToken() + if token.type is not Token.Punctuator or token.value != value: + self.throwUnexpectedToken(token) + + # Quietly expect a comma when in tolerant mode, otherwise delegates to expect(). + + def expectCommaSeparator(self): + if self.config.tolerant: + token = self.lookahead + if token.type is Token.Punctuator and token.value == ',': + self.nextToken() + elif token.type is Token.Punctuator and token.value == ';': + self.nextToken() + self.tolerateUnexpectedToken(token) + else: + self.tolerateUnexpectedToken(token, Messages.UnexpectedToken) + else: + self.expect(',') + + # Expect the next token to match the specified keyword. + # If not, an exception will be thrown. + + def expectKeyword(self, keyword): + token = self.nextToken() + if token.type is not Token.Keyword or token.value != keyword: + self.throwUnexpectedToken(token) + + # Return true if the next token matches the specified punctuator. + + def match(self, *value): + return self.lookahead.type is Token.Punctuator and self.lookahead.value in value + + # Return true if the next token matches the specified keyword + + def matchKeyword(self, *keyword): + return self.lookahead.type is Token.Keyword and self.lookahead.value in keyword + + # Return true if the next token matches the specified contextual keyword + # (where an identifier is sometimes a keyword depending on the context) + + def matchContextualKeyword(self, *keyword): + return self.lookahead.type is Token.Identifier and self.lookahead.value in keyword + + # Return true if the next token is an assignment operator + + def matchAssign(self): + if self.lookahead.type is not Token.Punctuator: + return False + + op = self.lookahead.value + return op in ('=', '*=', '**=', '/=', '%=', '+=', '-=', '<<=', '>>=', '>>>=', '&=', '^=', '|=') + + # Cover grammar support. + # + # When an assignment expression position starts with an left parenthesis, the determination of the type + # of the syntax is to be deferred arbitrarily long until the end of the parentheses pair (plus a lookahead) + # or the first comma. This situation also defers the determination of all the expressions nested in the pair. + # + # There are three productions that can be parsed in a parentheses pair that needs to be determined + # after the outermost pair is closed. They are: + # + # 1. AssignmentExpression + # 2. BindingElements + # 3. AssignmentTargets + # + # In order to avoid exponential backtracking, we use two flags to denote if the production can be + # binding element or assignment target. + # + # The three productions have the relationship: + # + # BindingElements ⊆ AssignmentTargets ⊆ AssignmentExpression + # + # with a single exception that CoverInitializedName when used directly in an Expression, generates + # an early error. Therefore, we need the third state, firstCoverInitializedNameError, to track the + # first usage of CoverInitializedName and report it when we reached the end of the parentheses pair. + # + # isolateCoverGrammar function runs the given parser function with a new cover grammar context, and it does not + # effect the current flags. This means the production the parser parses is only used as an expression. Therefore + # the CoverInitializedName check is conducted. + # + # inheritCoverGrammar function runs the given parse function with a new cover grammar context, and it propagates + # the flags outside of the parser. This means the production the parser parses is used as a part of a potential + # pattern. The CoverInitializedName check is deferred. + + def isolateCoverGrammar(self, parseFunction): + previousIsBindingElement = self.context.isBindingElement + previousIsAssignmentTarget = self.context.isAssignmentTarget + previousFirstCoverInitializedNameError = self.context.firstCoverInitializedNameError + + self.context.isBindingElement = True + self.context.isAssignmentTarget = True + self.context.firstCoverInitializedNameError = None + + result = parseFunction() + if self.context.firstCoverInitializedNameError is not None: + self.throwUnexpectedToken(self.context.firstCoverInitializedNameError) + + self.context.isBindingElement = previousIsBindingElement + self.context.isAssignmentTarget = previousIsAssignmentTarget + self.context.firstCoverInitializedNameError = previousFirstCoverInitializedNameError + + return result + + def inheritCoverGrammar(self, parseFunction): + previousIsBindingElement = self.context.isBindingElement + previousIsAssignmentTarget = self.context.isAssignmentTarget + previousFirstCoverInitializedNameError = self.context.firstCoverInitializedNameError + + self.context.isBindingElement = True + self.context.isAssignmentTarget = True + self.context.firstCoverInitializedNameError = None + + result = parseFunction() + + self.context.isBindingElement = self.context.isBindingElement and previousIsBindingElement + self.context.isAssignmentTarget = self.context.isAssignmentTarget and previousIsAssignmentTarget + self.context.firstCoverInitializedNameError = previousFirstCoverInitializedNameError or self.context.firstCoverInitializedNameError + + return result + + def consumeSemicolon(self): + if self.match(';'): + self.nextToken() + elif not self.hasLineTerminator: + if self.lookahead.type is not Token.EOF and not self.match('}'): + self.throwUnexpectedToken(self.lookahead) + self.lastMarker.index = self.startMarker.index + self.lastMarker.line = self.startMarker.line + self.lastMarker.column = self.startMarker.column + + # https://tc39.github.io/ecma262/#sec-primary-expression + + def parsePrimaryExpression(self): + node = self.createNode() + + typ = self.lookahead.type + if typ is Token.Identifier: + if (self.context.isModule or self.context.allowAwait) and self.lookahead.value == 'await': + self.tolerateUnexpectedToken(self.lookahead) + expr = self.parseFunctionExpression() if self.matchAsyncFunction() else self.finalize(node, Node.Identifier(self.nextToken().value)) + + elif typ in ( + Token.NumericLiteral, + Token.StringLiteral, + ): + if self.context.strict and self.lookahead.octal: + self.tolerateUnexpectedToken(self.lookahead, Messages.StrictOctalLiteral) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + token = self.nextToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.Literal(token.value, raw)) + + elif typ is Token.BooleanLiteral: + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + token = self.nextToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.Literal(token.value == 'true', raw)) + + elif typ is Token.NullLiteral: + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + token = self.nextToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.Literal(None, raw)) + + elif typ is Token.Template: + expr = self.parseTemplateLiteral() + + elif typ is Token.Punctuator: + value = self.lookahead.value + if value == '(': + self.context.isBindingElement = False + expr = self.inheritCoverGrammar(self.parseGroupExpression) + elif value == '[': + expr = self.inheritCoverGrammar(self.parseArrayInitializer) + elif value == '{': + expr = self.inheritCoverGrammar(self.parseObjectInitializer) + elif value in ('/', '/='): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + self.scanner.index = self.startMarker.index + token = self.nextRegexToken() + raw = self.getTokenRaw(token) + expr = self.finalize(node, Node.RegexLiteral(token.regex, raw, token.pattern, token.flags)) + else: + expr = self.throwUnexpectedToken(self.nextToken()) + + elif typ is Token.Keyword: + if not self.context.strict and self.context.allowYield and self.matchKeyword('yield'): + expr = self.parseIdentifierName() + elif not self.context.strict and self.matchKeyword('let'): + expr = self.finalize(node, Node.Identifier(self.nextToken().value)) + else: + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + if self.matchKeyword('function'): + expr = self.parseFunctionExpression() + elif self.matchKeyword('this'): + self.nextToken() + expr = self.finalize(node, Node.ThisExpression()) + elif self.matchKeyword('class'): + expr = self.parseClassExpression() + elif self.matchImportCall(): + expr = self.parseImportCall() + else: + expr = self.throwUnexpectedToken(self.nextToken()) + + else: + expr = self.throwUnexpectedToken(self.nextToken()) + + return expr + + # https://tc39.github.io/ecma262/#sec-array-initializer + + def parseSpreadElement(self): + node = self.createNode() + self.expect('...') + arg = self.inheritCoverGrammar(self.parseAssignmentExpression) + return self.finalize(node, Node.SpreadElement(arg)) + + def parseArrayInitializer(self): + node = self.createNode() + elements = [] + + self.expect('[') + while not self.match(']'): + if self.match(','): + self.nextToken() + elements.append(None) + elif self.match('...'): + element = self.parseSpreadElement() + if not self.match(']'): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + self.expect(',') + elements.append(element) + else: + elements.append(self.inheritCoverGrammar(self.parseAssignmentExpression)) + if not self.match(']'): + self.expect(',') + self.expect(']') + + return self.finalize(node, Node.ArrayExpression(elements)) + + # https://tc39.github.io/ecma262/#sec-object-initializer + + def parsePropertyMethod(self, params): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = params.simple + body = self.isolateCoverGrammar(self.parseFunctionSourceElements) + if self.context.strict and params.firstRestricted: + self.tolerateUnexpectedToken(params.firstRestricted, params.message) + if self.context.strict and params.stricted: + self.tolerateUnexpectedToken(params.stricted, params.message) + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + + return body + + def parsePropertyMethodFunction(self): + isGenerator = False + node = self.createNode() + + previousAllowYield = self.context.allowYield + self.context.allowYield = True + params = self.parseFormalParameters() + method = self.parsePropertyMethod(params) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, params.params, method, isGenerator)) + + def parsePropertyMethodAsyncFunction(self): + node = self.createNode() + + previousAllowYield = self.context.allowYield + previousAwait = self.context.allowAwait + self.context.allowYield = False + self.context.allowAwait = True + params = self.parseFormalParameters() + method = self.parsePropertyMethod(params) + self.context.allowYield = previousAllowYield + self.context.allowAwait = previousAwait + + return self.finalize(node, Node.AsyncFunctionExpression(None, params.params, method)) + + def parseObjectPropertyKey(self): + node = self.createNode() + token = self.nextToken() + + typ = token.type + if typ in ( + Token.StringLiteral, + Token.NumericLiteral, + ): + if self.context.strict and token.octal: + self.tolerateUnexpectedToken(token, Messages.StrictOctalLiteral) + raw = self.getTokenRaw(token) + key = self.finalize(node, Node.Literal(token.value, raw)) + + elif typ in ( + Token.Identifier, + Token.BooleanLiteral, + Token.NullLiteral, + Token.Keyword, + ): + key = self.finalize(node, Node.Identifier(token.value)) + + elif typ is Token.Punctuator: + if token.value == '[': + key = self.isolateCoverGrammar(self.parseAssignmentExpression) + self.expect(']') + else: + key = self.throwUnexpectedToken(token) + + else: + key = self.throwUnexpectedToken(token) + + return key + + def isPropertyKey(self, key, value): + return ( + (key.type is Syntax.Identifier and key.name == value) or + (key.type is Syntax.Literal and key.value == value) + ) + + def parseObjectProperty(self, hasProto): + node = self.createNode() + token = self.lookahead + + key = None + value = None + + computed = False + method = False + shorthand = False + isAsync = False + + if token.type is Token.Identifier: + id = token.value + self.nextToken() + computed = self.match('[') + isAsync = not self.hasLineTerminator and (id == 'async') and not (self.match(':', '(', '*', ',')) + key = self.parseObjectPropertyKey() if isAsync else self.finalize(node, Node.Identifier(id)) + elif self.match('*'): + self.nextToken() + else: + computed = self.match('[') + key = self.parseObjectPropertyKey() + + lookaheadPropertyKey = self.qualifiedPropertyName(self.lookahead) + if token.type is Token.Identifier and not isAsync and token.value == 'get' and lookaheadPropertyKey: + kind = 'get' + computed = self.match('[') + key = self.parseObjectPropertyKey() + self.context.allowYield = False + value = self.parseGetterMethod() + + elif token.type is Token.Identifier and not isAsync and token.value == 'set' and lookaheadPropertyKey: + kind = 'set' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseSetterMethod() + + elif token.type is Token.Punctuator and token.value == '*' and lookaheadPropertyKey: + kind = 'init' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseGeneratorMethod() + method = True + + else: + if not key: + self.throwUnexpectedToken(self.lookahead) + + kind = 'init' + if self.match(':') and not isAsync: + if not computed and self.isPropertyKey(key, '__proto__'): + if hasProto.value: + self.tolerateError(Messages.DuplicateProtoProperty) + hasProto.value = True + self.nextToken() + value = self.inheritCoverGrammar(self.parseAssignmentExpression) + + elif self.match('('): + value = self.parsePropertyMethodAsyncFunction() if isAsync else self.parsePropertyMethodFunction() + method = True + + elif token.type is Token.Identifier: + id = self.finalize(node, Node.Identifier(token.value)) + if self.match('='): + self.context.firstCoverInitializedNameError = self.lookahead + self.nextToken() + shorthand = True + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + value = self.finalize(node, Node.AssignmentPattern(id, init)) + else: + shorthand = True + value = id + else: + self.throwUnexpectedToken(self.nextToken()) + + return self.finalize(node, Node.Property(kind, key, computed, value, method, shorthand)) + + def parseObjectInitializer(self): + node = self.createNode() + + self.expect('{') + properties = [] + hasProto = Value(False) + while not self.match('}'): + properties.append(self.parseSpreadElement() if self.match('...') else self.parseObjectProperty(hasProto)) + if not self.match('}'): + self.expectCommaSeparator() + self.expect('}') + + return self.finalize(node, Node.ObjectExpression(properties)) + + # https://tc39.github.io/ecma262/#sec-template-literals + + def parseTemplateHead(self): + assert self.lookahead.head, 'Template literal must start with a template head' + + node = self.createNode() + token = self.nextToken() + raw = token.value + cooked = token.cooked + + return self.finalize(node, Node.TemplateElement(raw, cooked, token.tail)) + + def parseTemplateElement(self): + if self.lookahead.type is not Token.Template: + self.throwUnexpectedToken() + + node = self.createNode() + token = self.nextToken() + raw = token.value + cooked = token.cooked + + return self.finalize(node, Node.TemplateElement(raw, cooked, token.tail)) + + def parseTemplateLiteral(self): + node = self.createNode() + + expressions = [] + quasis = [] + + quasi = self.parseTemplateHead() + quasis.append(quasi) + while not quasi.tail: + expressions.append(self.parseExpression()) + quasi = self.parseTemplateElement() + quasis.append(quasi) + + return self.finalize(node, Node.TemplateLiteral(quasis, expressions)) + + # https://tc39.github.io/ecma262/#sec-grouping-operator + + def reinterpretExpressionAsPattern(self, expr): + typ = expr.type + if typ in ( + Syntax.Identifier, + Syntax.MemberExpression, + Syntax.RestElement, + Syntax.AssignmentPattern, + ): + pass + elif typ is Syntax.SpreadElement: + expr.type = Syntax.RestElement + self.reinterpretExpressionAsPattern(expr.argument) + elif typ is Syntax.ArrayExpression: + expr.type = Syntax.ArrayPattern + for elem in expr.elements: + if elem is not None: + self.reinterpretExpressionAsPattern(elem) + elif typ is Syntax.ObjectExpression: + expr.type = Syntax.ObjectPattern + for prop in expr.properties: + self.reinterpretExpressionAsPattern(prop if prop.type is Syntax.SpreadElement else prop.value) + elif typ is Syntax.AssignmentExpression: + expr.type = Syntax.AssignmentPattern + del expr.operator + self.reinterpretExpressionAsPattern(expr.left) + else: + # Allow other node type for tolerant parsing. + pass + + def parseGroupExpression(self): + self.expect('(') + if self.match(')'): + self.nextToken() + if not self.match('=>'): + self.expect('=>') + expr = Node.ArrowParameterPlaceHolder([]) + else: + startToken = self.lookahead + params = [] + if self.match('...'): + expr = self.parseRestElement(params) + self.expect(')') + if not self.match('=>'): + self.expect('=>') + expr = Node.ArrowParameterPlaceHolder([expr]) + else: + arrow = False + self.context.isBindingElement = True + expr = self.inheritCoverGrammar(self.parseAssignmentExpression) + + if self.match(','): + expressions = [] + + self.context.isAssignmentTarget = False + expressions.append(expr) + while self.lookahead.type is not Token.EOF: + if not self.match(','): + break + self.nextToken() + if self.match(')'): + self.nextToken() + for expression in expressions: + self.reinterpretExpressionAsPattern(expression) + arrow = True + expr = Node.ArrowParameterPlaceHolder(expressions) + elif self.match('...'): + if not self.context.isBindingElement: + self.throwUnexpectedToken(self.lookahead) + expressions.append(self.parseRestElement(params)) + self.expect(')') + if not self.match('=>'): + self.expect('=>') + self.context.isBindingElement = False + for expression in expressions: + self.reinterpretExpressionAsPattern(expression) + arrow = True + expr = Node.ArrowParameterPlaceHolder(expressions) + else: + expressions.append(self.inheritCoverGrammar(self.parseAssignmentExpression)) + if arrow: + break + if not arrow: + expr = self.finalize(self.startNode(startToken), Node.SequenceExpression(expressions)) + + if not arrow: + self.expect(')') + if self.match('=>'): + if expr.type is Syntax.Identifier and expr.name == 'yield': + arrow = True + expr = Node.ArrowParameterPlaceHolder([expr]) + if not arrow: + if not self.context.isBindingElement: + self.throwUnexpectedToken(self.lookahead) + + if expr.type is Syntax.SequenceExpression: + for expression in expr.expressions: + self.reinterpretExpressionAsPattern(expression) + else: + self.reinterpretExpressionAsPattern(expr) + + if expr.type is Syntax.SequenceExpression: + parameters = expr.expressions + else: + parameters = [expr] + expr = Node.ArrowParameterPlaceHolder(parameters) + self.context.isBindingElement = False + + return expr + + # https://tc39.github.io/ecma262/#sec-left-hand-side-expressions + + def parseArguments(self): + self.expect('(') + args = [] + if not self.match(')'): + while True: + if self.match('...'): + expr = self.parseSpreadElement() + else: + expr = self.isolateCoverGrammar(self.parseAssignmentExpression) + args.append(expr) + if self.match(')'): + break + self.expectCommaSeparator() + if self.match(')'): + break + self.expect(')') + + return args + + def isIdentifierName(self, token): + return ( + token.type is Token.Identifier or + token.type is Token.Keyword or + token.type is Token.BooleanLiteral or + token.type is Token.NullLiteral + ) + + def parseIdentifierName(self): + node = self.createNode() + token = self.nextToken() + if not self.isIdentifierName(token): + self.throwUnexpectedToken(token) + return self.finalize(node, Node.Identifier(token.value)) + + def parseNewExpression(self): + node = self.createNode() + + id = self.parseIdentifierName() + assert id.name == 'new', 'New expression must start with `new`' + + if self.match('.'): + self.nextToken() + if self.lookahead.type is Token.Identifier and self.context.inFunctionBody and self.lookahead.value == 'target': + property = self.parseIdentifierName() + expr = Node.MetaProperty(id, property) + else: + self.throwUnexpectedToken(self.lookahead) + elif self.matchKeyword('import'): + self.throwUnexpectedToken(self.lookahead) + else: + callee = self.isolateCoverGrammar(self.parseLeftHandSideExpression) + args = self.parseArguments() if self.match('(') else [] + expr = Node.NewExpression(callee, args) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + return self.finalize(node, expr) + + def parseAsyncArgument(self): + arg = self.parseAssignmentExpression() + self.context.firstCoverInitializedNameError = None + return arg + + def parseAsyncArguments(self): + self.expect('(') + args = [] + if not self.match(')'): + while True: + if self.match('...'): + expr = self.parseSpreadElement() + else: + expr = self.isolateCoverGrammar(self.parseAsyncArgument) + args.append(expr) + if self.match(')'): + break + self.expectCommaSeparator() + if self.match(')'): + break + self.expect(')') + + return args + + def matchImportCall(self): + match = self.matchKeyword('import') + if match: + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.scanner.lex() + self.scanner.restoreState(state) + match = (next.type is Token.Punctuator) and (next.value == '(') + + return match + + def parseImportCall(self): + node = self.createNode() + self.expectKeyword('import') + return self.finalize(node, Node.Import()) + + def parseLeftHandSideExpressionAllowCall(self): + startToken = self.lookahead + maybeAsync = self.matchContextualKeyword('async') + + previousAllowIn = self.context.allowIn + self.context.allowIn = True + + if self.matchKeyword('super') and self.context.inFunctionBody: + expr = self.createNode() + self.nextToken() + expr = self.finalize(expr, Node.Super()) + if not self.match('(') and not self.match('.') and not self.match('['): + self.throwUnexpectedToken(self.lookahead) + else: + expr = self.inheritCoverGrammar(self.parseNewExpression if self.matchKeyword('new') else self.parsePrimaryExpression) + + while True: + if self.match('.'): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('.') + property = self.parseIdentifierName() + expr = self.finalize(self.startNode(startToken), Node.StaticMemberExpression(expr, property)) + + elif self.match('('): + asyncArrow = maybeAsync and (startToken.lineNumber == self.lookahead.lineNumber) + self.context.isBindingElement = False + self.context.isAssignmentTarget = False + if asyncArrow: + args = self.parseAsyncArguments() + else: + args = self.parseArguments() + if expr.type is Syntax.Import and len(args) != 1: + self.tolerateError(Messages.BadImportCallArity) + expr = self.finalize(self.startNode(startToken), Node.CallExpression(expr, args)) + if asyncArrow and self.match('=>'): + for arg in args: + self.reinterpretExpressionAsPattern(arg) + expr = Node.AsyncArrowParameterPlaceHolder(args) + elif self.match('['): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('[') + property = self.isolateCoverGrammar(self.parseExpression) + self.expect(']') + expr = self.finalize(self.startNode(startToken), Node.ComputedMemberExpression(expr, property)) + + elif self.lookahead.type is Token.Template and self.lookahead.head: + quasi = self.parseTemplateLiteral() + expr = self.finalize(self.startNode(startToken), Node.TaggedTemplateExpression(expr, quasi)) + + else: + break + + self.context.allowIn = previousAllowIn + + return expr + + def parseSuper(self): + node = self.createNode() + + self.expectKeyword('super') + if not self.match('[') and not self.match('.'): + self.throwUnexpectedToken(self.lookahead) + + return self.finalize(node, Node.Super()) + + def parseLeftHandSideExpression(self): + assert self.context.allowIn, 'callee of new expression always allow in keyword.' + + node = self.startNode(self.lookahead) + if self.matchKeyword('super') and self.context.inFunctionBody: + expr = self.parseSuper() + else: + expr = self.inheritCoverGrammar(self.parseNewExpression if self.matchKeyword('new') else self.parsePrimaryExpression) + + while True: + if self.match('['): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('[') + property = self.isolateCoverGrammar(self.parseExpression) + self.expect(']') + expr = self.finalize(node, Node.ComputedMemberExpression(expr, property)) + + elif self.match('.'): + self.context.isBindingElement = False + self.context.isAssignmentTarget = True + self.expect('.') + property = self.parseIdentifierName() + expr = self.finalize(node, Node.StaticMemberExpression(expr, property)) + + elif self.lookahead.type is Token.Template and self.lookahead.head: + quasi = self.parseTemplateLiteral() + expr = self.finalize(node, Node.TaggedTemplateExpression(expr, quasi)) + + else: + break + + return expr + + # https://tc39.github.io/ecma262/#sec-update-expressions + + def parseUpdateExpression(self): + startToken = self.lookahead + + if self.match('++', '--'): + node = self.startNode(startToken) + token = self.nextToken() + expr = self.inheritCoverGrammar(self.parseUnaryExpression) + if self.context.strict and expr.type is Syntax.Identifier and self.scanner.isRestrictedWord(expr.name): + self.tolerateError(Messages.StrictLHSPrefix) + if not self.context.isAssignmentTarget: + self.tolerateError(Messages.InvalidLHSInAssignment) + prefix = True + expr = self.finalize(node, Node.UpdateExpression(token.value, expr, prefix)) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + else: + expr = self.inheritCoverGrammar(self.parseLeftHandSideExpressionAllowCall) + if not self.hasLineTerminator and self.lookahead.type is Token.Punctuator: + if self.match('++', '--'): + if self.context.strict and expr.type is Syntax.Identifier and self.scanner.isRestrictedWord(expr.name): + self.tolerateError(Messages.StrictLHSPostfix) + if not self.context.isAssignmentTarget: + self.tolerateError(Messages.InvalidLHSInAssignment) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + operator = self.nextToken().value + prefix = False + expr = self.finalize(self.startNode(startToken), Node.UpdateExpression(operator, expr, prefix)) + + return expr + + # https://tc39.github.io/ecma262/#sec-unary-operators + + def parseAwaitExpression(self): + node = self.createNode() + self.nextToken() + argument = self.parseUnaryExpression() + return self.finalize(node, Node.AwaitExpression(argument)) + + def parseUnaryExpression(self): + if ( + self.match('+', '-', '~', '!') or + self.matchKeyword('delete', 'void', 'typeof') + ): + node = self.startNode(self.lookahead) + token = self.nextToken() + expr = self.inheritCoverGrammar(self.parseUnaryExpression) + expr = self.finalize(node, Node.UnaryExpression(token.value, expr)) + if self.context.strict and expr.operator == 'delete' and expr.argument.type is Syntax.Identifier: + self.tolerateError(Messages.StrictDelete) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + elif self.context.allowAwait and self.matchContextualKeyword('await'): + expr = self.parseAwaitExpression() + else: + expr = self.parseUpdateExpression() + + return expr + + def parseExponentiationExpression(self): + startToken = self.lookahead + + expr = self.inheritCoverGrammar(self.parseUnaryExpression) + if expr.type is not Syntax.UnaryExpression and self.match('**'): + self.nextToken() + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + left = expr + right = self.isolateCoverGrammar(self.parseExponentiationExpression) + expr = self.finalize(self.startNode(startToken), Node.BinaryExpression('**', left, right)) + + return expr + + # https://tc39.github.io/ecma262/#sec-exp-operator + # https://tc39.github.io/ecma262/#sec-multiplicative-operators + # https://tc39.github.io/ecma262/#sec-additive-operators + # https://tc39.github.io/ecma262/#sec-bitwise-shift-operators + # https://tc39.github.io/ecma262/#sec-relational-operators + # https://tc39.github.io/ecma262/#sec-equality-operators + # https://tc39.github.io/ecma262/#sec-binary-bitwise-operators + # https://tc39.github.io/ecma262/#sec-binary-logical-operators + + def binaryPrecedence(self, token): + op = token.value + if token.type is Token.Punctuator: + precedence = self.operatorPrecedence.get(op, 0) + elif token.type is Token.Keyword: + precedence = 7 if (op == 'instanceof' or (self.context.allowIn and op == 'in')) else 0 + else: + precedence = 0 + return precedence + + def parseBinaryExpression(self): + startToken = self.lookahead + + expr = self.inheritCoverGrammar(self.parseExponentiationExpression) + + token = self.lookahead + prec = self.binaryPrecedence(token) + if prec > 0: + self.nextToken() + + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + markers = [startToken, self.lookahead] + left = expr + right = self.isolateCoverGrammar(self.parseExponentiationExpression) + + stack = [left, token.value, right] + precedences = [prec] + while True: + prec = self.binaryPrecedence(self.lookahead) + if prec <= 0: + break + + # Reduce: make a binary expression from the three topmost entries. + while len(stack) > 2 and prec <= precedences[-1]: + right = stack.pop() + operator = stack.pop() + precedences.pop() + left = stack.pop() + markers.pop() + node = self.startNode(markers[-1]) + stack.append(self.finalize(node, Node.BinaryExpression(operator, left, right))) + + # Shift. + stack.append(self.nextToken().value) + precedences.append(prec) + markers.append(self.lookahead) + stack.append(self.isolateCoverGrammar(self.parseExponentiationExpression)) + + # Final reduce to clean-up the stack. + i = len(stack) - 1 + expr = stack[i] + + lastMarker = markers.pop() + while i > 1: + marker = markers.pop() + lastLineStart = lastMarker.lineStart if lastMarker else 0 + node = self.startNode(marker, lastLineStart) + operator = stack[i - 1] + expr = self.finalize(node, Node.BinaryExpression(operator, stack[i - 2], expr)) + i -= 2 + lastMarker = marker + + return expr + + # https://tc39.github.io/ecma262/#sec-conditional-operator + + def parseConditionalExpression(self): + startToken = self.lookahead + + expr = self.inheritCoverGrammar(self.parseBinaryExpression) + if self.match('?'): + self.nextToken() + + previousAllowIn = self.context.allowIn + self.context.allowIn = True + consequent = self.isolateCoverGrammar(self.parseAssignmentExpression) + self.context.allowIn = previousAllowIn + + self.expect(':') + alternate = self.isolateCoverGrammar(self.parseAssignmentExpression) + + expr = self.finalize(self.startNode(startToken), Node.ConditionalExpression(expr, consequent, alternate)) + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + + return expr + + # https://tc39.github.io/ecma262/#sec-assignment-operators + + def checkPatternParam(self, options, param): + typ = param.type + if typ is Syntax.Identifier: + self.validateParam(options, param, param.name) + elif typ is Syntax.RestElement: + self.checkPatternParam(options, param.argument) + elif typ is Syntax.AssignmentPattern: + self.checkPatternParam(options, param.left) + elif typ is Syntax.ArrayPattern: + for element in param.elements: + if element is not None: + self.checkPatternParam(options, element) + elif typ is Syntax.ObjectPattern: + for prop in param.properties: + self.checkPatternParam(options, prop if prop.type is Syntax.RestElement else prop.value) + + options.simple = options.simple and isinstance(param, Node.Identifier) + + def reinterpretAsCoverFormalsList(self, expr): + params = [expr] + + asyncArrow = False + typ = expr.type + if typ is Syntax.Identifier: + pass + elif typ is Syntax.ArrowParameterPlaceHolder: + params = expr.params + asyncArrow = expr.isAsync + else: + return None + + options = Params( + simple=True, + paramSet={}, + ) + + for param in params: + if param.type is Syntax.AssignmentPattern: + if param.right.type is Syntax.YieldExpression: + if param.right.argument: + self.throwUnexpectedToken(self.lookahead) + param.right.type = Syntax.Identifier + param.right.name = 'yield' + del param.right.argument + del param.right.delegate + elif asyncArrow and param.type is Syntax.Identifier and param.name == 'await': + self.throwUnexpectedToken(self.lookahead) + self.checkPatternParam(options, param) + + if self.context.strict or not self.context.allowYield: + for param in params: + if param.type is Syntax.YieldExpression: + self.throwUnexpectedToken(self.lookahead) + + if options.message is Messages.StrictParamDupe: + token = options.stricted if self.context.strict else options.firstRestricted + self.throwUnexpectedToken(token, options.message) + + return Params( + simple=options.simple, + params=params, + stricted=options.stricted, + firstRestricted=options.firstRestricted, + message=options.message + ) + + def parseAssignmentExpression(self): + if not self.context.allowYield and self.matchKeyword('yield'): + expr = self.parseYieldExpression() + else: + startToken = self.lookahead + token = startToken + expr = self.parseConditionalExpression() + + if token.type is Token.Identifier and (token.lineNumber == self.lookahead.lineNumber) and token.value == 'async': + if self.lookahead.type is Token.Identifier or self.matchKeyword('yield'): + arg = self.parsePrimaryExpression() + self.reinterpretExpressionAsPattern(arg) + expr = Node.AsyncArrowParameterPlaceHolder([arg]) + + if expr.type is Syntax.ArrowParameterPlaceHolder or self.match('=>'): + + # https://tc39.github.io/ecma262/#sec-arrow-function-definitions + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + isAsync = expr.isAsync + list = self.reinterpretAsCoverFormalsList(expr) + + if list: + if self.hasLineTerminator: + self.tolerateUnexpectedToken(self.lookahead) + self.context.firstCoverInitializedNameError = None + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = list.simple + + previousAllowYield = self.context.allowYield + previousAwait = self.context.allowAwait + self.context.allowYield = True + self.context.allowAwait = isAsync + + node = self.startNode(startToken) + self.expect('=>') + if self.match('{'): + previousAllowIn = self.context.allowIn + self.context.allowIn = True + body = self.parseFunctionSourceElements() + self.context.allowIn = previousAllowIn + else: + body = self.isolateCoverGrammar(self.parseAssignmentExpression) + expression = body.type is not Syntax.BlockStatement + + if self.context.strict and list.firstRestricted: + self.throwUnexpectedToken(list.firstRestricted, list.message) + if self.context.strict and list.stricted: + self.tolerateUnexpectedToken(list.stricted, list.message) + if isAsync: + expr = self.finalize(node, Node.AsyncArrowFunctionExpression(list.params, body, expression)) + else: + expr = self.finalize(node, Node.ArrowFunctionExpression(list.params, body, expression)) + + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + self.context.allowYield = previousAllowYield + self.context.allowAwait = previousAwait + else: + if self.matchAssign(): + if not self.context.isAssignmentTarget: + self.tolerateError(Messages.InvalidLHSInAssignment) + + if self.context.strict and expr.type is Syntax.Identifier: + id = expr + if self.scanner.isRestrictedWord(id.name): + self.tolerateUnexpectedToken(token, Messages.StrictLHSAssignment) + if self.scanner.isStrictModeReservedWord(id.name): + self.tolerateUnexpectedToken(token, Messages.StrictReservedWord) + + if not self.match('='): + self.context.isAssignmentTarget = False + self.context.isBindingElement = False + else: + self.reinterpretExpressionAsPattern(expr) + + token = self.nextToken() + operator = token.value + right = self.isolateCoverGrammar(self.parseAssignmentExpression) + expr = self.finalize(self.startNode(startToken), Node.AssignmentExpression(operator, expr, right)) + self.context.firstCoverInitializedNameError = None + + return expr + + # https://tc39.github.io/ecma262/#sec-comma-operator + + def parseExpression(self): + startToken = self.lookahead + expr = self.isolateCoverGrammar(self.parseAssignmentExpression) + + if self.match(','): + expressions = [] + expressions.append(expr) + while self.lookahead.type is not Token.EOF: + if not self.match(','): + break + self.nextToken() + expressions.append(self.isolateCoverGrammar(self.parseAssignmentExpression)) + + expr = self.finalize(self.startNode(startToken), Node.SequenceExpression(expressions)) + + return expr + + # https://tc39.github.io/ecma262/#sec-block + + def parseStatementListItem(self): + self.context.isAssignmentTarget = True + self.context.isBindingElement = True + if self.lookahead.type is Token.Keyword: + value = self.lookahead.value + if value == 'export': + if not self.context.isModule: + self.tolerateUnexpectedToken(self.lookahead, Messages.IllegalExportDeclaration) + statement = self.parseExportDeclaration() + elif value == 'import': + if self.matchImportCall(): + statement = self.parseExpressionStatement() + else: + if not self.context.isModule: + self.tolerateUnexpectedToken(self.lookahead, Messages.IllegalImportDeclaration) + statement = self.parseImportDeclaration() + elif value == 'const': + statement = self.parseLexicalDeclaration(Params(inFor=False)) + elif value == 'function': + statement = self.parseFunctionDeclaration() + elif value == 'class': + statement = self.parseClassDeclaration() + elif value == 'let': + statement = self.parseLexicalDeclaration(Params(inFor=False)) if self.isLexicalDeclaration() else self.parseStatement() + else: + statement = self.parseStatement() + else: + statement = self.parseStatement() + + return statement + + def parseBlock(self): + node = self.createNode() + + self.expect('{') + block = [] + while True: + if self.match('}'): + break + block.append(self.parseStatementListItem()) + self.expect('}') + + return self.finalize(node, Node.BlockStatement(block)) + + # https://tc39.github.io/ecma262/#sec-let-and-const-declarations + + def parseLexicalBinding(self, kind, options): + node = self.createNode() + params = [] + id = self.parsePattern(params, kind) + + if self.context.strict and id.type is Syntax.Identifier: + if self.scanner.isRestrictedWord(id.name): + self.tolerateError(Messages.StrictVarName) + + init = None + if kind == 'const': + if not self.matchKeyword('in') and not self.matchContextualKeyword('of'): + if self.match('='): + self.nextToken() + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + else: + self.throwError(Messages.DeclarationMissingInitializer, 'const') + elif (not options.inFor and id.type is not Syntax.Identifier) or self.match('='): + self.expect('=') + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + + return self.finalize(node, Node.VariableDeclarator(id, init)) + + def parseBindingList(self, kind, options): + lst = [self.parseLexicalBinding(kind, options)] + + while self.match(','): + self.nextToken() + lst.append(self.parseLexicalBinding(kind, options)) + + return lst + + def isLexicalDeclaration(self): + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.scanner.lex() + self.scanner.restoreState(state) + + return ( + (next.type is Token.Identifier) or + (next.type is Token.Punctuator and next.value == '[') or + (next.type is Token.Punctuator and next.value == '{') or + (next.type is Token.Keyword and next.value == 'let') or + (next.type is Token.Keyword and next.value == 'yield') + ) + + def parseLexicalDeclaration(self, options): + node = self.createNode() + kind = self.nextToken().value + assert kind == 'let' or kind == 'const', 'Lexical declaration must be either or const' + + declarations = self.parseBindingList(kind, options) + self.consumeSemicolon() + + return self.finalize(node, Node.VariableDeclaration(declarations, kind)) + + # https://tc39.github.io/ecma262/#sec-destructuring-binding-patterns + + def parseBindingRestElement(self, params, kind=None): + node = self.createNode() + + self.expect('...') + arg = self.parsePattern(params, kind) + + return self.finalize(node, Node.RestElement(arg)) + + def parseArrayPattern(self, params, kind=None): + node = self.createNode() + + self.expect('[') + elements = [] + while not self.match(']'): + if self.match(','): + self.nextToken() + elements.append(None) + else: + if self.match('...'): + elements.append(self.parseBindingRestElement(params, kind)) + break + else: + elements.append(self.parsePatternWithDefault(params, kind)) + if not self.match(']'): + self.expect(',') + self.expect(']') + + return self.finalize(node, Node.ArrayPattern(elements)) + + def parsePropertyPattern(self, params, kind=None): + node = self.createNode() + + computed = False + shorthand = False + method = False + + key = None + + if self.lookahead.type is Token.Identifier: + keyToken = self.lookahead + key = self.parseVariableIdentifier() + init = self.finalize(node, Node.Identifier(keyToken.value)) + if self.match('='): + params.append(keyToken) + shorthand = True + self.nextToken() + expr = self.parseAssignmentExpression() + value = self.finalize(self.startNode(keyToken), Node.AssignmentPattern(init, expr)) + elif not self.match(':'): + params.append(keyToken) + shorthand = True + value = init + else: + self.expect(':') + value = self.parsePatternWithDefault(params, kind) + else: + computed = self.match('[') + key = self.parseObjectPropertyKey() + self.expect(':') + value = self.parsePatternWithDefault(params, kind) + + return self.finalize(node, Node.Property('init', key, computed, value, method, shorthand)) + + def parseRestProperty(self, params, kind): + node = self.createNode() + self.expect('...') + arg = self.parsePattern(params) + if self.match('='): + self.throwError(Messages.DefaultRestProperty) + if not self.match('}'): + self.throwError(Messages.PropertyAfterRestProperty) + return self.finalize(node, Node.RestElement(arg)) + + def parseObjectPattern(self, params, kind=None): + node = self.createNode() + properties = [] + + self.expect('{') + while not self.match('}'): + properties.append(self.parseRestProperty(params, kind) if self.match('...') else self.parsePropertyPattern(params, kind)) + if not self.match('}'): + self.expect(',') + self.expect('}') + + return self.finalize(node, Node.ObjectPattern(properties)) + + def parsePattern(self, params, kind=None): + if self.match('['): + pattern = self.parseArrayPattern(params, kind) + elif self.match('{'): + pattern = self.parseObjectPattern(params, kind) + else: + if self.matchKeyword('let') and (kind in ('const', 'let')): + self.tolerateUnexpectedToken(self.lookahead, Messages.LetInLexicalBinding) + params.append(self.lookahead) + pattern = self.parseVariableIdentifier(kind) + + return pattern + + def parsePatternWithDefault(self, params, kind=None): + startToken = self.lookahead + + pattern = self.parsePattern(params, kind) + if self.match('='): + self.nextToken() + previousAllowYield = self.context.allowYield + self.context.allowYield = True + right = self.isolateCoverGrammar(self.parseAssignmentExpression) + self.context.allowYield = previousAllowYield + pattern = self.finalize(self.startNode(startToken), Node.AssignmentPattern(pattern, right)) + + return pattern + + # https://tc39.github.io/ecma262/#sec-variable-statement + + def parseVariableIdentifier(self, kind=None): + node = self.createNode() + + token = self.nextToken() + if token.type is Token.Keyword and token.value == 'yield': + if self.context.strict: + self.tolerateUnexpectedToken(token, Messages.StrictReservedWord) + elif not self.context.allowYield: + self.throwUnexpectedToken(token) + elif token.type is not Token.Identifier: + if self.context.strict and token.type is Token.Keyword and self.scanner.isStrictModeReservedWord(token.value): + self.tolerateUnexpectedToken(token, Messages.StrictReservedWord) + else: + if self.context.strict or token.value != 'let' or kind != 'var': + self.throwUnexpectedToken(token) + elif (self.context.isModule or self.context.allowAwait) and token.type is Token.Identifier and token.value == 'await': + self.tolerateUnexpectedToken(token) + + return self.finalize(node, Node.Identifier(token.value)) + + def parseVariableDeclaration(self, options): + node = self.createNode() + + params = [] + id = self.parsePattern(params, 'var') + + if self.context.strict and id.type is Syntax.Identifier: + if self.scanner.isRestrictedWord(id.name): + self.tolerateError(Messages.StrictVarName) + + init = None + if self.match('='): + self.nextToken() + init = self.isolateCoverGrammar(self.parseAssignmentExpression) + elif id.type is not Syntax.Identifier and not options.inFor: + self.expect('=') + + return self.finalize(node, Node.VariableDeclarator(id, init)) + + def parseVariableDeclarationList(self, options): + opt = Params(inFor=options.inFor) + + lst = [] + lst.append(self.parseVariableDeclaration(opt)) + while self.match(','): + self.nextToken() + lst.append(self.parseVariableDeclaration(opt)) + + return lst + + def parseVariableStatement(self): + node = self.createNode() + self.expectKeyword('var') + declarations = self.parseVariableDeclarationList(Params(inFor=False)) + self.consumeSemicolon() + + return self.finalize(node, Node.VariableDeclaration(declarations, 'var')) + + # https://tc39.github.io/ecma262/#sec-empty-statement + + def parseEmptyStatement(self): + node = self.createNode() + self.expect(';') + return self.finalize(node, Node.EmptyStatement()) + + # https://tc39.github.io/ecma262/#sec-expression-statement + + def parseExpressionStatement(self): + node = self.createNode() + expr = self.parseExpression() + self.consumeSemicolon() + return self.finalize(node, Node.ExpressionStatement(expr)) + + # https://tc39.github.io/ecma262/#sec-if-statement + + def parseIfClause(self): + if self.context.strict and self.matchKeyword('function'): + self.tolerateError(Messages.StrictFunction) + return self.parseStatement() + + def parseIfStatement(self): + node = self.createNode() + alternate = None + + self.expectKeyword('if') + self.expect('(') + test = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + consequent = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + consequent = self.parseIfClause() + if self.matchKeyword('else'): + self.nextToken() + alternate = self.parseIfClause() + + return self.finalize(node, Node.IfStatement(test, consequent, alternate)) + + # https://tc39.github.io/ecma262/#sec-do-while-statement + + def parseDoWhileStatement(self): + node = self.createNode() + self.expectKeyword('do') + + previousInIteration = self.context.inIteration + self.context.inIteration = True + body = self.parseStatement() + self.context.inIteration = previousInIteration + + self.expectKeyword('while') + self.expect('(') + test = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + else: + self.expect(')') + if self.match(';'): + self.nextToken() + + return self.finalize(node, Node.DoWhileStatement(body, test)) + + # https://tc39.github.io/ecma262/#sec-while-statement + + def parseWhileStatement(self): + node = self.createNode() + + self.expectKeyword('while') + self.expect('(') + test = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + body = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + + previousInIteration = self.context.inIteration + self.context.inIteration = True + body = self.parseStatement() + self.context.inIteration = previousInIteration + + return self.finalize(node, Node.WhileStatement(test, body)) + + # https://tc39.github.io/ecma262/#sec-for-statement + # https://tc39.github.io/ecma262/#sec-for-in-and-for-of-statements + + def parseForStatement(self): + init = None + test = None + update = None + forIn = True + left = None + right = None + + node = self.createNode() + self.expectKeyword('for') + self.expect('(') + + if self.match(';'): + self.nextToken() + else: + if self.matchKeyword('var'): + init = self.createNode() + self.nextToken() + + previousAllowIn = self.context.allowIn + self.context.allowIn = False + declarations = self.parseVariableDeclarationList(Params(inFor=True)) + self.context.allowIn = previousAllowIn + + if len(declarations) == 1 and self.matchKeyword('in'): + decl = declarations[0] + if decl.init and (decl.id.type is Syntax.ArrayPattern or decl.id.type is Syntax.ObjectPattern or self.context.strict): + self.tolerateError(Messages.ForInOfLoopInitializer, 'for-in') + init = self.finalize(init, Node.VariableDeclaration(declarations, 'var')) + self.nextToken() + left = init + right = self.parseExpression() + init = None + elif len(declarations) == 1 and declarations[0].init is None and self.matchContextualKeyword('of'): + init = self.finalize(init, Node.VariableDeclaration(declarations, 'var')) + self.nextToken() + left = init + right = self.parseAssignmentExpression() + init = None + forIn = False + else: + init = self.finalize(init, Node.VariableDeclaration(declarations, 'var')) + self.expect(';') + elif self.matchKeyword('const', 'let'): + init = self.createNode() + kind = self.nextToken().value + + if not self.context.strict and self.lookahead.value == 'in': + init = self.finalize(init, Node.Identifier(kind)) + self.nextToken() + left = init + right = self.parseExpression() + init = None + else: + previousAllowIn = self.context.allowIn + self.context.allowIn = False + declarations = self.parseBindingList(kind, Params(inFor=True)) + self.context.allowIn = previousAllowIn + + if len(declarations) == 1 and declarations[0].init is None and self.matchKeyword('in'): + init = self.finalize(init, Node.VariableDeclaration(declarations, kind)) + self.nextToken() + left = init + right = self.parseExpression() + init = None + elif len(declarations) == 1 and declarations[0].init is None and self.matchContextualKeyword('of'): + init = self.finalize(init, Node.VariableDeclaration(declarations, kind)) + self.nextToken() + left = init + right = self.parseAssignmentExpression() + init = None + forIn = False + else: + self.consumeSemicolon() + init = self.finalize(init, Node.VariableDeclaration(declarations, kind)) + else: + initStartToken = self.lookahead + previousAllowIn = self.context.allowIn + self.context.allowIn = False + init = self.inheritCoverGrammar(self.parseAssignmentExpression) + self.context.allowIn = previousAllowIn + + if self.matchKeyword('in'): + if not self.context.isAssignmentTarget or init.type is Syntax.AssignmentExpression: + self.tolerateError(Messages.InvalidLHSInForIn) + + self.nextToken() + self.reinterpretExpressionAsPattern(init) + left = init + right = self.parseExpression() + init = None + elif self.matchContextualKeyword('of'): + if not self.context.isAssignmentTarget or init.type is Syntax.AssignmentExpression: + self.tolerateError(Messages.InvalidLHSInForLoop) + + self.nextToken() + self.reinterpretExpressionAsPattern(init) + left = init + right = self.parseAssignmentExpression() + init = None + forIn = False + else: + if self.match(','): + initSeq = [init] + while self.match(','): + self.nextToken() + initSeq.append(self.isolateCoverGrammar(self.parseAssignmentExpression)) + init = self.finalize(self.startNode(initStartToken), Node.SequenceExpression(initSeq)) + self.expect(';') + + if left is None: + if not self.match(';'): + test = self.parseExpression() + self.expect(';') + if not self.match(')'): + update = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + body = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + + previousInIteration = self.context.inIteration + self.context.inIteration = True + body = self.isolateCoverGrammar(self.parseStatement) + self.context.inIteration = previousInIteration + + if left is None: + return self.finalize(node, Node.ForStatement(init, test, update, body)) + + if forIn: + return self.finalize(node, Node.ForInStatement(left, right, body)) + + return self.finalize(node, Node.ForOfStatement(left, right, body)) + + # https://tc39.github.io/ecma262/#sec-continue-statement + + def parseContinueStatement(self): + node = self.createNode() + self.expectKeyword('continue') + + label = None + if self.lookahead.type is Token.Identifier and not self.hasLineTerminator: + id = self.parseVariableIdentifier() + label = id + + key = '$' + id.name + if key not in self.context.labelSet: + self.throwError(Messages.UnknownLabel, id.name) + + self.consumeSemicolon() + if label is None and not self.context.inIteration: + self.throwError(Messages.IllegalContinue) + + return self.finalize(node, Node.ContinueStatement(label)) + + # https://tc39.github.io/ecma262/#sec-break-statement + + def parseBreakStatement(self): + node = self.createNode() + self.expectKeyword('break') + + label = None + if self.lookahead.type is Token.Identifier and not self.hasLineTerminator: + id = self.parseVariableIdentifier() + + key = '$' + id.name + if key not in self.context.labelSet: + self.throwError(Messages.UnknownLabel, id.name) + label = id + + self.consumeSemicolon() + if label is None and not self.context.inIteration and not self.context.inSwitch: + self.throwError(Messages.IllegalBreak) + + return self.finalize(node, Node.BreakStatement(label)) + + # https://tc39.github.io/ecma262/#sec-return-statement + + def parseReturnStatement(self): + if not self.context.inFunctionBody: + self.tolerateError(Messages.IllegalReturn) + + node = self.createNode() + self.expectKeyword('return') + + hasArgument = ( + ( + not self.match(';') and not self.match('}') and + not self.hasLineTerminator and self.lookahead.type is not Token.EOF + ) or + self.lookahead.type is Token.StringLiteral or + self.lookahead.type is Token.Template + ) + argument = self.parseExpression() if hasArgument else None + self.consumeSemicolon() + + return self.finalize(node, Node.ReturnStatement(argument)) + + # https://tc39.github.io/ecma262/#sec-with-statement + + def parseWithStatement(self): + if self.context.strict: + self.tolerateError(Messages.StrictModeWith) + + node = self.createNode() + + self.expectKeyword('with') + self.expect('(') + object = self.parseExpression() + + if not self.match(')') and self.config.tolerant: + self.tolerateUnexpectedToken(self.nextToken()) + body = self.finalize(self.createNode(), Node.EmptyStatement()) + else: + self.expect(')') + body = self.parseStatement() + + return self.finalize(node, Node.WithStatement(object, body)) + + # https://tc39.github.io/ecma262/#sec-switch-statement + + def parseSwitchCase(self): + node = self.createNode() + + if self.matchKeyword('default'): + self.nextToken() + test = None + else: + self.expectKeyword('case') + test = self.parseExpression() + self.expect(':') + + consequent = [] + while True: + if self.match('}') or self.matchKeyword('default', 'case'): + break + consequent.append(self.parseStatementListItem()) + + return self.finalize(node, Node.SwitchCase(test, consequent)) + + def parseSwitchStatement(self): + node = self.createNode() + self.expectKeyword('switch') + + self.expect('(') + discriminant = self.parseExpression() + self.expect(')') + + previousInSwitch = self.context.inSwitch + self.context.inSwitch = True + + cases = [] + defaultFound = False + self.expect('{') + while True: + if self.match('}'): + break + clause = self.parseSwitchCase() + if clause.test is None: + if defaultFound: + self.throwError(Messages.MultipleDefaultsInSwitch) + defaultFound = True + cases.append(clause) + self.expect('}') + + self.context.inSwitch = previousInSwitch + + return self.finalize(node, Node.SwitchStatement(discriminant, cases)) + + # https://tc39.github.io/ecma262/#sec-labelled-statements + + def parseLabelledStatement(self): + node = self.createNode() + expr = self.parseExpression() + + if expr.type is Syntax.Identifier and self.match(':'): + self.nextToken() + + id = expr + key = '$' + id.name + if key in self.context.labelSet: + self.throwError(Messages.Redeclaration, 'Label', id.name) + + self.context.labelSet[key] = True + if self.matchKeyword('class'): + self.tolerateUnexpectedToken(self.lookahead) + body = self.parseClassDeclaration() + elif self.matchKeyword('function'): + token = self.lookahead + declaration = self.parseFunctionDeclaration() + if self.context.strict: + self.tolerateUnexpectedToken(token, Messages.StrictFunction) + elif declaration.generator: + self.tolerateUnexpectedToken(token, Messages.GeneratorInLegacyContext) + body = declaration + else: + body = self.parseStatement() + del self.context.labelSet[key] + + statement = Node.LabeledStatement(id, body) + else: + self.consumeSemicolon() + statement = Node.ExpressionStatement(expr) + + return self.finalize(node, statement) + + # https://tc39.github.io/ecma262/#sec-throw-statement + + def parseThrowStatement(self): + node = self.createNode() + self.expectKeyword('throw') + + if self.hasLineTerminator: + self.throwError(Messages.NewlineAfterThrow) + + argument = self.parseExpression() + self.consumeSemicolon() + + return self.finalize(node, Node.ThrowStatement(argument)) + + # https://tc39.github.io/ecma262/#sec-try-statement + + def parseCatchClause(self): + node = self.createNode() + + self.expectKeyword('catch') + + self.expect('(') + if self.match(')'): + self.throwUnexpectedToken(self.lookahead) + + params = [] + param = self.parsePattern(params) + paramMap = {} + for p in params: + key = '$' + p.value + if key in paramMap: + self.tolerateError(Messages.DuplicateBinding, p.value) + paramMap[key] = True + + if self.context.strict and param.type is Syntax.Identifier: + if self.scanner.isRestrictedWord(param.name): + self.tolerateError(Messages.StrictCatchVariable) + + self.expect(')') + body = self.parseBlock() + + return self.finalize(node, Node.CatchClause(param, body)) + + def parseFinallyClause(self): + self.expectKeyword('finally') + return self.parseBlock() + + def parseTryStatement(self): + node = self.createNode() + self.expectKeyword('try') + + block = self.parseBlock() + handler = self.parseCatchClause() if self.matchKeyword('catch') else None + finalizer = self.parseFinallyClause() if self.matchKeyword('finally') else None + + if not handler and not finalizer: + self.throwError(Messages.NoCatchOrFinally) + + return self.finalize(node, Node.TryStatement(block, handler, finalizer)) + + # https://tc39.github.io/ecma262/#sec-debugger-statement + + def parseDebuggerStatement(self): + node = self.createNode() + self.expectKeyword('debugger') + self.consumeSemicolon() + return self.finalize(node, Node.DebuggerStatement()) + + # https://tc39.github.io/ecma262/#sec-ecmascript-language-statements-and-declarations + + def parseStatement(self): + typ = self.lookahead.type + if typ in ( + Token.BooleanLiteral, + Token.NullLiteral, + Token.NumericLiteral, + Token.StringLiteral, + Token.Template, + Token.RegularExpression, + ): + statement = self.parseExpressionStatement() + + elif typ is Token.Punctuator: + value = self.lookahead.value + if value == '{': + statement = self.parseBlock() + elif value == '(': + statement = self.parseExpressionStatement() + elif value == ';': + statement = self.parseEmptyStatement() + else: + statement = self.parseExpressionStatement() + + elif typ is Token.Identifier: + statement = self.parseFunctionDeclaration() if self.matchAsyncFunction() else self.parseLabelledStatement() + + elif typ is Token.Keyword: + value = self.lookahead.value + if value == 'break': + statement = self.parseBreakStatement() + elif value == 'continue': + statement = self.parseContinueStatement() + elif value == 'debugger': + statement = self.parseDebuggerStatement() + elif value == 'do': + statement = self.parseDoWhileStatement() + elif value == 'for': + statement = self.parseForStatement() + elif value == 'function': + statement = self.parseFunctionDeclaration() + elif value == 'if': + statement = self.parseIfStatement() + elif value == 'return': + statement = self.parseReturnStatement() + elif value == 'switch': + statement = self.parseSwitchStatement() + elif value == 'throw': + statement = self.parseThrowStatement() + elif value == 'try': + statement = self.parseTryStatement() + elif value == 'var': + statement = self.parseVariableStatement() + elif value == 'while': + statement = self.parseWhileStatement() + elif value == 'with': + statement = self.parseWithStatement() + else: + statement = self.parseExpressionStatement() + + else: + statement = self.throwUnexpectedToken(self.lookahead) + + return statement + + # https://tc39.github.io/ecma262/#sec-function-definitions + + def parseFunctionSourceElements(self): + node = self.createNode() + + self.expect('{') + body = self.parseDirectivePrologues() + + previousLabelSet = self.context.labelSet + previousInIteration = self.context.inIteration + previousInSwitch = self.context.inSwitch + previousInFunctionBody = self.context.inFunctionBody + + self.context.labelSet = {} + self.context.inIteration = False + self.context.inSwitch = False + self.context.inFunctionBody = True + + while self.lookahead.type is not Token.EOF: + if self.match('}'): + break + body.append(self.parseStatementListItem()) + + self.expect('}') + + self.context.labelSet = previousLabelSet + self.context.inIteration = previousInIteration + self.context.inSwitch = previousInSwitch + self.context.inFunctionBody = previousInFunctionBody + + return self.finalize(node, Node.BlockStatement(body)) + + def validateParam(self, options, param, name): + key = '$' + name + if self.context.strict: + if self.scanner.isRestrictedWord(name): + options.stricted = param + options.message = Messages.StrictParamName + if key in options.paramSet: + options.stricted = param + options.message = Messages.StrictParamDupe + elif not options.firstRestricted: + if self.scanner.isRestrictedWord(name): + options.firstRestricted = param + options.message = Messages.StrictParamName + elif self.scanner.isStrictModeReservedWord(name): + options.firstRestricted = param + options.message = Messages.StrictReservedWord + elif key in options.paramSet: + options.stricted = param + options.message = Messages.StrictParamDupe + + options.paramSet[key] = True + + def parseRestElement(self, params): + node = self.createNode() + + self.expect('...') + arg = self.parsePattern(params) + if self.match('='): + self.throwError(Messages.DefaultRestParameter) + if not self.match(')'): + self.throwError(Messages.ParameterAfterRestParameter) + + return self.finalize(node, Node.RestElement(arg)) + + def parseFormalParameter(self, options): + params = [] + param = self.parseRestElement(params) if self.match('...') else self.parsePatternWithDefault(params) + for p in params: + self.validateParam(options, p, p.value) + options.simple = options.simple and isinstance(param, Node.Identifier) + options.params.append(param) + + def parseFormalParameters(self, firstRestricted=None): + options = Params( + simple=True, + params=[], + firstRestricted=firstRestricted + ) + + self.expect('(') + if not self.match(')'): + options.paramSet = {} + while self.lookahead.type is not Token.EOF: + self.parseFormalParameter(options) + if self.match(')'): + break + self.expect(',') + if self.match(')'): + break + self.expect(')') + + return Params( + simple=options.simple, + params=options.params, + stricted=options.stricted, + firstRestricted=options.firstRestricted, + message=options.message + ) + + def matchAsyncFunction(self): + match = self.matchContextualKeyword('async') + if match: + state = self.scanner.saveState() + self.scanner.scanComments() + next = self.scanner.lex() + self.scanner.restoreState(state) + + match = (state.lineNumber == next.lineNumber) and (next.type is Token.Keyword) and (next.value == 'function') + + return match + + def parseFunctionDeclaration(self, identifierIsOptional=False): + node = self.createNode() + + isAsync = self.matchContextualKeyword('async') + if isAsync: + self.nextToken() + + self.expectKeyword('function') + + isGenerator = False if isAsync else self.match('*') + if isGenerator: + self.nextToken() + + id = None + firstRestricted = None + + if not identifierIsOptional or not self.match('('): + token = self.lookahead + id = self.parseVariableIdentifier() + if self.context.strict: + if self.scanner.isRestrictedWord(token.value): + self.tolerateUnexpectedToken(token, Messages.StrictFunctionName) + else: + if self.scanner.isRestrictedWord(token.value): + firstRestricted = token + message = Messages.StrictFunctionName + elif self.scanner.isStrictModeReservedWord(token.value): + firstRestricted = token + message = Messages.StrictReservedWord + + previousAllowAwait = self.context.allowAwait + previousAllowYield = self.context.allowYield + self.context.allowAwait = isAsync + self.context.allowYield = not isGenerator + + formalParameters = self.parseFormalParameters(firstRestricted) + params = formalParameters.params + stricted = formalParameters.stricted + firstRestricted = formalParameters.firstRestricted + if formalParameters.message: + message = formalParameters.message + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = formalParameters.simple + body = self.parseFunctionSourceElements() + if self.context.strict and firstRestricted: + self.throwUnexpectedToken(firstRestricted, message) + if self.context.strict and stricted: + self.tolerateUnexpectedToken(stricted, message) + + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + self.context.allowAwait = previousAllowAwait + self.context.allowYield = previousAllowYield + + if isAsync: + return self.finalize(node, Node.AsyncFunctionDeclaration(id, params, body)) + + return self.finalize(node, Node.FunctionDeclaration(id, params, body, isGenerator)) + + def parseFunctionExpression(self): + node = self.createNode() + + isAsync = self.matchContextualKeyword('async') + if isAsync: + self.nextToken() + + self.expectKeyword('function') + + isGenerator = False if isAsync else self.match('*') + if isGenerator: + self.nextToken() + + id = None + firstRestricted = None + + previousAllowAwait = self.context.allowAwait + previousAllowYield = self.context.allowYield + self.context.allowAwait = isAsync + self.context.allowYield = not isGenerator + + if not self.match('('): + token = self.lookahead + id = self.parseIdentifierName() if not self.context.strict and not isGenerator and self.matchKeyword('yield') else self.parseVariableIdentifier() + if self.context.strict: + if self.scanner.isRestrictedWord(token.value): + self.tolerateUnexpectedToken(token, Messages.StrictFunctionName) + else: + if self.scanner.isRestrictedWord(token.value): + firstRestricted = token + message = Messages.StrictFunctionName + elif self.scanner.isStrictModeReservedWord(token.value): + firstRestricted = token + message = Messages.StrictReservedWord + + formalParameters = self.parseFormalParameters(firstRestricted) + params = formalParameters.params + stricted = formalParameters.stricted + firstRestricted = formalParameters.firstRestricted + if formalParameters.message: + message = formalParameters.message + + previousStrict = self.context.strict + previousAllowStrictDirective = self.context.allowStrictDirective + self.context.allowStrictDirective = formalParameters.simple + body = self.parseFunctionSourceElements() + if self.context.strict and firstRestricted: + self.throwUnexpectedToken(firstRestricted, message) + if self.context.strict and stricted: + self.tolerateUnexpectedToken(stricted, message) + self.context.strict = previousStrict + self.context.allowStrictDirective = previousAllowStrictDirective + self.context.allowAwait = previousAllowAwait + self.context.allowYield = previousAllowYield + + if isAsync: + return self.finalize(node, Node.AsyncFunctionExpression(id, params, body)) + + return self.finalize(node, Node.FunctionExpression(id, params, body, isGenerator)) + + # https://tc39.github.io/ecma262/#sec-directive-prologues-and-the-use-strict-directive + + def parseDirective(self): + token = self.lookahead + + node = self.createNode() + expr = self.parseExpression() + directive = self.getTokenRaw(token)[1:-1] if expr.type is Syntax.Literal else None + self.consumeSemicolon() + + return self.finalize(node, Node.Directive(expr, directive) if directive else Node.ExpressionStatement(expr)) + + def parseDirectivePrologues(self): + firstRestricted = None + + body = [] + while True: + token = self.lookahead + if token.type is not Token.StringLiteral: + break + + statement = self.parseDirective() + body.append(statement) + directive = statement.directive + if not isinstance(directive, basestring): + break + + if directive == 'use strict': + self.context.strict = True + if firstRestricted: + self.tolerateUnexpectedToken(firstRestricted, Messages.StrictOctalLiteral) + if not self.context.allowStrictDirective: + self.tolerateUnexpectedToken(token, Messages.IllegalLanguageModeDirective) + else: + if not firstRestricted and token.octal: + firstRestricted = token + + return body + + # https://tc39.github.io/ecma262/#sec-method-definitions + + def qualifiedPropertyName(self, token): + typ = token.type + if typ in ( + Token.Identifier, + Token.StringLiteral, + Token.BooleanLiteral, + Token.NullLiteral, + Token.NumericLiteral, + Token.Keyword, + ): + return True + elif typ is Token.Punctuator: + return token.value == '[' + return False + + def parseGetterMethod(self): + node = self.createNode() + + isGenerator = False + previousAllowYield = self.context.allowYield + self.context.allowYield = not isGenerator + formalParameters = self.parseFormalParameters() + if len(formalParameters.params) > 0: + self.tolerateError(Messages.BadGetterArity) + method = self.parsePropertyMethod(formalParameters) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, formalParameters.params, method, isGenerator)) + + def parseSetterMethod(self): + node = self.createNode() + + isGenerator = False + previousAllowYield = self.context.allowYield + self.context.allowYield = not isGenerator + formalParameters = self.parseFormalParameters() + if len(formalParameters.params) != 1: + self.tolerateError(Messages.BadSetterArity) + elif isinstance(formalParameters.params[0], Node.RestElement): + self.tolerateError(Messages.BadSetterRestParameter) + method = self.parsePropertyMethod(formalParameters) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, formalParameters.params, method, isGenerator)) + + def parseGeneratorMethod(self): + node = self.createNode() + + isGenerator = True + previousAllowYield = self.context.allowYield + + self.context.allowYield = True + params = self.parseFormalParameters() + self.context.allowYield = False + method = self.parsePropertyMethod(params) + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.FunctionExpression(None, params.params, method, isGenerator)) + + # https://tc39.github.io/ecma262/#sec-generator-function-definitions + + def isStartOfExpression(self): + start = True + + value = self.lookahead.value + typ = self.lookahead.type + if typ is Token.Punctuator: + start = value in ('[', '(', '{', '+', '-', '!', '~', '++', '--', '/', '/=') # regular expression literal ) + + elif typ is Token.Keyword: + start = value in ('class', 'delete', 'function', 'let', 'new', 'super', 'this', 'typeof', 'void', 'yield') + + return start + + def parseYieldExpression(self): + node = self.createNode() + self.expectKeyword('yield') + + argument = None + delegate = False + if not self.hasLineTerminator: + previousAllowYield = self.context.allowYield + self.context.allowYield = False + delegate = self.match('*') + if delegate: + self.nextToken() + argument = self.parseAssignmentExpression() + elif self.isStartOfExpression(): + argument = self.parseAssignmentExpression() + self.context.allowYield = previousAllowYield + + return self.finalize(node, Node.YieldExpression(argument, delegate)) + + # https://tc39.github.io/ecma262/#sec-class-definitions + + def parseClassElement(self, hasConstructor): + token = self.lookahead + node = self.createNode() + + kind = '' + key = None + value = None + computed = False + isStatic = False + isAsync = False + + if self.match('*'): + self.nextToken() + + else: + computed = self.match('[') + key = self.parseObjectPropertyKey() + id = key + if id.name == 'static' and (self.qualifiedPropertyName(self.lookahead) or self.match('*')): + token = self.lookahead + isStatic = True + computed = self.match('[') + if self.match('*'): + self.nextToken() + else: + key = self.parseObjectPropertyKey() + if token.type is Token.Identifier and not self.hasLineTerminator and token.value == 'async': + punctuator = self.lookahead.value + if punctuator != ':' and punctuator != '(' and punctuator != '*': + isAsync = True + token = self.lookahead + key = self.parseObjectPropertyKey() + if token.type is Token.Identifier and token.value == 'constructor': + self.tolerateUnexpectedToken(token, Messages.ConstructorIsAsync) + + lookaheadPropertyKey = self.qualifiedPropertyName(self.lookahead) + if token.type is Token.Identifier: + if token.value == 'get' and lookaheadPropertyKey: + kind = 'get' + computed = self.match('[') + key = self.parseObjectPropertyKey() + self.context.allowYield = False + value = self.parseGetterMethod() + elif token.value == 'set' and lookaheadPropertyKey: + kind = 'set' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseSetterMethod() + elif self.config.classProperties and not self.match('('): + kind = 'init' + id = self.finalize(node, Node.Identifier(token.value)) + if self.match('='): + self.nextToken() + value = self.parseAssignmentExpression() + + elif token.type is Token.Punctuator and token.value == '*' and lookaheadPropertyKey: + kind = 'method' + computed = self.match('[') + key = self.parseObjectPropertyKey() + value = self.parseGeneratorMethod() + + if not kind and key and self.match('('): + kind = 'method' + value = self.parsePropertyMethodAsyncFunction() if isAsync else self.parsePropertyMethodFunction() + + if not kind: + self.throwUnexpectedToken(self.lookahead) + + if not computed: + if isStatic and self.isPropertyKey(key, 'prototype'): + self.throwUnexpectedToken(token, Messages.StaticPrototype) + if not isStatic and self.isPropertyKey(key, 'constructor'): + if kind != 'method' or (value and value.generator): + self.throwUnexpectedToken(token, Messages.ConstructorSpecialMethod) + if hasConstructor.value: + self.throwUnexpectedToken(token, Messages.DuplicateConstructor) + else: + hasConstructor.value = True + kind = 'constructor' + + if kind in ('constructor', 'method', 'get', 'set'): + return self.finalize(node, Node.MethodDefinition(key, computed, value, kind, isStatic)) + + else: + return self.finalize(node, Node.FieldDefinition(key, computed, value, kind, isStatic)) + + def parseClassElementList(self): + body = [] + hasConstructor = Value(False) + + self.expect('{') + while not self.match('}'): + if self.match(';'): + self.nextToken() + else: + body.append(self.parseClassElement(hasConstructor)) + self.expect('}') + + return body + + def parseClassBody(self): + node = self.createNode() + elementList = self.parseClassElementList() + + return self.finalize(node, Node.ClassBody(elementList)) + + def parseClassDeclaration(self, identifierIsOptional=False): + node = self.createNode() + + previousStrict = self.context.strict + self.context.strict = True + self.expectKeyword('class') + + id = None if identifierIsOptional and self.lookahead.type is not Token.Identifier else self.parseVariableIdentifier() + superClass = None + if self.matchKeyword('extends'): + self.nextToken() + superClass = self.isolateCoverGrammar(self.parseLeftHandSideExpressionAllowCall) + classBody = self.parseClassBody() + self.context.strict = previousStrict + + return self.finalize(node, Node.ClassDeclaration(id, superClass, classBody)) + + def parseClassExpression(self): + node = self.createNode() + + previousStrict = self.context.strict + self.context.strict = True + self.expectKeyword('class') + id = self.parseVariableIdentifier() if self.lookahead.type is Token.Identifier else None + superClass = None + if self.matchKeyword('extends'): + self.nextToken() + superClass = self.isolateCoverGrammar(self.parseLeftHandSideExpressionAllowCall) + classBody = self.parseClassBody() + self.context.strict = previousStrict + + return self.finalize(node, Node.ClassExpression(id, superClass, classBody)) + + # https://tc39.github.io/ecma262/#sec-scripts + # https://tc39.github.io/ecma262/#sec-modules + + def parseModule(self): + self.context.strict = True + self.context.isModule = True + self.scanner.isModule = True + node = self.createNode() + body = self.parseDirectivePrologues() + while self.lookahead.type is not Token.EOF: + body.append(self.parseStatementListItem()) + return self.finalize(node, Node.Module(body)) + + def parseScript(self): + node = self.createNode() + body = self.parseDirectivePrologues() + while self.lookahead.type is not Token.EOF: + body.append(self.parseStatementListItem()) + return self.finalize(node, Node.Script(body)) + + # https://tc39.github.io/ecma262/#sec-imports + + def parseModuleSpecifier(self): + node = self.createNode() + + if self.lookahead.type is not Token.StringLiteral: + self.throwError(Messages.InvalidModuleSpecifier) + + token = self.nextToken() + raw = self.getTokenRaw(token) + return self.finalize(node, Node.Literal(token.value, raw)) + + # import {} ... + def parseImportSpecifier(self): + node = self.createNode() + + if self.lookahead.type is Token.Identifier: + imported = self.parseVariableIdentifier() + local = imported + if self.matchContextualKeyword('as'): + self.nextToken() + local = self.parseVariableIdentifier() + else: + imported = self.parseIdentifierName() + local = imported + if self.matchContextualKeyword('as'): + self.nextToken() + local = self.parseVariableIdentifier() + else: + self.throwUnexpectedToken(self.nextToken()) + + return self.finalize(node, Node.ImportSpecifier(local, imported)) + + # {foo, bar as bas + def parseNamedImports(self): + self.expect('{') + specifiers = [] + while not self.match('}'): + specifiers.append(self.parseImportSpecifier()) + if not self.match('}'): + self.expect(',') + self.expect('}') + + return specifiers + + # import ... + def parseImportDefaultSpecifier(self): + node = self.createNode() + local = self.parseIdentifierName() + return self.finalize(node, Node.ImportDefaultSpecifier(local)) + + # import <* as foo> ... + def parseImportNamespaceSpecifier(self): + node = self.createNode() + + self.expect('*') + if not self.matchContextualKeyword('as'): + self.throwError(Messages.NoAsAfterImportNamespace) + self.nextToken() + local = self.parseIdentifierName() + + return self.finalize(node, Node.ImportNamespaceSpecifier(local)) + + def parseImportDeclaration(self): + if self.context.inFunctionBody: + self.throwError(Messages.IllegalImportDeclaration) + + node = self.createNode() + self.expectKeyword('import') + + specifiers = [] + if self.lookahead.type is Token.StringLiteral: + # import 'foo' + src = self.parseModuleSpecifier() + else: + if self.match('{'): + # import {bar + specifiers.extend(self.parseNamedImports()) + elif self.match('*'): + # import * as foo + specifiers.append(self.parseImportNamespaceSpecifier()) + elif self.isIdentifierName(self.lookahead) and not self.matchKeyword('default'): + # import foo + specifiers.append(self.parseImportDefaultSpecifier()) + if self.match(','): + self.nextToken() + if self.match('*'): + # import foo, * as foo + specifiers.append(self.parseImportNamespaceSpecifier()) + elif self.match('{'): + # import foo, {bar + specifiers.extend(self.parseNamedImports()) + else: + self.throwUnexpectedToken(self.lookahead) + else: + self.throwUnexpectedToken(self.nextToken()) + + if not self.matchContextualKeyword('from'): + message = Messages.UnexpectedToken if self.lookahead.value else Messages.MissingFromClause + self.throwError(message, self.lookahead.value) + self.nextToken() + src = self.parseModuleSpecifier() + self.consumeSemicolon() + + return self.finalize(node, Node.ImportDeclaration(specifiers, src)) + + # https://tc39.github.io/ecma262/#sec-exports + + def parseExportSpecifier(self): + node = self.createNode() + + local = self.parseIdentifierName() + exported = local + if self.matchContextualKeyword('as'): + self.nextToken() + exported = self.parseIdentifierName() + + return self.finalize(node, Node.ExportSpecifier(local, exported)) + + def parseExportDefaultSpecifier(self): + node = self.createNode() + local = self.parseIdentifierName() + return self.finalize(node, Node.ExportDefaultSpecifier(local)) + + def parseExportDeclaration(self): + if self.context.inFunctionBody: + self.throwError(Messages.IllegalExportDeclaration) + + node = self.createNode() + self.expectKeyword('export') + + if self.matchKeyword('default'): + # export default ... + self.nextToken() + if self.matchKeyword('function'): + # export default function foo (: + # export default function (: + declaration = self.parseFunctionDeclaration(True) + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + elif self.matchKeyword('class'): + # export default class foo { + declaration = self.parseClassDeclaration(True) + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + elif self.matchContextualKeyword('async'): + # export default async function f (: + # export default async function (: + # export default async x => x + declaration = self.parseFunctionDeclaration(True) if self.matchAsyncFunction() else self.parseAssignmentExpression() + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + else: + if self.matchContextualKeyword('from'): + self.throwError(Messages.UnexpectedToken, self.lookahead.value) + # export default {} + # export default [] + # export default (1 + 2) + if self.match('{'): + declaration = self.parseObjectInitializer() + elif self.match('['): + declaration = self.parseArrayInitializer() + else: + declaration = self.parseAssignmentExpression() + self.consumeSemicolon() + exportDeclaration = self.finalize(node, Node.ExportDefaultDeclaration(declaration)) + + elif self.match('*'): + # export * from 'foo' + self.nextToken() + if not self.matchContextualKeyword('from'): + message = Messages.UnexpectedToken if self.lookahead.value else Messages.MissingFromClause + self.throwError(message, self.lookahead.value) + self.nextToken() + src = self.parseModuleSpecifier() + self.consumeSemicolon() + exportDeclaration = self.finalize(node, Node.ExportAllDeclaration(src)) + + elif self.lookahead.type is Token.Keyword: + # export var f = 1 + value = self.lookahead.value + if value in ( + 'let', + 'const', + ): + declaration = self.parseLexicalDeclaration(Params(inFor=False)) + elif value in ( + 'var', + 'class', + 'function', + ): + declaration = self.parseStatementListItem() + else: + self.throwUnexpectedToken(self.lookahead) + exportDeclaration = self.finalize(node, Node.ExportNamedDeclaration(declaration, [], None)) + + elif self.matchAsyncFunction(): + declaration = self.parseFunctionDeclaration() + exportDeclaration = self.finalize(node, Node.ExportNamedDeclaration(declaration, [], None)) + + else: + specifiers = [] + source = None + isExportFromIdentifier = False + + expectSpecifiers = True + if self.lookahead.type is Token.Identifier: + specifiers.append(self.parseExportDefaultSpecifier()) + if self.match(','): + self.nextToken() + else: + expectSpecifiers = False + + if expectSpecifiers: + self.expect('{') + while not self.match('}'): + isExportFromIdentifier = isExportFromIdentifier or self.matchKeyword('default') + specifiers.append(self.parseExportSpecifier()) + if not self.match('}'): + self.expect(',') + self.expect('}') + + if self.matchContextualKeyword('from'): + # export {default} from 'foo' + # export {foo} from 'foo' + self.nextToken() + source = self.parseModuleSpecifier() + self.consumeSemicolon() + elif isExportFromIdentifier: + # export {default}; # missing fromClause + message = Messages.UnexpectedToken if self.lookahead.value else Messages.MissingFromClause + self.throwError(message, self.lookahead.value) + else: + # export {foo} + self.consumeSemicolon() + exportDeclaration = self.finalize(node, Node.ExportNamedDeclaration(None, specifiers, source)) + + return exportDeclaration diff --git a/third_party/python/esprima/esprima/scanner.py b/third_party/python/esprima/esprima/scanner.py new file mode 100644 index 000000000000..53502a51d37e --- /dev/null +++ b/third_party/python/esprima/esprima/scanner.py @@ -0,0 +1,1189 @@ +# -*- coding: utf-8 -*- +# Copyright JS Foundation and other contributors, https://js.foundation/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, unicode_literals + +import re + +from .objects import Object +from .compat import xrange, unicode, uchr, uord +from .character import Character, HEX_CONV, OCTAL_CONV +from .messages import Messages +from .token import Token + + +def hexValue(ch): + return HEX_CONV[ch] + + +def octalValue(ch): + return OCTAL_CONV[ch] + + +class RegExp(Object): + def __init__(self, pattern=None, flags=None): + self.pattern = pattern + self.flags = flags + + +class Position(Object): + def __init__(self, line=None, column=None, offset=None): + self.line = line + self.column = column + self.offset = offset + + +class SourceLocation(Object): + def __init__(self, start=None, end=None, source=None): + self.start = start + self.end = end + self.source = source + + +class Comment(Object): + def __init__(self, multiLine=None, slice=None, range=None, loc=None): + self.multiLine = multiLine + self.slice = slice + self.range = range + self.loc = loc + + +class RawToken(Object): + def __init__(self, type=None, value=None, pattern=None, flags=None, regex=None, octal=None, cooked=None, head=None, tail=None, lineNumber=None, lineStart=None, start=None, end=None): + self.type = type + self.value = value + self.pattern = pattern + self.flags = flags + self.regex = regex + self.octal = octal + self.cooked = cooked + self.head = head + self.tail = tail + self.lineNumber = lineNumber + self.lineStart = lineStart + self.start = start + self.end = end + + +class ScannerState(Object): + def __init__(self, index=None, lineNumber=None, lineStart=None): + self.index = index + self.lineNumber = lineNumber + self.lineStart = lineStart + + +class Octal(object): + def __init__(self, octal, code): + self.octal = octal + self.code = code + + +class Scanner(object): + def __init__(self, code, handler): + self.source = unicode(code) + '\x00' + self.errorHandler = handler + self.trackComment = False + self.isModule = False + + self.length = len(code) + self.index = 0 + self.lineNumber = 1 if self.length > 0 else 0 + self.lineStart = 0 + self.curlyStack = [] + + def saveState(self): + return ScannerState( + index=self.index, + lineNumber=self.lineNumber, + lineStart=self.lineStart + ) + + def restoreState(self, state): + self.index = state.index + self.lineNumber = state.lineNumber + self.lineStart = state.lineStart + + def eof(self): + return self.index >= self.length + + def throwUnexpectedToken(self, message=Messages.UnexpectedTokenIllegal): + return self.errorHandler.throwError(self.index, self.lineNumber, + self.index - self.lineStart + 1, message) + + def tolerateUnexpectedToken(self, message=Messages.UnexpectedTokenIllegal): + self.errorHandler.tolerateError(self.index, self.lineNumber, + self.index - self.lineStart + 1, message) + + # https://tc39.github.io/ecma262/#sec-comments + + def skipSingleLineComment(self, offset): + comments = [] + + if self.trackComment: + start = self.index - offset + loc = SourceLocation( + start=Position( + line=self.lineNumber, + column=self.index - self.lineStart - offset + ), + end=Position() + ) + + while not self.eof(): + ch = self.source[self.index] + self.index += 1 + if Character.isLineTerminator(ch): + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart - 1 + ) + entry = Comment( + multiLine=False, + slice=[start + offset, self.index - 1], + range=[start, self.index - 1], + loc=loc + ) + comments.append(entry) + + if ch == '\r' and self.source[self.index] == '\n': + self.index += 1 + + self.lineNumber += 1 + self.lineStart = self.index + return comments + + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart + ) + entry = Comment( + multiLine=False, + slice=[start + offset, self.index], + range=[start, self.index], + loc=loc + ) + comments.append(entry) + + return comments + + def skipMultiLineComment(self): + comments = [] + + if self.trackComment: + comments = [] + start = self.index - 2 + loc = SourceLocation( + start=Position( + line=self.lineNumber, + column=self.index - self.lineStart - 2 + ), + end=Position() + ) + + while not self.eof(): + ch = self.source[self.index] + if Character.isLineTerminator(ch): + if ch == '\r' and self.source[self.index + 1] == '\n': + self.index += 1 + + self.lineNumber += 1 + self.index += 1 + self.lineStart = self.index + elif ch == '*': + # Block comment ends with '*/'. + if self.source[self.index + 1] == '/': + self.index += 2 + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart + ) + entry = Comment( + multiLine=True, + slice=[start + 2, self.index - 2], + range=[start, self.index], + loc=loc + ) + comments.append(entry) + + return comments + + self.index += 1 + else: + self.index += 1 + + # Ran off the end of the file - the whole thing is a comment + if self.trackComment: + loc.end = Position( + line=self.lineNumber, + column=self.index - self.lineStart + ) + entry = Comment( + multiLine=True, + slice=[start + 2, self.index], + range=[start, self.index], + loc=loc + ) + comments.append(entry) + + self.tolerateUnexpectedToken() + return comments + + def scanComments(self): + comments = [] + + start = self.index == 0 + while not self.eof(): + ch = self.source[self.index] + + if Character.isWhiteSpace(ch): + self.index += 1 + elif Character.isLineTerminator(ch): + self.index += 1 + if ch == '\r' and self.source[self.index] == '\n': + self.index += 1 + + self.lineNumber += 1 + self.lineStart = self.index + start = True + elif ch == '/': # U+002F is '/' + ch = self.source[self.index + 1] + if ch == '/': + self.index += 2 + comment = self.skipSingleLineComment(2) + if self.trackComment: + comments.extend(comment) + + start = True + elif ch == '*': # U+002A is '*' + self.index += 2 + comment = self.skipMultiLineComment() + if self.trackComment: + comments.extend(comment) + + else: + break + + elif start and ch == '-': # U+002D is '-' + # U+003E is '>' + if self.source[self.index + 1:self.index + 3] == '->': + # '-->' is a single-line comment + self.index += 3 + comment = self.skipSingleLineComment(3) + if self.trackComment: + comments.extend(comment) + + else: + break + + elif ch == '<' and not self.isModule: # U+003C is '<' + if self.source[self.index + 1:self.index + 4] == '!--': + self.index += 4 # `