From 658caae5b1caf431d011e11f5ab81599dfe7b829 Mon Sep 17 00:00:00 2001 From: Logan Rosen Date: Wed, 6 Sep 2023 16:14:29 +0000 Subject: [PATCH] Bug 1786490 - update to black 23.3.0 r=sylvestre,mach-reviewers,ahochheiden Differential Revision: https://phabricator.services.mozilla.com/D186086 --- .../python/click/click-7.1.2.dist-info/RECORD | 22 - .../python/click/click-7.1.2.dist-info/WHEEL | 6 - .../LICENSE.rst | 0 .../METADATA | 33 +- .../python/click/click-8.1.6.dist-info/RECORD | 22 + .../click-8.1.6.dist-info}/WHEEL | 2 +- .../top_level.txt | 0 third_party/python/click/click/__init__.py | 138 +- .../python/click/click/_bashcomplete.py | 375 - third_party/python/click/click/_compat.py | 841 +- .../python/click/click/_termui_impl.py | 430 +- third_party/python/click/click/_textwrap.py | 20 +- third_party/python/click/click/_unicodefun.py | 131 - third_party/python/click/click/_winconsole.py | 219 +- third_party/python/click/click/core.py | 2314 ++++-- third_party/python/click/click/decorators.py | 592 +- third_party/python/click/click/exceptions.py | 205 +- third_party/python/click/click/formatting.py | 110 +- third_party/python/click/click/globals.py | 37 +- third_party/python/click/click/parser.py | 301 +- third_party/python/click/click/py.typed | 0 .../python/click/click/shell_completion.py | 593 ++ third_party/python/click/click/termui.py | 497 +- third_party/python/click/click/testing.py | 347 +- third_party/python/click/click/types.py | 953 ++- third_party/python/click/click/utils.py | 557 +- .../packaging-21.3.dist-info/METADATA | 453 -- .../packaging/packaging-21.3.dist-info/RECORD | 19 - .../packaging-21.3.dist-info/top_level.txt | 1 - .../LICENSE | 0 .../LICENSE.APACHE | 0 .../LICENSE.BSD | 0 .../packaging-23.1.dist-info/METADATA | 99 + .../packaging/packaging-23.1.dist-info/RECORD | 21 + .../packaging-23.1.dist-info}/WHEEL | 4 +- .../python/packaging/packaging/__about__.py | 26 - .../python/packaging/packaging/__init__.py | 30 +- .../python/packaging/packaging/_elffile.py | 108 + .../python/packaging/packaging/_manylinux.py | 147 +- .../python/packaging/packaging/_musllinux.py | 72 +- .../python/packaging/packaging/_parser.py | 353 + .../python/packaging/packaging/_tokenizer.py | 192 + .../python/packaging/packaging/markers.py | 206 +- .../python/packaging/packaging/metadata.py | 408 + .../packaging/packaging/requirements.py | 121 +- .../python/packaging/packaging/specifiers.py | 914 ++- .../python/packaging/packaging/tags.py | 79 +- .../python/packaging/packaging/utils.py | 11 +- .../python/packaging/packaging/version.py | 334 +- third_party/python/poetry.lock | 37 +- .../pyparsing-2.4.7.dist-info/LICENSE | 18 - .../pyparsing-2.4.7.dist-info/METADATA | 104 - .../pyparsing-2.4.7.dist-info/RECORD | 6 - .../pyparsing-2.4.7.dist-info/top_level.txt | 1 - third_party/python/pyparsing/pyparsing.py | 7107 ----------------- third_party/python/requirements.in | 3 +- third_party/python/requirements.txt | 15 +- tools/lint/python/black_requirements.in | 2 +- tools/lint/python/black_requirements.txt | 167 +- 59 files changed, 7682 insertions(+), 12121 deletions(-) delete mode 100644 third_party/python/click/click-7.1.2.dist-info/RECORD delete mode 100644 third_party/python/click/click-7.1.2.dist-info/WHEEL rename third_party/python/click/{click-7.1.2.dist-info => click-8.1.6.dist-info}/LICENSE.rst (100%) rename third_party/python/click/{click-7.1.2.dist-info => click-8.1.6.dist-info}/METADATA (72%) create mode 100644 third_party/python/click/click-8.1.6.dist-info/RECORD rename third_party/python/{packaging/packaging-21.3.dist-info => click/click-8.1.6.dist-info}/WHEEL (65%) rename third_party/python/click/{click-7.1.2.dist-info => click-8.1.6.dist-info}/top_level.txt (100%) delete mode 100644 third_party/python/click/click/_bashcomplete.py delete mode 100644 third_party/python/click/click/_unicodefun.py create mode 100644 third_party/python/click/click/py.typed create mode 100644 third_party/python/click/click/shell_completion.py delete mode 100644 third_party/python/packaging/packaging-21.3.dist-info/METADATA delete mode 100644 third_party/python/packaging/packaging-21.3.dist-info/RECORD delete mode 100644 third_party/python/packaging/packaging-21.3.dist-info/top_level.txt rename third_party/python/packaging/{packaging-21.3.dist-info => packaging-23.1.dist-info}/LICENSE (100%) rename third_party/python/packaging/{packaging-21.3.dist-info => packaging-23.1.dist-info}/LICENSE.APACHE (100%) rename third_party/python/packaging/{packaging-21.3.dist-info => packaging-23.1.dist-info}/LICENSE.BSD (100%) create mode 100644 third_party/python/packaging/packaging-23.1.dist-info/METADATA create mode 100644 third_party/python/packaging/packaging-23.1.dist-info/RECORD rename third_party/python/{pyparsing/pyparsing-2.4.7.dist-info => packaging/packaging-23.1.dist-info}/WHEEL (53%) delete mode 100644 third_party/python/packaging/packaging/__about__.py create mode 100644 third_party/python/packaging/packaging/_elffile.py create mode 100644 third_party/python/packaging/packaging/_parser.py create mode 100644 third_party/python/packaging/packaging/_tokenizer.py create mode 100644 third_party/python/packaging/packaging/metadata.py delete mode 100644 third_party/python/pyparsing/pyparsing-2.4.7.dist-info/LICENSE delete mode 100644 third_party/python/pyparsing/pyparsing-2.4.7.dist-info/METADATA delete mode 100644 third_party/python/pyparsing/pyparsing-2.4.7.dist-info/RECORD delete mode 100644 third_party/python/pyparsing/pyparsing-2.4.7.dist-info/top_level.txt delete mode 100644 third_party/python/pyparsing/pyparsing.py diff --git a/third_party/python/click/click-7.1.2.dist-info/RECORD b/third_party/python/click/click-7.1.2.dist-info/RECORD deleted file mode 100644 index 847406774ca8..000000000000 --- a/third_party/python/click/click-7.1.2.dist-info/RECORD +++ /dev/null @@ -1,22 +0,0 @@ -click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463 -click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309 -click/_compat.py,sha256=AoMaYnZ-3pwtNXuHtlb6_UXsayoG0QZiHKIRy2VFezc,24169 -click/_termui_impl.py,sha256=yNktUMAdjYOU1HMkq915jR3zgAzUNtGSQqSTSSMn3eQ,20702 -click/_textwrap.py,sha256=ajCzkzFly5tjm9foQ5N9_MOeaYJMBjAltuFa69n4iXY,1197 -click/_unicodefun.py,sha256=apLSNEBZgUsQNPMUv072zJ1swqnm0dYVT5TqcIWTt6w,4201 -click/_winconsole.py,sha256=6YDu6Rq1Wxx4w9uinBMK2LHvP83aerZM9GQurlk3QDo,10010 -click/core.py,sha256=V6DJzastGhrC6WTDwV9MSLwcJUdX2Uf1ypmgkjBdn_Y,77650 -click/decorators.py,sha256=3TvEO_BkaHl7k6Eh1G5eC7JK4LKPdpFqH9JP0QDyTlM,11215 -click/exceptions.py,sha256=3pQAyyMFzx5A3eV0Y27WtDTyGogZRbrC6_o5DjjKBbw,8118 -click/formatting.py,sha256=Wb4gqFEpWaKPgAbOvnkCl8p-bEZx5KpM5ZSByhlnJNk,9281 -click/globals.py,sha256=ht7u2kUGI08pAarB4e4yC8Lkkxy6gJfRZyzxEj8EbWQ,1501 -click/parser.py,sha256=mFK-k58JtPpqO0AC36WAr0t5UfzEw1mvgVSyn7WCe9M,15691 -click/termui.py,sha256=G7QBEKIepRIGLvNdGwBTYiEtSImRxvTO_AglVpyHH2s,23998 -click/testing.py,sha256=EUEsDUqNXFgCLhZ0ZFOROpaVDA5I_rijwnNPE6qICgA,12854 -click/types.py,sha256=wuubik4VqgqAw5dvbYFkDt-zSAx97y9TQXuXcVaRyQA,25045 -click/utils.py,sha256=4VEcJ7iEHwjnFuzEuRtkT99o5VG3zqSD7Q2CVzv13WU,15940 -click-7.1.2.dist-info/LICENSE.rst,sha256=morRBqOU6FO_4h9C9OctWSgZoigF2ZG18ydQKSkrZY0,1475 -click-7.1.2.dist-info/METADATA,sha256=LrRgakZKV7Yg3qJqX_plu2WhFW81MzP3EqQmZhHIO8M,2868 -click-7.1.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 -click-7.1.2.dist-info/top_level.txt,sha256=J1ZQogalYS4pphY_lPECoNMfw0HzTSrZglC4Yfwo4xA,6 -click-7.1.2.dist-info/RECORD,, diff --git a/third_party/python/click/click-7.1.2.dist-info/WHEEL b/third_party/python/click/click-7.1.2.dist-info/WHEEL deleted file mode 100644 index ef99c6cf3283..000000000000 --- a/third_party/python/click/click-7.1.2.dist-info/WHEEL +++ /dev/null @@ -1,6 +0,0 @@ -Wheel-Version: 1.0 -Generator: bdist_wheel (0.34.2) -Root-Is-Purelib: true -Tag: py2-none-any -Tag: py3-none-any - diff --git a/third_party/python/click/click-7.1.2.dist-info/LICENSE.rst b/third_party/python/click/click-8.1.6.dist-info/LICENSE.rst similarity index 100% rename from third_party/python/click/click-7.1.2.dist-info/LICENSE.rst rename to third_party/python/click/click-8.1.6.dist-info/LICENSE.rst diff --git a/third_party/python/click/click-7.1.2.dist-info/METADATA b/third_party/python/click/click-8.1.6.dist-info/METADATA similarity index 72% rename from third_party/python/click/click-7.1.2.dist-info/METADATA rename to third_party/python/click/click-8.1.6.dist-info/METADATA index 00d697493a2f..15f296eb7ce6 100644 --- a/third_party/python/click/click-7.1.2.dist-info/METADATA +++ b/third_party/python/click/click-8.1.6.dist-info/METADATA @@ -1,23 +1,27 @@ Metadata-Version: 2.1 Name: click -Version: 7.1.2 +Version: 8.1.6 Summary: Composable command line interface toolkit Home-page: https://palletsprojects.com/p/click/ Maintainer: Pallets Maintainer-email: contact@palletsprojects.com License: BSD-3-Clause +Project-URL: Donate, https://palletsprojects.com/donate Project-URL: Documentation, https://click.palletsprojects.com/ -Project-URL: Code, https://github.com/pallets/click -Project-URL: Issue tracker, https://github.com/pallets/click/issues -Platform: UNKNOWN +Project-URL: Changes, https://click.palletsprojects.com/changes/ +Project-URL: Source Code, https://github.com/pallets/click/ +Project-URL: Issue Tracker, https://github.com/pallets/click/issues/ +Project-URL: Chat, https://discord.gg/pallets Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 3 -Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* +Requires-Python: >=3.7 +Description-Content-Type: text/x-rst +License-File: LICENSE.rst +Requires-Dist: colorama ; platform_system == "Windows" +Requires-Dist: importlib-metadata ; python_version < "3.8" \$ click\_ ========== @@ -47,7 +51,7 @@ Install and update using `pip`_: $ pip install -U click -.. _pip: https://pip.pypa.io/en/stable/quickstart/ +.. _pip: https://pip.pypa.io/en/stable/getting-started/ A Simple Example @@ -91,12 +95,9 @@ donate today`_. Links ----- -- Website: https://palletsprojects.com/p/click/ - Documentation: https://click.palletsprojects.com/ -- Releases: https://pypi.org/project/click/ -- Code: https://github.com/pallets/click -- Issue tracker: https://github.com/pallets/click/issues -- Test status: https://dev.azure.com/pallets/click/_build -- Official chat: https://discord.gg/t6rrQZH - - +- Changes: https://click.palletsprojects.com/changes/ +- PyPI Releases: https://pypi.org/project/click/ +- Source Code: https://github.com/pallets/click +- Issue Tracker: https://github.com/pallets/click/issues +- Chat: https://discord.gg/pallets diff --git a/third_party/python/click/click-8.1.6.dist-info/RECORD b/third_party/python/click/click-8.1.6.dist-info/RECORD new file mode 100644 index 000000000000..440c733cafaf --- /dev/null +++ b/third_party/python/click/click-8.1.6.dist-info/RECORD @@ -0,0 +1,22 @@ +click/__init__.py,sha256=85FPxtPII3PQQmI--C_LEOelVOl_T6w935vVDG-JG2A,3138 +click/_compat.py,sha256=IGKh_J5QdfKELitnRfTGHneejWxoCw_NX9tfMbdcg3w,18730 +click/_termui_impl.py,sha256=3dFYv4445Nw-rFvZOTBMBPYwB1bxnmNk9Du6Dm_oBSU,24069 +click/_textwrap.py,sha256=10fQ64OcBUMuK7mFvh8363_uoOxPlRItZBmKzRJDgoY,1353 +click/_winconsole.py,sha256=5ju3jQkcZD0W27WEMGqmEP4y_crUVzPCqsX_FYb7BO0,7860 +click/core.py,sha256=j6oEWtGgGna8JarD6WxhXmNnxLnfRjwXglbBc-8jr7U,114086 +click/decorators.py,sha256=-ZlbGYgV-oI8jr_oH4RpuL1PFS-5QmeuEAsLDAYgxtw,18719 +click/exceptions.py,sha256=fyROO-47HWFDjt2qupo7A3J32VlpM-ovJnfowu92K3s,9273 +click/formatting.py,sha256=Frf0-5W33-loyY_i9qrwXR8-STnW3m5gvyxLVUdyxyk,9706 +click/globals.py,sha256=TP-qM88STzc7f127h35TD_v920FgfOD2EwzqA0oE8XU,1961 +click/parser.py,sha256=LKyYQE9ZLj5KgIDXkrcTHQRXIggfoivX14_UVIn56YA,19067 +click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +click/shell_completion.py,sha256=azDxqZAVS6SRTF7alFCFoCMam3g5IyaMUO-PoOT4imY,18396 +click/termui.py,sha256=H7Q8FpmPelhJ2ovOhfCRhjMtCpNyjFXryAMLZODqsdc,28324 +click/testing.py,sha256=1Qd4kS5bucn1hsNIRryd0WtTMuCpkA93grkWxT8POsU,16084 +click/types.py,sha256=TZvz3hKvBztf-Hpa2enOmP4eznSPLzijjig5b_0XMxE,36391 +click/utils.py,sha256=1476UduUNY6UePGU4m18uzVHLt1sKM2PP3yWsQhbItM,20298 +click-8.1.6.dist-info/LICENSE.rst,sha256=morRBqOU6FO_4h9C9OctWSgZoigF2ZG18ydQKSkrZY0,1475 +click-8.1.6.dist-info/METADATA,sha256=W1Mybcg8Rb1eG1EDtUDutr477A_PIJDi6UOpzqohBxo,3014 +click-8.1.6.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92 +click-8.1.6.dist-info/top_level.txt,sha256=J1ZQogalYS4pphY_lPECoNMfw0HzTSrZglC4Yfwo4xA,6 +click-8.1.6.dist-info/RECORD,, diff --git a/third_party/python/packaging/packaging-21.3.dist-info/WHEEL b/third_party/python/click/click-8.1.6.dist-info/WHEEL similarity index 65% rename from third_party/python/packaging/packaging-21.3.dist-info/WHEEL rename to third_party/python/click/click-8.1.6.dist-info/WHEEL index 5bad85fdc1cd..1f37c02f2eb2 100644 --- a/third_party/python/packaging/packaging-21.3.dist-info/WHEEL +++ b/third_party/python/click/click-8.1.6.dist-info/WHEEL @@ -1,5 +1,5 @@ Wheel-Version: 1.0 -Generator: bdist_wheel (0.37.0) +Generator: bdist_wheel (0.40.0) Root-Is-Purelib: true Tag: py3-none-any diff --git a/third_party/python/click/click-7.1.2.dist-info/top_level.txt b/third_party/python/click/click-8.1.6.dist-info/top_level.txt similarity index 100% rename from third_party/python/click/click-7.1.2.dist-info/top_level.txt rename to third_party/python/click/click-8.1.6.dist-info/top_level.txt diff --git a/third_party/python/click/click/__init__.py b/third_party/python/click/click/__init__.py index 2b6008f2dd41..520ae88a979f 100644 --- a/third_party/python/click/click/__init__.py +++ b/third_party/python/click/click/__init__.py @@ -4,76 +4,70 @@ writing command line scripts fun. Unlike other modules, it's based around a simple API that does not come with too much magic and is composable. """ -from .core import Argument -from .core import BaseCommand -from .core import Command -from .core import CommandCollection -from .core import Context -from .core import Group -from .core import MultiCommand -from .core import Option -from .core import Parameter -from .decorators import argument -from .decorators import command -from .decorators import confirmation_option -from .decorators import group -from .decorators import help_option -from .decorators import make_pass_decorator -from .decorators import option -from .decorators import pass_context -from .decorators import pass_obj -from .decorators import password_option -from .decorators import version_option -from .exceptions import Abort -from .exceptions import BadArgumentUsage -from .exceptions import BadOptionUsage -from .exceptions import BadParameter -from .exceptions import ClickException -from .exceptions import FileError -from .exceptions import MissingParameter -from .exceptions import NoSuchOption -from .exceptions import UsageError -from .formatting import HelpFormatter -from .formatting import wrap_text -from .globals import get_current_context -from .parser import OptionParser -from .termui import clear -from .termui import confirm -from .termui import echo_via_pager -from .termui import edit -from .termui import get_terminal_size -from .termui import getchar -from .termui import launch -from .termui import pause -from .termui import progressbar -from .termui import prompt -from .termui import secho -from .termui import style -from .termui import unstyle -from .types import BOOL -from .types import Choice -from .types import DateTime -from .types import File -from .types import FLOAT -from .types import FloatRange -from .types import INT -from .types import IntRange -from .types import ParamType -from .types import Path -from .types import STRING -from .types import Tuple -from .types import UNPROCESSED -from .types import UUID -from .utils import echo -from .utils import format_filename -from .utils import get_app_dir -from .utils import get_binary_stream -from .utils import get_os_args -from .utils import get_text_stream -from .utils import open_file +from .core import Argument as Argument +from .core import BaseCommand as BaseCommand +from .core import Command as Command +from .core import CommandCollection as CommandCollection +from .core import Context as Context +from .core import Group as Group +from .core import MultiCommand as MultiCommand +from .core import Option as Option +from .core import Parameter as Parameter +from .decorators import argument as argument +from .decorators import command as command +from .decorators import confirmation_option as confirmation_option +from .decorators import group as group +from .decorators import help_option as help_option +from .decorators import make_pass_decorator as make_pass_decorator +from .decorators import option as option +from .decorators import pass_context as pass_context +from .decorators import pass_obj as pass_obj +from .decorators import password_option as password_option +from .decorators import version_option as version_option +from .exceptions import Abort as Abort +from .exceptions import BadArgumentUsage as BadArgumentUsage +from .exceptions import BadOptionUsage as BadOptionUsage +from .exceptions import BadParameter as BadParameter +from .exceptions import ClickException as ClickException +from .exceptions import FileError as FileError +from .exceptions import MissingParameter as MissingParameter +from .exceptions import NoSuchOption as NoSuchOption +from .exceptions import UsageError as UsageError +from .formatting import HelpFormatter as HelpFormatter +from .formatting import wrap_text as wrap_text +from .globals import get_current_context as get_current_context +from .parser import OptionParser as OptionParser +from .termui import clear as clear +from .termui import confirm as confirm +from .termui import echo_via_pager as echo_via_pager +from .termui import edit as edit +from .termui import getchar as getchar +from .termui import launch as launch +from .termui import pause as pause +from .termui import progressbar as progressbar +from .termui import prompt as prompt +from .termui import secho as secho +from .termui import style as style +from .termui import unstyle as unstyle +from .types import BOOL as BOOL +from .types import Choice as Choice +from .types import DateTime as DateTime +from .types import File as File +from .types import FLOAT as FLOAT +from .types import FloatRange as FloatRange +from .types import INT as INT +from .types import IntRange as IntRange +from .types import ParamType as ParamType +from .types import Path as Path +from .types import STRING as STRING +from .types import Tuple as Tuple +from .types import UNPROCESSED as UNPROCESSED +from .types import UUID as UUID +from .utils import echo as echo +from .utils import format_filename as format_filename +from .utils import get_app_dir as get_app_dir +from .utils import get_binary_stream as get_binary_stream +from .utils import get_text_stream as get_text_stream +from .utils import open_file as open_file -# Controls if click should emit the warning about the use of unicode -# literals. -disable_unicode_literals_warning = False - -__version__ = "7.1.2" +__version__ = "8.1.6" diff --git a/third_party/python/click/click/_bashcomplete.py b/third_party/python/click/click/_bashcomplete.py deleted file mode 100644 index 8bca24480f75..000000000000 --- a/third_party/python/click/click/_bashcomplete.py +++ /dev/null @@ -1,375 +0,0 @@ -import copy -import os -import re - -from .core import Argument -from .core import MultiCommand -from .core import Option -from .parser import split_arg_string -from .types import Choice -from .utils import echo - -try: - from collections import abc -except ImportError: - import collections as abc - -WORDBREAK = "=" - -# Note, only BASH version 4.4 and later have the nosort option. -COMPLETION_SCRIPT_BASH = """ -%(complete_func)s() { - local IFS=$'\n' - COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\ - COMP_CWORD=$COMP_CWORD \\ - %(autocomplete_var)s=complete $1 ) ) - return 0 -} - -%(complete_func)setup() { - local COMPLETION_OPTIONS="" - local BASH_VERSION_ARR=(${BASH_VERSION//./ }) - # Only BASH version 4.4 and later have the nosort option. - if [ ${BASH_VERSION_ARR[0]} -gt 4 ] || ([ ${BASH_VERSION_ARR[0]} -eq 4 ] \ -&& [ ${BASH_VERSION_ARR[1]} -ge 4 ]); then - COMPLETION_OPTIONS="-o nosort" - fi - - complete $COMPLETION_OPTIONS -F %(complete_func)s %(script_names)s -} - -%(complete_func)setup -""" - -COMPLETION_SCRIPT_ZSH = """ -#compdef %(script_names)s - -%(complete_func)s() { - local -a completions - local -a completions_with_descriptions - local -a response - (( ! $+commands[%(script_names)s] )) && return 1 - - response=("${(@f)$( env COMP_WORDS=\"${words[*]}\" \\ - COMP_CWORD=$((CURRENT-1)) \\ - %(autocomplete_var)s=\"complete_zsh\" \\ - %(script_names)s )}") - - for key descr in ${(kv)response}; do - if [[ "$descr" == "_" ]]; then - completions+=("$key") - else - completions_with_descriptions+=("$key":"$descr") - fi - done - - if [ -n "$completions_with_descriptions" ]; then - _describe -V unsorted completions_with_descriptions -U - fi - - if [ -n "$completions" ]; then - compadd -U -V unsorted -a completions - fi - compstate[insert]="automenu" -} - -compdef %(complete_func)s %(script_names)s -""" - -COMPLETION_SCRIPT_FISH = ( - "complete --no-files --command %(script_names)s --arguments" - ' "(env %(autocomplete_var)s=complete_fish' - " COMP_WORDS=(commandline -cp) COMP_CWORD=(commandline -t)" - ' %(script_names)s)"' -) - -_completion_scripts = { - "bash": COMPLETION_SCRIPT_BASH, - "zsh": COMPLETION_SCRIPT_ZSH, - "fish": COMPLETION_SCRIPT_FISH, -} - -_invalid_ident_char_re = re.compile(r"[^a-zA-Z0-9_]") - - -def get_completion_script(prog_name, complete_var, shell): - cf_name = _invalid_ident_char_re.sub("", prog_name.replace("-", "_")) - script = _completion_scripts.get(shell, COMPLETION_SCRIPT_BASH) - return ( - script - % { - "complete_func": "_{}_completion".format(cf_name), - "script_names": prog_name, - "autocomplete_var": complete_var, - } - ).strip() + ";" - - -def resolve_ctx(cli, prog_name, args): - """Parse into a hierarchy of contexts. Contexts are connected - through the parent variable. - - :param cli: command definition - :param prog_name: the program that is running - :param args: full list of args - :return: the final context/command parsed - """ - ctx = cli.make_context(prog_name, args, resilient_parsing=True) - args = ctx.protected_args + ctx.args - while args: - if isinstance(ctx.command, MultiCommand): - if not ctx.command.chain: - cmd_name, cmd, args = ctx.command.resolve_command(ctx, args) - if cmd is None: - return ctx - ctx = cmd.make_context( - cmd_name, args, parent=ctx, resilient_parsing=True - ) - args = ctx.protected_args + ctx.args - else: - # Walk chained subcommand contexts saving the last one. - while args: - cmd_name, cmd, args = ctx.command.resolve_command(ctx, args) - if cmd is None: - return ctx - sub_ctx = cmd.make_context( - cmd_name, - args, - parent=ctx, - allow_extra_args=True, - allow_interspersed_args=False, - resilient_parsing=True, - ) - args = sub_ctx.args - ctx = sub_ctx - args = sub_ctx.protected_args + sub_ctx.args - else: - break - return ctx - - -def start_of_option(param_str): - """ - :param param_str: param_str to check - :return: whether or not this is the start of an option declaration - (i.e. starts "-" or "--") - """ - return param_str and param_str[:1] == "-" - - -def is_incomplete_option(all_args, cmd_param): - """ - :param all_args: the full original list of args supplied - :param cmd_param: the current command paramter - :return: whether or not the last option declaration (i.e. starts - "-" or "--") is incomplete and corresponds to this cmd_param. In - other words whether this cmd_param option can still accept - values - """ - if not isinstance(cmd_param, Option): - return False - if cmd_param.is_flag: - return False - last_option = None - for index, arg_str in enumerate( - reversed([arg for arg in all_args if arg != WORDBREAK]) - ): - if index + 1 > cmd_param.nargs: - break - if start_of_option(arg_str): - last_option = arg_str - - return True if last_option and last_option in cmd_param.opts else False - - -def is_incomplete_argument(current_params, cmd_param): - """ - :param current_params: the current params and values for this - argument as already entered - :param cmd_param: the current command parameter - :return: whether or not the last argument is incomplete and - corresponds to this cmd_param. In other words whether or not the - this cmd_param argument can still accept values - """ - if not isinstance(cmd_param, Argument): - return False - current_param_values = current_params[cmd_param.name] - if current_param_values is None: - return True - if cmd_param.nargs == -1: - return True - if ( - isinstance(current_param_values, abc.Iterable) - and cmd_param.nargs > 1 - and len(current_param_values) < cmd_param.nargs - ): - return True - return False - - -def get_user_autocompletions(ctx, args, incomplete, cmd_param): - """ - :param ctx: context associated with the parsed command - :param args: full list of args - :param incomplete: the incomplete text to autocomplete - :param cmd_param: command definition - :return: all the possible user-specified completions for the param - """ - results = [] - if isinstance(cmd_param.type, Choice): - # Choices don't support descriptions. - results = [ - (c, None) for c in cmd_param.type.choices if str(c).startswith(incomplete) - ] - elif cmd_param.autocompletion is not None: - dynamic_completions = cmd_param.autocompletion( - ctx=ctx, args=args, incomplete=incomplete - ) - results = [ - c if isinstance(c, tuple) else (c, None) for c in dynamic_completions - ] - return results - - -def get_visible_commands_starting_with(ctx, starts_with): - """ - :param ctx: context associated with the parsed command - :starts_with: string that visible commands must start with. - :return: all visible (not hidden) commands that start with starts_with. - """ - for c in ctx.command.list_commands(ctx): - if c.startswith(starts_with): - command = ctx.command.get_command(ctx, c) - if not command.hidden: - yield command - - -def add_subcommand_completions(ctx, incomplete, completions_out): - # Add subcommand completions. - if isinstance(ctx.command, MultiCommand): - completions_out.extend( - [ - (c.name, c.get_short_help_str()) - for c in get_visible_commands_starting_with(ctx, incomplete) - ] - ) - - # Walk up the context list and add any other completion - # possibilities from chained commands - while ctx.parent is not None: - ctx = ctx.parent - if isinstance(ctx.command, MultiCommand) and ctx.command.chain: - remaining_commands = [ - c - for c in get_visible_commands_starting_with(ctx, incomplete) - if c.name not in ctx.protected_args - ] - completions_out.extend( - [(c.name, c.get_short_help_str()) for c in remaining_commands] - ) - - -def get_choices(cli, prog_name, args, incomplete): - """ - :param cli: command definition - :param prog_name: the program that is running - :param args: full list of args - :param incomplete: the incomplete text to autocomplete - :return: all the possible completions for the incomplete - """ - all_args = copy.deepcopy(args) - - ctx = resolve_ctx(cli, prog_name, args) - if ctx is None: - return [] - - has_double_dash = "--" in all_args - - # In newer versions of bash long opts with '='s are partitioned, but - # it's easier to parse without the '=' - if start_of_option(incomplete) and WORDBREAK in incomplete: - partition_incomplete = incomplete.partition(WORDBREAK) - all_args.append(partition_incomplete[0]) - incomplete = partition_incomplete[2] - elif incomplete == WORDBREAK: - incomplete = "" - - completions = [] - if not has_double_dash and start_of_option(incomplete): - # completions for partial options - for param in ctx.command.params: - if isinstance(param, Option) and not param.hidden: - param_opts = [ - param_opt - for param_opt in param.opts + param.secondary_opts - if param_opt not in all_args or param.multiple - ] - completions.extend( - [(o, param.help) for o in param_opts if o.startswith(incomplete)] - ) - return completions - # completion for option values from user supplied values - for param in ctx.command.params: - if is_incomplete_option(all_args, param): - return get_user_autocompletions(ctx, all_args, incomplete, param) - # completion for argument values from user supplied values - for param in ctx.command.params: - if is_incomplete_argument(ctx.params, param): - return get_user_autocompletions(ctx, all_args, incomplete, param) - - add_subcommand_completions(ctx, incomplete, completions) - # Sort before returning so that proper ordering can be enforced in custom types. - return sorted(completions) - - -def do_complete(cli, prog_name, include_descriptions): - cwords = split_arg_string(os.environ["COMP_WORDS"]) - cword = int(os.environ["COMP_CWORD"]) - args = cwords[1:cword] - try: - incomplete = cwords[cword] - except IndexError: - incomplete = "" - - for item in get_choices(cli, prog_name, args, incomplete): - echo(item[0]) - if include_descriptions: - # ZSH has trouble dealing with empty array parameters when - # returned from commands, use '_' to indicate no description - # is present. - echo(item[1] if item[1] else "_") - - return True - - -def do_complete_fish(cli, prog_name): - cwords = split_arg_string(os.environ["COMP_WORDS"]) - incomplete = os.environ["COMP_CWORD"] - args = cwords[1:] - - for item in get_choices(cli, prog_name, args, incomplete): - if item[1]: - echo("{arg}\t{desc}".format(arg=item[0], desc=item[1])) - else: - echo(item[0]) - - return True - - -def bashcomplete(cli, prog_name, complete_var, complete_instr): - if "_" in complete_instr: - command, shell = complete_instr.split("_", 1) - else: - command = complete_instr - shell = "bash" - - if command == "source": - echo(get_completion_script(prog_name, complete_var, shell)) - return True - elif command == "complete": - if shell == "fish": - return do_complete_fish(cli, prog_name) - elif shell in {"bash", "zsh"}: - return do_complete(cli, prog_name, shell == "zsh") - - return False diff --git a/third_party/python/click/click/_compat.py b/third_party/python/click/click/_compat.py index 60cb115bc509..9153d150ce67 100644 --- a/third_party/python/click/click/_compat.py +++ b/third_party/python/click/click/_compat.py @@ -1,32 +1,24 @@ -# flake8: noqa import codecs import io import os import re import sys +import typing as t from weakref import WeakKeyDictionary -PY2 = sys.version_info[0] == 2 CYGWIN = sys.platform.startswith("cygwin") -MSYS2 = sys.platform.startswith("win") and ("GCC" in sys.version) -# Determine local App Engine environment, per Google's own suggestion -APP_ENGINE = "APPENGINE_RUNTIME" in os.environ and "Development/" in os.environ.get( - "SERVER_SOFTWARE", "" -) -WIN = sys.platform.startswith("win") and not APP_ENGINE and not MSYS2 -DEFAULT_COLUMNS = 80 - - +WIN = sys.platform.startswith("win") +auto_wrap_for_ansi: t.Optional[t.Callable[[t.TextIO], t.TextIO]] = None _ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]") -def get_filesystem_encoding(): - return sys.getfilesystemencoding() or sys.getdefaultencoding() - - def _make_text_stream( - stream, encoding, errors, force_readable=False, force_writable=False -): + stream: t.BinaryIO, + encoding: t.Optional[str], + errors: t.Optional[str], + force_readable: bool = False, + force_writable: bool = False, +) -> t.TextIO: if encoding is None: encoding = get_best_encoding(stream) if errors is None: @@ -41,7 +33,7 @@ def _make_text_stream( ) -def is_ascii_encoding(encoding): +def is_ascii_encoding(encoding: str) -> bool: """Checks if a given encoding is ascii.""" try: return codecs.lookup(encoding).name == "ascii" @@ -49,7 +41,7 @@ def is_ascii_encoding(encoding): return False -def get_best_encoding(stream): +def get_best_encoding(stream: t.IO[t.Any]) -> str: """Returns the default stream encoding if not found.""" rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() if is_ascii_encoding(rv): @@ -60,46 +52,30 @@ def get_best_encoding(stream): class _NonClosingTextIOWrapper(io.TextIOWrapper): def __init__( self, - stream, - encoding, - errors, - force_readable=False, - force_writable=False, - **extra - ): - self._stream = stream = _FixupStream(stream, force_readable, force_writable) - io.TextIOWrapper.__init__(self, stream, encoding, errors, **extra) + stream: t.BinaryIO, + encoding: t.Optional[str], + errors: t.Optional[str], + force_readable: bool = False, + force_writable: bool = False, + **extra: t.Any, + ) -> None: + self._stream = stream = t.cast( + t.BinaryIO, _FixupStream(stream, force_readable, force_writable) + ) + super().__init__(stream, encoding, errors, **extra) - # The io module is a place where the Python 3 text behavior - # was forced upon Python 2, so we need to unbreak - # it to look like Python 2. - if PY2: - - def write(self, x): - if isinstance(x, str) or is_bytes(x): - try: - self.flush() - except Exception: - pass - return self.buffer.write(str(x)) - return io.TextIOWrapper.write(self, x) - - def writelines(self, lines): - for line in lines: - self.write(line) - - def __del__(self): + def __del__(self) -> None: try: self.detach() except Exception: pass - def isatty(self): + def isatty(self) -> bool: # https://bitbucket.org/pypy/pypy/issue/1803 return self._stream.isatty() -class _FixupStream(object): +class _FixupStream: """The new io interface needs more from streams than streams traditionally implement. As such, this fix-up code is necessary in some circumstances. @@ -109,45 +85,47 @@ class _FixupStream(object): of jupyter notebook). """ - def __init__(self, stream, force_readable=False, force_writable=False): + def __init__( + self, + stream: t.BinaryIO, + force_readable: bool = False, + force_writable: bool = False, + ): self._stream = stream self._force_readable = force_readable self._force_writable = force_writable - def __getattr__(self, name): + def __getattr__(self, name: str) -> t.Any: return getattr(self._stream, name) - def read1(self, size): + def read1(self, size: int) -> bytes: f = getattr(self._stream, "read1", None) + if f is not None: - return f(size) - # We only dispatch to readline instead of read in Python 2 as we - # do not want cause problems with the different implementation - # of line buffering. - if PY2: - return self._stream.readline(size) + return t.cast(bytes, f(size)) + return self._stream.read(size) - def readable(self): + def readable(self) -> bool: if self._force_readable: return True x = getattr(self._stream, "readable", None) if x is not None: - return x() + return t.cast(bool, x()) try: self._stream.read(0) except Exception: return False return True - def writable(self): + def writable(self) -> bool: if self._force_writable: return True x = getattr(self._stream, "writable", None) if x is not None: - return x() + return t.cast(bool, x()) try: - self._stream.write("") + self._stream.write("") # type: ignore except Exception: try: self._stream.write(b"") @@ -155,10 +133,10 @@ class _FixupStream(object): return False return True - def seekable(self): + def seekable(self) -> bool: x = getattr(self._stream, "seekable", None) if x is not None: - return x() + return t.cast(bool, x()) try: self._stream.seek(self._stream.tell()) except Exception: @@ -166,356 +144,245 @@ class _FixupStream(object): return True -if PY2: - text_type = unicode - raw_input = raw_input - string_types = (str, unicode) - int_types = (int, long) - iteritems = lambda x: x.iteritems() - range_type = xrange - - def is_bytes(x): - return isinstance(x, (buffer, bytearray)) - - _identifier_re = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") - - # For Windows, we need to force stdout/stdin/stderr to binary if it's - # fetched for that. This obviously is not the most correct way to do - # it as it changes global state. Unfortunately, there does not seem to - # be a clear better way to do it as just reopening the file in binary - # mode does not change anything. - # - # An option would be to do what Python 3 does and to open the file as - # binary only, patch it back to the system, and then use a wrapper - # stream that converts newlines. It's not quite clear what's the - # correct option here. - # - # This code also lives in _winconsole for the fallback to the console - # emulation stream. - # - # There are also Windows environments where the `msvcrt` module is not - # available (which is why we use try-catch instead of the WIN variable - # here), such as the Google App Engine development server on Windows. In - # those cases there is just nothing we can do. - def set_binary_mode(f): - return f - +def _is_binary_reader(stream: t.IO[t.Any], default: bool = False) -> bool: try: - import msvcrt - except ImportError: - pass - else: + return isinstance(stream.read(0), bytes) + except Exception: + return default + # This happens in some cases where the stream was already + # closed. In this case, we assume the default. - def set_binary_mode(f): - try: - fileno = f.fileno() - except Exception: - pass - else: - msvcrt.setmode(fileno, os.O_BINARY) - return f +def _is_binary_writer(stream: t.IO[t.Any], default: bool = False) -> bool: try: - import fcntl - except ImportError: - pass + stream.write(b"") + except Exception: + try: + stream.write("") + return False + except Exception: + pass + return default + return True + + +def _find_binary_reader(stream: t.IO[t.Any]) -> t.Optional[t.BinaryIO]: + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_reader(stream, False): + return t.cast(t.BinaryIO, stream) + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_reader(buf, True): + return t.cast(t.BinaryIO, buf) + + return None + + +def _find_binary_writer(stream: t.IO[t.Any]) -> t.Optional[t.BinaryIO]: + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_writer(stream, False): + return t.cast(t.BinaryIO, stream) + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_writer(buf, True): + return t.cast(t.BinaryIO, buf) + + return None + + +def _stream_is_misconfigured(stream: t.TextIO) -> bool: + """A stream is misconfigured if its encoding is ASCII.""" + # If the stream does not have an encoding set, we assume it's set + # to ASCII. This appears to happen in certain unittest + # environments. It's not quite clear what the correct behavior is + # but this at least will force Click to recover somehow. + return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") + + +def _is_compat_stream_attr(stream: t.TextIO, attr: str, value: t.Optional[str]) -> bool: + """A stream attribute is compatible if it is equal to the + desired value or the desired value is unset and the attribute + has a value. + """ + stream_value = getattr(stream, attr, None) + return stream_value == value or (value is None and stream_value is not None) + + +def _is_compatible_text_stream( + stream: t.TextIO, encoding: t.Optional[str], errors: t.Optional[str] +) -> bool: + """Check if a stream's encoding and errors attributes are + compatible with the desired values. + """ + return _is_compat_stream_attr( + stream, "encoding", encoding + ) and _is_compat_stream_attr(stream, "errors", errors) + + +def _force_correct_text_stream( + text_stream: t.IO[t.Any], + encoding: t.Optional[str], + errors: t.Optional[str], + is_binary: t.Callable[[t.IO[t.Any], bool], bool], + find_binary: t.Callable[[t.IO[t.Any]], t.Optional[t.BinaryIO]], + force_readable: bool = False, + force_writable: bool = False, +) -> t.TextIO: + if is_binary(text_stream, False): + binary_reader = t.cast(t.BinaryIO, text_stream) else: + text_stream = t.cast(t.TextIO, text_stream) + # If the stream looks compatible, and won't default to a + # misconfigured ascii encoding, return it as-is. + if _is_compatible_text_stream(text_stream, encoding, errors) and not ( + encoding is None and _stream_is_misconfigured(text_stream) + ): + return text_stream - def set_binary_mode(f): - try: - fileno = f.fileno() - except Exception: - pass - else: - flags = fcntl.fcntl(fileno, fcntl.F_GETFL) - fcntl.fcntl(fileno, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) - return f + # Otherwise, get the underlying binary reader. + possible_binary_reader = find_binary(text_stream) - def isidentifier(x): - return _identifier_re.search(x) is not None + # If that's not possible, silently use the original reader + # and get mojibake instead of exceptions. + if possible_binary_reader is None: + return text_stream - def get_binary_stdin(): - return set_binary_mode(sys.stdin) + binary_reader = possible_binary_reader - def get_binary_stdout(): - _wrap_std_stream("stdout") - return set_binary_mode(sys.stdout) + # Default errors to replace instead of strict in order to get + # something that works. + if errors is None: + errors = "replace" - def get_binary_stderr(): - _wrap_std_stream("stderr") - return set_binary_mode(sys.stderr) - - def get_text_stdin(encoding=None, errors=None): - rv = _get_windows_console_stream(sys.stdin, encoding, errors) - if rv is not None: - return rv - return _make_text_stream(sys.stdin, encoding, errors, force_readable=True) - - def get_text_stdout(encoding=None, errors=None): - _wrap_std_stream("stdout") - rv = _get_windows_console_stream(sys.stdout, encoding, errors) - if rv is not None: - return rv - return _make_text_stream(sys.stdout, encoding, errors, force_writable=True) - - def get_text_stderr(encoding=None, errors=None): - _wrap_std_stream("stderr") - rv = _get_windows_console_stream(sys.stderr, encoding, errors) - if rv is not None: - return rv - return _make_text_stream(sys.stderr, encoding, errors, force_writable=True) - - def filename_to_ui(value): - if isinstance(value, bytes): - value = value.decode(get_filesystem_encoding(), "replace") - return value - - -else: - import io - - text_type = str - raw_input = input - string_types = (str,) - int_types = (int,) - range_type = range - isidentifier = lambda x: x.isidentifier() - iteritems = lambda x: iter(x.items()) - - def is_bytes(x): - return isinstance(x, (bytes, memoryview, bytearray)) - - def _is_binary_reader(stream, default=False): - try: - return isinstance(stream.read(0), bytes) - except Exception: - return default - # This happens in some cases where the stream was already - # closed. In this case, we assume the default. - - def _is_binary_writer(stream, default=False): - try: - stream.write(b"") - except Exception: - try: - stream.write("") - return False - except Exception: - pass - return default - return True - - def _find_binary_reader(stream): - # We need to figure out if the given stream is already binary. - # This can happen because the official docs recommend detaching - # the streams to get binary streams. Some code might do this, so - # we need to deal with this case explicitly. - if _is_binary_reader(stream, False): - return stream - - buf = getattr(stream, "buffer", None) - - # Same situation here; this time we assume that the buffer is - # actually binary in case it's closed. - if buf is not None and _is_binary_reader(buf, True): - return buf - - def _find_binary_writer(stream): - # We need to figure out if the given stream is already binary. - # This can happen because the official docs recommend detatching - # the streams to get binary streams. Some code might do this, so - # we need to deal with this case explicitly. - if _is_binary_writer(stream, False): - return stream - - buf = getattr(stream, "buffer", None) - - # Same situation here; this time we assume that the buffer is - # actually binary in case it's closed. - if buf is not None and _is_binary_writer(buf, True): - return buf - - def _stream_is_misconfigured(stream): - """A stream is misconfigured if its encoding is ASCII.""" - # If the stream does not have an encoding set, we assume it's set - # to ASCII. This appears to happen in certain unittest - # environments. It's not quite clear what the correct behavior is - # but this at least will force Click to recover somehow. - return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") - - def _is_compat_stream_attr(stream, attr, value): - """A stream attribute is compatible if it is equal to the - desired value or the desired value is unset and the attribute - has a value. - """ - stream_value = getattr(stream, attr, None) - return stream_value == value or (value is None and stream_value is not None) - - def _is_compatible_text_stream(stream, encoding, errors): - """Check if a stream's encoding and errors attributes are - compatible with the desired values. - """ - return _is_compat_stream_attr( - stream, "encoding", encoding - ) and _is_compat_stream_attr(stream, "errors", errors) - - def _force_correct_text_stream( - text_stream, + # Wrap the binary stream in a text stream with the correct + # encoding parameters. + return _make_text_stream( + binary_reader, encoding, errors, - is_binary, - find_binary, - force_readable=False, - force_writable=False, - ): - if is_binary(text_stream, False): - binary_reader = text_stream - else: - # If the stream looks compatible, and won't default to a - # misconfigured ascii encoding, return it as-is. - if _is_compatible_text_stream(text_stream, encoding, errors) and not ( - encoding is None and _stream_is_misconfigured(text_stream) - ): - return text_stream - - # Otherwise, get the underlying binary reader. - binary_reader = find_binary(text_stream) - - # If that's not possible, silently use the original reader - # and get mojibake instead of exceptions. - if binary_reader is None: - return text_stream - - # Default errors to replace instead of strict in order to get - # something that works. - if errors is None: - errors = "replace" - - # Wrap the binary stream in a text stream with the correct - # encoding parameters. - return _make_text_stream( - binary_reader, - encoding, - errors, - force_readable=force_readable, - force_writable=force_writable, - ) - - def _force_correct_text_reader(text_reader, encoding, errors, force_readable=False): - return _force_correct_text_stream( - text_reader, - encoding, - errors, - _is_binary_reader, - _find_binary_reader, - force_readable=force_readable, - ) - - def _force_correct_text_writer(text_writer, encoding, errors, force_writable=False): - return _force_correct_text_stream( - text_writer, - encoding, - errors, - _is_binary_writer, - _find_binary_writer, - force_writable=force_writable, - ) - - def get_binary_stdin(): - reader = _find_binary_reader(sys.stdin) - if reader is None: - raise RuntimeError("Was not able to determine binary stream for sys.stdin.") - return reader - - def get_binary_stdout(): - writer = _find_binary_writer(sys.stdout) - if writer is None: - raise RuntimeError( - "Was not able to determine binary stream for sys.stdout." - ) - return writer - - def get_binary_stderr(): - writer = _find_binary_writer(sys.stderr) - if writer is None: - raise RuntimeError( - "Was not able to determine binary stream for sys.stderr." - ) - return writer - - def get_text_stdin(encoding=None, errors=None): - rv = _get_windows_console_stream(sys.stdin, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_reader( - sys.stdin, encoding, errors, force_readable=True - ) - - def get_text_stdout(encoding=None, errors=None): - rv = _get_windows_console_stream(sys.stdout, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_writer( - sys.stdout, encoding, errors, force_writable=True - ) - - def get_text_stderr(encoding=None, errors=None): - rv = _get_windows_console_stream(sys.stderr, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_writer( - sys.stderr, encoding, errors, force_writable=True - ) - - def filename_to_ui(value): - if isinstance(value, bytes): - value = value.decode(get_filesystem_encoding(), "replace") - else: - value = value.encode("utf-8", "surrogateescape").decode("utf-8", "replace") - return value + force_readable=force_readable, + force_writable=force_writable, + ) -def get_streerror(e, default=None): - if hasattr(e, "strerror"): - msg = e.strerror - else: - if default is not None: - msg = default - else: - msg = str(e) - if isinstance(msg, bytes): - msg = msg.decode("utf-8", "replace") - return msg +def _force_correct_text_reader( + text_reader: t.IO[t.Any], + encoding: t.Optional[str], + errors: t.Optional[str], + force_readable: bool = False, +) -> t.TextIO: + return _force_correct_text_stream( + text_reader, + encoding, + errors, + _is_binary_reader, + _find_binary_reader, + force_readable=force_readable, + ) -def _wrap_io_open(file, mode, encoding, errors): - """On Python 2, :func:`io.open` returns a text file wrapper that - requires passing ``unicode`` to ``write``. Need to open the file in - binary mode then wrap it in a subclass that can write ``str`` and - ``unicode``. +def _force_correct_text_writer( + text_writer: t.IO[t.Any], + encoding: t.Optional[str], + errors: t.Optional[str], + force_writable: bool = False, +) -> t.TextIO: + return _force_correct_text_stream( + text_writer, + encoding, + errors, + _is_binary_writer, + _find_binary_writer, + force_writable=force_writable, + ) - Also handles not passing ``encoding`` and ``errors`` in binary mode. - """ + +def get_binary_stdin() -> t.BinaryIO: + reader = _find_binary_reader(sys.stdin) + if reader is None: + raise RuntimeError("Was not able to determine binary stream for sys.stdin.") + return reader + + +def get_binary_stdout() -> t.BinaryIO: + writer = _find_binary_writer(sys.stdout) + if writer is None: + raise RuntimeError("Was not able to determine binary stream for sys.stdout.") + return writer + + +def get_binary_stderr() -> t.BinaryIO: + writer = _find_binary_writer(sys.stderr) + if writer is None: + raise RuntimeError("Was not able to determine binary stream for sys.stderr.") + return writer + + +def get_text_stdin( + encoding: t.Optional[str] = None, errors: t.Optional[str] = None +) -> t.TextIO: + rv = _get_windows_console_stream(sys.stdin, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_reader(sys.stdin, encoding, errors, force_readable=True) + + +def get_text_stdout( + encoding: t.Optional[str] = None, errors: t.Optional[str] = None +) -> t.TextIO: + rv = _get_windows_console_stream(sys.stdout, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stdout, encoding, errors, force_writable=True) + + +def get_text_stderr( + encoding: t.Optional[str] = None, errors: t.Optional[str] = None +) -> t.TextIO: + rv = _get_windows_console_stream(sys.stderr, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stderr, encoding, errors, force_writable=True) + + +def _wrap_io_open( + file: t.Union[str, "os.PathLike[str]", int], + mode: str, + encoding: t.Optional[str], + errors: t.Optional[str], +) -> t.IO[t.Any]: + """Handles not passing ``encoding`` and ``errors`` in binary mode.""" + if "b" in mode: + return open(file, mode) + + return open(file, mode, encoding=encoding, errors=errors) + + +def open_stream( + filename: "t.Union[str, os.PathLike[str]]", + mode: str = "r", + encoding: t.Optional[str] = None, + errors: t.Optional[str] = "strict", + atomic: bool = False, +) -> t.Tuple[t.IO[t.Any], bool]: binary = "b" in mode + filename = os.fspath(filename) - if binary: - kwargs = {} - else: - kwargs = {"encoding": encoding, "errors": errors} - - if not PY2 or binary: - return io.open(file, mode, **kwargs) - - f = io.open(file, "{}b".format(mode.replace("t", ""))) - return _make_text_stream(f, **kwargs) - - -def open_stream(filename, mode="r", encoding=None, errors="strict", atomic=False): - binary = "b" in mode - - # Standard streams first. These are simple because they don't need - # special handling for the atomic flag. It's entirely ignored. - if filename == "-": + # Standard streams first. These are simple because they ignore the + # atomic flag. Use fsdecode to handle Path("-"). + if os.fsdecode(filename) == "-": if any(m in mode for m in ["w", "a", "x"]): if binary: return get_binary_stdout(), False @@ -549,7 +416,7 @@ def open_stream(filename, mode="r", encoding=None, errors="strict", atomic=False import random try: - perm = os.stat(filename).st_mode + perm: t.Optional[int] = os.stat(filename).st_mode except OSError: perm = None @@ -561,7 +428,7 @@ def open_stream(filename, mode="r", encoding=None, errors="strict", atomic=False while True: tmp_filename = os.path.join( os.path.dirname(filename), - ".__atomic-write{:08x}".format(random.randrange(1 << 32)), + f".__atomic-write{random.randrange(1 << 32):08x}", ) try: fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm) @@ -580,76 +447,55 @@ def open_stream(filename, mode="r", encoding=None, errors="strict", atomic=False os.chmod(tmp_filename, perm) # in case perm includes bits in umask f = _wrap_io_open(fd, mode, encoding, errors) - return _AtomicFile(f, tmp_filename, os.path.realpath(filename)), True + af = _AtomicFile(f, tmp_filename, os.path.realpath(filename)) + return t.cast(t.IO[t.Any], af), True -# Used in a destructor call, needs extra protection from interpreter cleanup. -if hasattr(os, "replace"): - _replace = os.replace - _can_replace = True -else: - _replace = os.rename - _can_replace = not WIN - - -class _AtomicFile(object): - def __init__(self, f, tmp_filename, real_filename): +class _AtomicFile: + def __init__(self, f: t.IO[t.Any], tmp_filename: str, real_filename: str) -> None: self._f = f self._tmp_filename = tmp_filename self._real_filename = real_filename self.closed = False @property - def name(self): + def name(self) -> str: return self._real_filename - def close(self, delete=False): + def close(self, delete: bool = False) -> None: if self.closed: return self._f.close() - if not _can_replace: - try: - os.remove(self._real_filename) - except OSError: - pass - _replace(self._tmp_filename, self._real_filename) + os.replace(self._tmp_filename, self._real_filename) self.closed = True - def __getattr__(self, name): + def __getattr__(self, name: str) -> t.Any: return getattr(self._f, name) - def __enter__(self): + def __enter__(self) -> "_AtomicFile": return self - def __exit__(self, exc_type, exc_value, tb): + def __exit__(self, exc_type: t.Optional[t.Type[BaseException]], *_: t.Any) -> None: self.close(delete=exc_type is not None) - def __repr__(self): + def __repr__(self) -> str: return repr(self._f) -auto_wrap_for_ansi = None -colorama = None -get_winterm_size = None - - -def strip_ansi(value): +def strip_ansi(value: str) -> str: return _ansi_re.sub("", value) -def _is_jupyter_kernel_output(stream): - if WIN: - # TODO: Couldn't test on Windows, should't try to support until - # someone tests the details wrt colorama. - return - +def _is_jupyter_kernel_output(stream: t.IO[t.Any]) -> bool: while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)): stream = stream._stream return stream.__class__.__module__.startswith("ipykernel.") -def should_strip_ansi(stream=None, color=None): +def should_strip_ansi( + stream: t.Optional[t.IO[t.Any]] = None, color: t.Optional[bool] = None +) -> bool: if color is None: if stream is None: stream = sys.stdin @@ -657,100 +503,90 @@ def should_strip_ansi(stream=None, color=None): return not color -# If we're on Windows, we provide transparent integration through -# colorama. This will make ANSI colors through the echo function -# work automatically. -if WIN: - # Windows has a smaller terminal - DEFAULT_COLUMNS = 79 +# On Windows, wrap the output streams with colorama to support ANSI +# color codes. +# NOTE: double check is needed so mypy does not analyze this on Linux +if sys.platform.startswith("win") and WIN: + from ._winconsole import _get_windows_console_stream - from ._winconsole import _get_windows_console_stream, _wrap_std_stream - - def _get_argv_encoding(): + def _get_argv_encoding() -> str: import locale return locale.getpreferredencoding() - if PY2: + _ansi_stream_wrappers: t.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary() - def raw_input(prompt=""): - sys.stderr.flush() - if prompt: - stdout = _default_text_stdout() - stdout.write(prompt) - stdin = _default_text_stdin() - return stdin.readline().rstrip("\r\n") + def auto_wrap_for_ansi( + stream: t.TextIO, color: t.Optional[bool] = None + ) -> t.TextIO: + """Support ANSI color and style codes on Windows by wrapping a + stream with colorama. + """ + try: + cached = _ansi_stream_wrappers.get(stream) + except Exception: + cached = None + + if cached is not None: + return cached - try: import colorama - except ImportError: - pass - else: - _ansi_stream_wrappers = WeakKeyDictionary() - def auto_wrap_for_ansi(stream, color=None): - """This function wraps a stream so that calls through colorama - are issued to the win32 console API to recolor on demand. It - also ensures to reset the colors if a write call is interrupted - to not destroy the console afterwards. - """ + strip = should_strip_ansi(stream, color) + ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip) + rv = t.cast(t.TextIO, ansi_wrapper.stream) + _write = rv.write + + def _safe_write(s): try: - cached = _ansi_stream_wrappers.get(stream) - except Exception: - cached = None - if cached is not None: - return cached - strip = should_strip_ansi(stream, color) - ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip) - rv = ansi_wrapper.stream - _write = rv.write + return _write(s) + except BaseException: + ansi_wrapper.reset_all() + raise - def _safe_write(s): - try: - return _write(s) - except: - ansi_wrapper.reset_all() - raise + rv.write = _safe_write - rv.write = _safe_write - try: - _ansi_stream_wrappers[stream] = rv - except Exception: - pass - return rv - - def get_winterm_size(): - win = colorama.win32.GetConsoleScreenBufferInfo( - colorama.win32.STDOUT - ).srWindow - return win.Right - win.Left, win.Bottom - win.Top + try: + _ansi_stream_wrappers[stream] = rv + except Exception: + pass + return rv else: - def _get_argv_encoding(): - return getattr(sys.stdin, "encoding", None) or get_filesystem_encoding() + def _get_argv_encoding() -> str: + return getattr(sys.stdin, "encoding", None) or sys.getfilesystemencoding() - _get_windows_console_stream = lambda *x: None - _wrap_std_stream = lambda *x: None + def _get_windows_console_stream( + f: t.TextIO, encoding: t.Optional[str], errors: t.Optional[str] + ) -> t.Optional[t.TextIO]: + return None -def term_len(x): +def term_len(x: str) -> int: return len(strip_ansi(x)) -def isatty(stream): +def isatty(stream: t.IO[t.Any]) -> bool: try: return stream.isatty() except Exception: return False -def _make_cached_stream_func(src_func, wrapper_func): - cache = WeakKeyDictionary() +def _make_cached_stream_func( + src_func: t.Callable[[], t.Optional[t.TextIO]], + wrapper_func: t.Callable[[], t.TextIO], +) -> t.Callable[[], t.Optional[t.TextIO]]: + cache: t.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary() - def func(): + def func() -> t.Optional[t.TextIO]: stream = src_func() + + if stream is None: + return None + try: rv = cache.get(stream) except Exception: @@ -759,7 +595,6 @@ def _make_cached_stream_func(src_func, wrapper_func): return rv rv = wrapper_func() try: - stream = src_func() # In case wrapper_func() modified the stream cache[stream] = rv except Exception: pass @@ -773,13 +608,15 @@ _default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_std _default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr) -binary_streams = { +binary_streams: t.Mapping[str, t.Callable[[], t.BinaryIO]] = { "stdin": get_binary_stdin, "stdout": get_binary_stdout, "stderr": get_binary_stderr, } -text_streams = { +text_streams: t.Mapping[ + str, t.Callable[[t.Optional[str], t.Optional[str]], t.TextIO] +] = { "stdin": get_text_stdin, "stdout": get_text_stdout, "stderr": get_text_stderr, diff --git a/third_party/python/click/click/_termui_impl.py b/third_party/python/click/click/_termui_impl.py index 88bec37701cf..f744657753ca 100644 --- a/third_party/python/click/click/_termui_impl.py +++ b/third_party/python/click/click/_termui_impl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This module contains implementations for the termui module. To keep the import time of Click down, some infrequently used functionality is @@ -9,20 +8,24 @@ import math import os import sys import time +import typing as t +from gettext import gettext as _ +from io import StringIO +from types import TracebackType from ._compat import _default_text_stdout from ._compat import CYGWIN from ._compat import get_best_encoding -from ._compat import int_types from ._compat import isatty from ._compat import open_stream -from ._compat import range_type from ._compat import strip_ansi from ._compat import term_len from ._compat import WIN from .exceptions import ClickException from .utils import echo +V = t.TypeVar("V") + if os.name == "nt": BEFORE_BAR = "\r" AFTER_BAR = "\n" @@ -31,42 +34,25 @@ else: AFTER_BAR = "\033[?25h\n" -def _length_hint(obj): - """Returns the length hint of an object.""" - try: - return len(obj) - except (AttributeError, TypeError): - try: - get_hint = type(obj).__length_hint__ - except AttributeError: - return None - try: - hint = get_hint(obj) - except TypeError: - return None - if hint is NotImplemented or not isinstance(hint, int_types) or hint < 0: - return None - return hint - - -class ProgressBar(object): +class ProgressBar(t.Generic[V]): def __init__( self, - iterable, - length=None, - fill_char="#", - empty_char=" ", - bar_template="%(bar)s", - info_sep=" ", - show_eta=True, - show_percent=None, - show_pos=False, - item_show_func=None, - label=None, - file=None, - color=None, - width=30, - ): + iterable: t.Optional[t.Iterable[V]], + length: t.Optional[int] = None, + fill_char: str = "#", + empty_char: str = " ", + bar_template: str = "%(bar)s", + info_sep: str = " ", + show_eta: bool = True, + show_percent: t.Optional[bool] = None, + show_pos: bool = False, + item_show_func: t.Optional[t.Callable[[t.Optional[V]], t.Optional[str]]] = None, + label: t.Optional[str] = None, + file: t.Optional[t.TextIO] = None, + color: t.Optional[bool] = None, + update_min_steps: int = 1, + width: int = 30, + ) -> None: self.fill_char = fill_char self.empty_char = empty_char self.bar_template = bar_template @@ -75,50 +61,69 @@ class ProgressBar(object): self.show_percent = show_percent self.show_pos = show_pos self.item_show_func = item_show_func - self.label = label or "" + self.label: str = label or "" + if file is None: file = _default_text_stdout() + + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if file is None: + file = StringIO() + self.file = file self.color = color - self.width = width - self.autowidth = width == 0 + self.update_min_steps = update_min_steps + self._completed_intervals = 0 + self.width: int = width + self.autowidth: bool = width == 0 if length is None: - length = _length_hint(iterable) + from operator import length_hint + + length = length_hint(iterable, -1) + + if length == -1: + length = None if iterable is None: if length is None: raise TypeError("iterable or length is required") - iterable = range_type(length) - self.iter = iter(iterable) + iterable = t.cast(t.Iterable[V], range(length)) + self.iter: t.Iterable[V] = iter(iterable) self.length = length - self.length_known = length is not None self.pos = 0 - self.avg = [] + self.avg: t.List[float] = [] + self.last_eta: float + self.start: float self.start = self.last_eta = time.time() - self.eta_known = False - self.finished = False - self.max_width = None - self.entered = False - self.current_item = None - self.is_hidden = not isatty(self.file) - self._last_line = None - self.short_limit = 0.5 + self.eta_known: bool = False + self.finished: bool = False + self.max_width: t.Optional[int] = None + self.entered: bool = False + self.current_item: t.Optional[V] = None + self.is_hidden: bool = not isatty(self.file) + self._last_line: t.Optional[str] = None - def __enter__(self): + def __enter__(self) -> "ProgressBar[V]": self.entered = True self.render_progress() return self - def __exit__(self, exc_type, exc_value, tb): + def __exit__( + self, + exc_type: t.Optional[t.Type[BaseException]], + exc_value: t.Optional[BaseException], + tb: t.Optional[TracebackType], + ) -> None: self.render_finish() - def __iter__(self): + def __iter__(self) -> t.Iterator[V]: if not self.entered: raise RuntimeError("You need to use progress bars in a with block.") self.render_progress() return self.generator() - def __next__(self): + def __next__(self) -> V: # Iteration is defined in terms of a generator function, # returned by iter(self); use that to define next(). This works # because `self.iter` is an iterable consumed by that generator, @@ -126,37 +131,31 @@ class ProgressBar(object): # twice works and does "what you want". return next(iter(self)) - # Python 2 compat - next = __next__ - - def is_fast(self): - return time.time() - self.start <= self.short_limit - - def render_finish(self): - if self.is_hidden or self.is_fast(): + def render_finish(self) -> None: + if self.is_hidden: return self.file.write(AFTER_BAR) self.file.flush() @property - def pct(self): + def pct(self) -> float: if self.finished: return 1.0 - return min(self.pos / (float(self.length) or 1), 1.0) + return min(self.pos / (float(self.length or 1) or 1), 1.0) @property - def time_per_iteration(self): + def time_per_iteration(self) -> float: if not self.avg: return 0.0 return sum(self.avg) / float(len(self.avg)) @property - def eta(self): - if self.length_known and not self.finished: + def eta(self) -> float: + if self.length is not None and not self.finished: return self.time_per_iteration * (self.length - self.pos) return 0.0 - def format_eta(self): + def format_eta(self) -> str: if self.eta_known: t = int(self.eta) seconds = t % 60 @@ -166,44 +165,44 @@ class ProgressBar(object): hours = t % 24 t //= 24 if t > 0: - return "{}d {:02}:{:02}:{:02}".format(t, hours, minutes, seconds) + return f"{t}d {hours:02}:{minutes:02}:{seconds:02}" else: - return "{:02}:{:02}:{:02}".format(hours, minutes, seconds) + return f"{hours:02}:{minutes:02}:{seconds:02}" return "" - def format_pos(self): + def format_pos(self) -> str: pos = str(self.pos) - if self.length_known: - pos += "/{}".format(self.length) + if self.length is not None: + pos += f"/{self.length}" return pos - def format_pct(self): - return "{: 4}%".format(int(self.pct * 100))[1:] + def format_pct(self) -> str: + return f"{int(self.pct * 100): 4}%"[1:] - def format_bar(self): - if self.length_known: + def format_bar(self) -> str: + if self.length is not None: bar_length = int(self.pct * self.width) bar = self.fill_char * bar_length bar += self.empty_char * (self.width - bar_length) elif self.finished: bar = self.fill_char * self.width else: - bar = list(self.empty_char * (self.width or 1)) + chars = list(self.empty_char * (self.width or 1)) if self.time_per_iteration != 0: - bar[ + chars[ int( (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5) * self.width ) ] = self.fill_char - bar = "".join(bar) + bar = "".join(chars) return bar - def format_progress_line(self): + def format_progress_line(self) -> str: show_percent = self.show_percent info_bits = [] - if self.length_known and show_percent is None: + if self.length is not None and show_percent is None: show_percent = not self.show_pos if self.show_pos: @@ -226,10 +225,16 @@ class ProgressBar(object): } ).rstrip() - def render_progress(self): - from .termui import get_terminal_size + def render_progress(self) -> None: + import shutil if self.is_hidden: + # Only output the label as it changes if the output is not a + # TTY. Use file=stderr if you expect to be piping stdout. + if self._last_line != self.label: + self._last_line = self.label + echo(self.label, file=self.file, color=self.color) + return buf = [] @@ -238,10 +243,10 @@ class ProgressBar(object): old_width = self.width self.width = 0 clutter_length = term_len(self.format_progress_line()) - new_width = max(0, get_terminal_size()[0] - clutter_length) + new_width = max(0, shutil.get_terminal_size().columns - clutter_length) if new_width < old_width: buf.append(BEFORE_BAR) - buf.append(" " * self.max_width) + buf.append(" " * self.max_width) # type: ignore self.max_width = new_width self.width = new_width @@ -260,14 +265,14 @@ class ProgressBar(object): line = "".join(buf) # Render the line only if it changed. - if line != self._last_line and not self.is_fast(): + if line != self._last_line: self._last_line = line echo(line, file=self.file, color=self.color, nl=False) self.file.flush() - def make_step(self, n_steps): + def make_step(self, n_steps: int) -> None: self.pos += n_steps - if self.length_known and self.pos >= self.length: + if self.length is not None and self.pos >= self.length: self.finished = True if (time.time() - self.last_eta) < 1.0: @@ -285,18 +290,40 @@ class ProgressBar(object): self.avg = self.avg[-6:] + [step] - self.eta_known = self.length_known + self.eta_known = self.length is not None - def update(self, n_steps): - self.make_step(n_steps) - self.render_progress() + def update(self, n_steps: int, current_item: t.Optional[V] = None) -> None: + """Update the progress bar by advancing a specified number of + steps, and optionally set the ``current_item`` for this new + position. - def finish(self): - self.eta_known = 0 + :param n_steps: Number of steps to advance. + :param current_item: Optional item to set as ``current_item`` + for the updated position. + + .. versionchanged:: 8.0 + Added the ``current_item`` optional parameter. + + .. versionchanged:: 8.0 + Only render when the number of steps meets the + ``update_min_steps`` threshold. + """ + if current_item is not None: + self.current_item = current_item + + self._completed_intervals += n_steps + + if self._completed_intervals >= self.update_min_steps: + self.make_step(self._completed_intervals) + self.render_progress() + self._completed_intervals = 0 + + def finish(self) -> None: + self.eta_known = False self.current_item = None self.finished = True - def generator(self): + def generator(self) -> t.Iterator[V]: """Return a generator which yields the items added to the bar during construction, and updates the progress bar *after* the yielded block returns. @@ -312,20 +339,33 @@ class ProgressBar(object): raise RuntimeError("You need to use progress bars in a with block.") if self.is_hidden: - for rv in self.iter: - yield rv + yield from self.iter else: for rv in self.iter: self.current_item = rv + + # This allows show_item_func to be updated before the + # item is processed. Only trigger at the beginning of + # the update interval. + if self._completed_intervals == 0: + self.render_progress() + yield rv self.update(1) + self.finish() self.render_progress() -def pager(generator, color=None): +def pager(generator: t.Iterable[str], color: t.Optional[bool] = None) -> None: """Decide what method to use for paging through text.""" stdout = _default_text_stdout() + + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if stdout is None: + stdout = StringIO() + if not isatty(sys.stdin) or not isatty(stdout): return _nullpager(stdout, generator, color) pager_cmd = (os.environ.get("PAGER", None) or "").strip() @@ -345,14 +385,14 @@ def pager(generator, color=None): fd, filename = tempfile.mkstemp() os.close(fd) try: - if hasattr(os, "system") and os.system('more "{}"'.format(filename)) == 0: + if hasattr(os, "system") and os.system(f'more "{filename}"') == 0: return _pipepager(generator, "more", color) return _nullpager(stdout, generator, color) finally: os.unlink(filename) -def _pipepager(generator, cmd, color): +def _pipepager(generator: t.Iterable[str], cmd: str, color: t.Optional[bool]) -> None: """Page through text by feeding it to another program. Invoking a pager through this might support colors. """ @@ -364,7 +404,7 @@ def _pipepager(generator, cmd, color): # condition that cmd_detail = cmd.rsplit("/", 1)[-1].split() if color is None and cmd_detail[0] == "less": - less_flags = "{}{}".format(os.environ.get("LESS", ""), " ".join(cmd_detail[1:])) + less_flags = f"{os.environ.get('LESS', '')}{' '.join(cmd_detail[1:])}" if not less_flags: env["LESS"] = "-R" color = True @@ -372,17 +412,18 @@ def _pipepager(generator, cmd, color): color = True c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, env=env) - encoding = get_best_encoding(c.stdin) + stdin = t.cast(t.BinaryIO, c.stdin) + encoding = get_best_encoding(stdin) try: for text in generator: if not color: text = strip_ansi(text) - c.stdin.write(text.encode(encoding, "replace")) - except (IOError, KeyboardInterrupt): + stdin.write(text.encode(encoding, "replace")) + except (OSError, KeyboardInterrupt): pass else: - c.stdin.close() + stdin.close() # Less doesn't respect ^C, but catches it for its own UI purposes (aborting # search or other commands inside less). @@ -401,11 +442,13 @@ def _pipepager(generator, cmd, color): break -def _tempfilepager(generator, cmd, color): +def _tempfilepager( + generator: t.Iterable[str], cmd: str, color: t.Optional[bool] +) -> None: """Page through text by invoking a program on a temporary file.""" import tempfile - filename = tempfile.mktemp() + fd, filename = tempfile.mkstemp() # TODO: This never terminates if the passed generator never terminates. text = "".join(generator) if not color: @@ -414,12 +457,15 @@ def _tempfilepager(generator, cmd, color): with open_stream(filename, "wb")[0] as f: f.write(text.encode(encoding)) try: - os.system('{} "{}"'.format(cmd, filename)) + os.system(f'{cmd} "{filename}"') finally: + os.close(fd) os.unlink(filename) -def _nullpager(stream, generator, color): +def _nullpager( + stream: t.TextIO, generator: t.Iterable[str], color: t.Optional[bool] +) -> None: """Simply print unformatted text. This is the ultimate fallback.""" for text in generator: if not color: @@ -427,14 +473,20 @@ def _nullpager(stream, generator, color): stream.write(text) -class Editor(object): - def __init__(self, editor=None, env=None, require_save=True, extension=".txt"): +class Editor: + def __init__( + self, + editor: t.Optional[str] = None, + env: t.Optional[t.Mapping[str, str]] = None, + require_save: bool = True, + extension: str = ".txt", + ) -> None: self.editor = editor self.env = env self.require_save = require_save self.extension = extension - def get_editor(self): + def get_editor(self) -> str: if self.editor is not None: return self.editor for key in "VISUAL", "EDITOR": @@ -444,48 +496,62 @@ class Editor(object): if WIN: return "notepad" for editor in "sensible-editor", "vim", "nano": - if os.system("which {} >/dev/null 2>&1".format(editor)) == 0: + if os.system(f"which {editor} >/dev/null 2>&1") == 0: return editor return "vi" - def edit_file(self, filename): + def edit_file(self, filename: str) -> None: import subprocess editor = self.get_editor() + environ: t.Optional[t.Dict[str, str]] = None + if self.env: environ = os.environ.copy() environ.update(self.env) - else: - environ = None + try: - c = subprocess.Popen( - '{} "{}"'.format(editor, filename), env=environ, shell=True, - ) + c = subprocess.Popen(f'{editor} "{filename}"', env=environ, shell=True) exit_code = c.wait() if exit_code != 0: - raise ClickException("{}: Editing failed!".format(editor)) + raise ClickException( + _("{editor}: Editing failed").format(editor=editor) + ) except OSError as e: - raise ClickException("{}: Editing failed: {}".format(editor, e)) + raise ClickException( + _("{editor}: Editing failed: {e}").format(editor=editor, e=e) + ) from e - def edit(self, text): + def edit(self, text: t.Optional[t.AnyStr]) -> t.Optional[t.AnyStr]: import tempfile - text = text or "" - if text and not text.endswith("\n"): - text += "\n" + if not text: + data = b"" + elif isinstance(text, (bytes, bytearray)): + data = text + else: + if text and not text.endswith("\n"): + text += "\n" + + if WIN: + data = text.replace("\n", "\r\n").encode("utf-8-sig") + else: + data = text.encode("utf-8") fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension) - try: - if WIN: - encoding = "utf-8-sig" - text = text.replace("\n", "\r\n") - else: - encoding = "utf-8" - text = text.encode(encoding) + f: t.BinaryIO - f = os.fdopen(fd, "wb") - f.write(text) - f.close() + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + + # If the filesystem resolution is 1 second, like Mac OS + # 10.12 Extended, or 2 seconds, like FAT32, and the editor + # closes very fast, require_save can fail. Set the modified + # time to be 2 seconds in the past to work around this. + os.utime(name, (os.path.getatime(name), os.path.getmtime(name) - 2)) + # Depending on the resolution, the exact value might not be + # recorded, so get the new recorded value. timestamp = os.path.getmtime(name) self.edit_file(name) @@ -493,26 +559,26 @@ class Editor(object): if self.require_save and os.path.getmtime(name) == timestamp: return None - f = open(name, "rb") - try: + with open(name, "rb") as f: rv = f.read() - finally: - f.close() - return rv.decode("utf-8-sig").replace("\r\n", "\n") + + if isinstance(text, (bytes, bytearray)): + return rv + + return rv.decode("utf-8-sig").replace("\r\n", "\n") # type: ignore finally: os.unlink(name) -def open_url(url, wait=False, locate=False): +def open_url(url: str, wait: bool = False, locate: bool = False) -> int: import subprocess - def _unquote_file(url): - try: - import urllib - except ImportError: - import urllib + def _unquote_file(url: str) -> str: + from urllib.parse import unquote + if url.startswith("file://"): - url = urllib.unquote(url[7:]) + url = unquote(url[7:]) + return url if sys.platform == "darwin": @@ -529,19 +595,21 @@ def open_url(url, wait=False, locate=False): null.close() elif WIN: if locate: - url = _unquote_file(url) - args = 'explorer /select,"{}"'.format(_unquote_file(url.replace('"', ""))) + url = _unquote_file(url.replace('"', "")) + args = f'explorer /select,"{url}"' else: - args = 'start {} "" "{}"'.format( - "/WAIT" if wait else "", url.replace('"', "") - ) + url = url.replace('"', "") + wait_str = "/WAIT" if wait else "" + args = f'start {wait_str} "" "{url}"' return os.system(args) elif CYGWIN: if locate: - url = _unquote_file(url) - args = 'cygstart "{}"'.format(os.path.dirname(url).replace('"', "")) + url = os.path.dirname(_unquote_file(url).replace('"', "")) + args = f'cygstart "{url}"' else: - args = 'cygstart {} "{}"'.format("-w" if wait else "", url.replace('"', "")) + url = url.replace('"', "") + wait_str = "-w" if wait else "" + args = f'cygstart {wait_str} "{url}"' return os.system(args) try: @@ -562,23 +630,27 @@ def open_url(url, wait=False, locate=False): return 1 -def _translate_ch_to_exc(ch): - if ch == u"\x03": +def _translate_ch_to_exc(ch: str) -> t.Optional[BaseException]: + if ch == "\x03": raise KeyboardInterrupt() - if ch == u"\x04" and not WIN: # Unix-like, Ctrl+D + + if ch == "\x04" and not WIN: # Unix-like, Ctrl+D raise EOFError() - if ch == u"\x1a" and WIN: # Windows, Ctrl+Z + + if ch == "\x1a" and WIN: # Windows, Ctrl+Z raise EOFError() + return None + if WIN: import msvcrt @contextlib.contextmanager - def raw_terminal(): - yield + def raw_terminal() -> t.Iterator[int]: + yield -1 - def getchar(echo): + def getchar(echo: bool) -> str: # The function `getch` will return a bytes object corresponding to # the pressed character. Since Windows 10 build 1803, it will also # return \x00 when called a second time after pressing a regular key. @@ -608,50 +680,60 @@ if WIN: # # Anyway, Click doesn't claim to do this Right(tm), and using `getwch` # is doing the right thing in more situations than with `getch`. + func: t.Callable[[], str] + if echo: - func = msvcrt.getwche + func = msvcrt.getwche # type: ignore else: - func = msvcrt.getwch + func = msvcrt.getwch # type: ignore rv = func() - if rv in (u"\x00", u"\xe0"): + + if rv in ("\x00", "\xe0"): # \x00 and \xe0 are control characters that indicate special key, # see above. rv += func() + _translate_ch_to_exc(rv) return rv - else: import tty import termios @contextlib.contextmanager - def raw_terminal(): + def raw_terminal() -> t.Iterator[int]: + f: t.Optional[t.TextIO] + fd: int + if not isatty(sys.stdin): f = open("/dev/tty") fd = f.fileno() else: fd = sys.stdin.fileno() f = None + try: old_settings = termios.tcgetattr(fd) + try: tty.setraw(fd) yield fd finally: termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) sys.stdout.flush() + if f is not None: f.close() except termios.error: pass - def getchar(echo): + def getchar(echo: bool) -> str: with raw_terminal() as fd: - ch = os.read(fd, 32) - ch = ch.decode(get_best_encoding(sys.stdin), "replace") + ch = os.read(fd, 32).decode(get_best_encoding(sys.stdin), "replace") + if echo and isatty(sys.stdout): sys.stdout.write(ch) + _translate_ch_to_exc(ch) return ch diff --git a/third_party/python/click/click/_textwrap.py b/third_party/python/click/click/_textwrap.py index 6959087b7f31..b47dcbd4264e 100644 --- a/third_party/python/click/click/_textwrap.py +++ b/third_party/python/click/click/_textwrap.py @@ -1,9 +1,16 @@ import textwrap +import typing as t from contextlib import contextmanager class TextWrapper(textwrap.TextWrapper): - def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): + def _handle_long_word( + self, + reversed_chunks: t.List[str], + cur_line: t.List[str], + cur_len: int, + width: int, + ) -> None: space_left = max(width - cur_len, 1) if self.break_long_words: @@ -16,22 +23,27 @@ class TextWrapper(textwrap.TextWrapper): cur_line.append(reversed_chunks.pop()) @contextmanager - def extra_indent(self, indent): + def extra_indent(self, indent: str) -> t.Iterator[None]: old_initial_indent = self.initial_indent old_subsequent_indent = self.subsequent_indent self.initial_indent += indent self.subsequent_indent += indent + try: yield finally: self.initial_indent = old_initial_indent self.subsequent_indent = old_subsequent_indent - def indent_only(self, text): + def indent_only(self, text: str) -> str: rv = [] + for idx, line in enumerate(text.splitlines()): indent = self.initial_indent + if idx > 0: indent = self.subsequent_indent - rv.append(indent + line) + + rv.append(f"{indent}{line}") + return "\n".join(rv) diff --git a/third_party/python/click/click/_unicodefun.py b/third_party/python/click/click/_unicodefun.py deleted file mode 100644 index 781c36522726..000000000000 --- a/third_party/python/click/click/_unicodefun.py +++ /dev/null @@ -1,131 +0,0 @@ -import codecs -import os -import sys - -from ._compat import PY2 - - -def _find_unicode_literals_frame(): - import __future__ - - if not hasattr(sys, "_getframe"): # not all Python implementations have it - return 0 - frm = sys._getframe(1) - idx = 1 - while frm is not None: - if frm.f_globals.get("__name__", "").startswith("click."): - frm = frm.f_back - idx += 1 - elif frm.f_code.co_flags & __future__.unicode_literals.compiler_flag: - return idx - else: - break - return 0 - - -def _check_for_unicode_literals(): - if not __debug__: - return - - from . import disable_unicode_literals_warning - - if not PY2 or disable_unicode_literals_warning: - return - bad_frame = _find_unicode_literals_frame() - if bad_frame <= 0: - return - from warnings import warn - - warn( - Warning( - "Click detected the use of the unicode_literals __future__" - " import. This is heavily discouraged because it can" - " introduce subtle bugs in your code. You should instead" - ' use explicit u"" literals for your unicode strings. For' - " more information see" - " https://click.palletsprojects.com/python3/" - ), - stacklevel=bad_frame, - ) - - -def _verify_python3_env(): - """Ensures that the environment is good for unicode on Python 3.""" - if PY2: - return - try: - import locale - - fs_enc = codecs.lookup(locale.getpreferredencoding()).name - except Exception: - fs_enc = "ascii" - if fs_enc != "ascii": - return - - extra = "" - if os.name == "posix": - import subprocess - - try: - rv = subprocess.Popen( - ["locale", "-a"], stdout=subprocess.PIPE, stderr=subprocess.PIPE - ).communicate()[0] - except OSError: - rv = b"" - good_locales = set() - has_c_utf8 = False - - # Make sure we're operating on text here. - if isinstance(rv, bytes): - rv = rv.decode("ascii", "replace") - - for line in rv.splitlines(): - locale = line.strip() - if locale.lower().endswith((".utf-8", ".utf8")): - good_locales.add(locale) - if locale.lower() in ("c.utf8", "c.utf-8"): - has_c_utf8 = True - - extra += "\n\n" - if not good_locales: - extra += ( - "Additional information: on this system no suitable" - " UTF-8 locales were discovered. This most likely" - " requires resolving by reconfiguring the locale" - " system." - ) - elif has_c_utf8: - extra += ( - "This system supports the C.UTF-8 locale which is" - " recommended. You might be able to resolve your issue" - " by exporting the following environment variables:\n\n" - " export LC_ALL=C.UTF-8\n" - " export LANG=C.UTF-8" - ) - else: - extra += ( - "This system lists a couple of UTF-8 supporting locales" - " that you can pick from. The following suitable" - " locales were discovered: {}".format(", ".join(sorted(good_locales))) - ) - - bad_locale = None - for locale in os.environ.get("LC_ALL"), os.environ.get("LANG"): - if locale and locale.lower().endswith((".utf-8", ".utf8")): - bad_locale = locale - if locale is not None: - break - if bad_locale is not None: - extra += ( - "\n\nClick discovered that you exported a UTF-8 locale" - " but the locale system could not pick up from it" - " because it does not exist. The exported locale is" - " '{}' but it is not supported".format(bad_locale) - ) - - raise RuntimeError( - "Click will abort further execution because Python 3 was" - " configured to use ASCII as encoding for the environment." - " Consult https://click.palletsprojects.com/python3/ for" - " mitigation steps.{}".format(extra) - ) diff --git a/third_party/python/click/click/_winconsole.py b/third_party/python/click/click/_winconsole.py index b6c4274af0e8..6b20df315b23 100644 --- a/third_party/python/click/click/_winconsole.py +++ b/third_party/python/click/click/_winconsole.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # This module is based on the excellent work by Adam Bartoš who # provided a lot of what went into the implementation here in # the discussion to issue1602 in the Python bug tracker. @@ -6,13 +5,11 @@ # There are some general differences in regards to how this works # compared to the original patches as we do not need to patch # the entire interpreter but just work in our little world of -# echo and prmopt. -import ctypes +# echo and prompt. import io -import os import sys import time -import zlib +import typing as t from ctypes import byref from ctypes import c_char from ctypes import c_char_p @@ -22,28 +19,18 @@ from ctypes import c_ulong from ctypes import c_void_p from ctypes import POINTER from ctypes import py_object -from ctypes import windll -from ctypes import WinError -from ctypes import WINFUNCTYPE +from ctypes import Structure from ctypes.wintypes import DWORD from ctypes.wintypes import HANDLE from ctypes.wintypes import LPCWSTR from ctypes.wintypes import LPWSTR -import msvcrt - from ._compat import _NonClosingTextIOWrapper -from ._compat import PY2 -from ._compat import text_type - -try: - from ctypes import pythonapi - - PyObject_GetBuffer = pythonapi.PyObject_GetBuffer - PyBuffer_Release = pythonapi.PyBuffer_Release -except ImportError: - pythonapi = None +assert sys.platform == "win32" +import msvcrt # noqa: E402 +from ctypes import windll # noqa: E402 +from ctypes import WINFUNCTYPE # noqa: E402 c_ssize_p = POINTER(c_ssize_t) @@ -57,16 +44,12 @@ GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( ("CommandLineToArgvW", windll.shell32) ) -LocalFree = WINFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)( - ("LocalFree", windll.kernel32) -) - +LocalFree = WINFUNCTYPE(c_void_p, c_void_p)(("LocalFree", windll.kernel32)) STDIN_HANDLE = GetStdHandle(-10) STDOUT_HANDLE = GetStdHandle(-11) STDERR_HANDLE = GetStdHandle(-12) - PyBUF_SIMPLE = 0 PyBUF_WRITABLE = 1 @@ -81,36 +64,37 @@ STDERR_FILENO = 2 EOF = b"\x1a" MAX_BYTES_WRITTEN = 32767 - -class Py_buffer(ctypes.Structure): - _fields_ = [ - ("buf", c_void_p), - ("obj", py_object), - ("len", c_ssize_t), - ("itemsize", c_ssize_t), - ("readonly", c_int), - ("ndim", c_int), - ("format", c_char_p), - ("shape", c_ssize_p), - ("strides", c_ssize_p), - ("suboffsets", c_ssize_p), - ("internal", c_void_p), - ] - - if PY2: - _fields_.insert(-1, ("smalltable", c_ssize_t * 2)) - - -# On PyPy we cannot get buffers so our ability to operate here is -# serverly limited. -if pythonapi is None: +try: + from ctypes import pythonapi +except ImportError: + # On PyPy we cannot get buffers so our ability to operate here is + # severely limited. get_buffer = None else: + class Py_buffer(Structure): + _fields_ = [ + ("buf", c_void_p), + ("obj", py_object), + ("len", c_ssize_t), + ("itemsize", c_ssize_t), + ("readonly", c_int), + ("ndim", c_int), + ("format", c_char_p), + ("shape", c_ssize_p), + ("strides", c_ssize_p), + ("suboffsets", c_ssize_p), + ("internal", c_void_p), + ] + + PyObject_GetBuffer = pythonapi.PyObject_GetBuffer + PyBuffer_Release = pythonapi.PyBuffer_Release + def get_buffer(obj, writable=False): buf = Py_buffer() flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE PyObject_GetBuffer(py_object(obj), byref(buf), flags) + try: buffer_type = c_char * buf.len return buffer_type.from_address(buf.buf) @@ -123,7 +107,7 @@ class _WindowsConsoleRawIOBase(io.RawIOBase): self.handle = handle def isatty(self): - io.RawIOBase.isatty(self) + super().isatty() return True @@ -155,7 +139,7 @@ class _WindowsConsoleReader(_WindowsConsoleRawIOBase): # wait for KeyboardInterrupt time.sleep(0.1) if not rv: - raise OSError("Windows error: {}".format(GetLastError())) + raise OSError(f"Windows error: {GetLastError()}") if buffer[0] == EOF: return 0 @@ -172,7 +156,7 @@ class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): return "ERROR_SUCCESS" elif errno == ERROR_NOT_ENOUGH_MEMORY: return "ERROR_NOT_ENOUGH_MEMORY" - return "Windows error {}".format(errno) + return f"Windows error {errno}" def write(self, b): bytes_to_be_written = len(b) @@ -194,17 +178,17 @@ class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): return bytes_written -class ConsoleStream(object): - def __init__(self, text_stream, byte_stream): +class ConsoleStream: + def __init__(self, text_stream: t.TextIO, byte_stream: t.BinaryIO) -> None: self._text_stream = text_stream self.buffer = byte_stream @property - def name(self): + def name(self) -> str: return self.buffer.name - def write(self, x): - if isinstance(x, text_type): + def write(self, x: t.AnyStr) -> int: + if isinstance(x, str): return self._text_stream.write(x) try: self.flush() @@ -212,159 +196,84 @@ class ConsoleStream(object): pass return self.buffer.write(x) - def writelines(self, lines): + def writelines(self, lines: t.Iterable[t.AnyStr]) -> None: for line in lines: self.write(line) - def __getattr__(self, name): + def __getattr__(self, name: str) -> t.Any: return getattr(self._text_stream, name) - def isatty(self): + def isatty(self) -> bool: return self.buffer.isatty() def __repr__(self): - return "".format( - self.name, self.encoding - ) + return f"" -class WindowsChunkedWriter(object): - """ - Wraps a stream (such as stdout), acting as a transparent proxy for all - attribute access apart from method 'write()' which we wrap to write in - limited chunks due to a Windows limitation on binary console streams. - """ - - def __init__(self, wrapped): - # double-underscore everything to prevent clashes with names of - # attributes on the wrapped stream object. - self.__wrapped = wrapped - - def __getattr__(self, name): - return getattr(self.__wrapped, name) - - def write(self, text): - total_to_write = len(text) - written = 0 - - while written < total_to_write: - to_write = min(total_to_write - written, MAX_BYTES_WRITTEN) - self.__wrapped.write(text[written : written + to_write]) - written += to_write - - -_wrapped_std_streams = set() - - -def _wrap_std_stream(name): - # Python 2 & Windows 7 and below - if ( - PY2 - and sys.getwindowsversion()[:2] <= (6, 1) - and name not in _wrapped_std_streams - ): - setattr(sys, name, WindowsChunkedWriter(getattr(sys, name))) - _wrapped_std_streams.add(name) - - -def _get_text_stdin(buffer_stream): +def _get_text_stdin(buffer_stream: t.BinaryIO) -> t.TextIO: text_stream = _NonClosingTextIOWrapper( io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), "utf-16-le", "strict", line_buffering=True, ) - return ConsoleStream(text_stream, buffer_stream) + return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) -def _get_text_stdout(buffer_stream): +def _get_text_stdout(buffer_stream: t.BinaryIO) -> t.TextIO: text_stream = _NonClosingTextIOWrapper( io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), "utf-16-le", "strict", line_buffering=True, ) - return ConsoleStream(text_stream, buffer_stream) + return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) -def _get_text_stderr(buffer_stream): +def _get_text_stderr(buffer_stream: t.BinaryIO) -> t.TextIO: text_stream = _NonClosingTextIOWrapper( io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), "utf-16-le", "strict", line_buffering=True, ) - return ConsoleStream(text_stream, buffer_stream) + return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) -if PY2: - - def _hash_py_argv(): - return zlib.crc32("\x00".join(sys.argv[1:])) - - _initial_argv_hash = _hash_py_argv() - - def _get_windows_argv(): - argc = c_int(0) - argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) - if not argv_unicode: - raise WinError() - try: - argv = [argv_unicode[i] for i in range(0, argc.value)] - finally: - LocalFree(argv_unicode) - del argv_unicode - - if not hasattr(sys, "frozen"): - argv = argv[1:] - while len(argv) > 0: - arg = argv[0] - if not arg.startswith("-") or arg == "-": - break - argv = argv[1:] - if arg.startswith(("-c", "-m")): - break - - return argv[1:] - - -_stream_factories = { +_stream_factories: t.Mapping[int, t.Callable[[t.BinaryIO], t.TextIO]] = { 0: _get_text_stdin, 1: _get_text_stdout, 2: _get_text_stderr, } -def _is_console(f): +def _is_console(f: t.TextIO) -> bool: if not hasattr(f, "fileno"): return False try: fileno = f.fileno() - except OSError: + except (OSError, io.UnsupportedOperation): return False handle = msvcrt.get_osfhandle(fileno) return bool(GetConsoleMode(handle, byref(DWORD()))) -def _get_windows_console_stream(f, encoding, errors): +def _get_windows_console_stream( + f: t.TextIO, encoding: t.Optional[str], errors: t.Optional[str] +) -> t.Optional[t.TextIO]: if ( get_buffer is not None - and encoding in ("utf-16-le", None) - and errors in ("strict", None) + and encoding in {"utf-16-le", None} + and errors in {"strict", None} and _is_console(f) ): func = _stream_factories.get(f.fileno()) if func is not None: - if not PY2: - f = getattr(f, "buffer", None) - if f is None: - return None - else: - # If we are on Python 2 we need to set the stream that we - # deal with to binary mode as otherwise the exercise if a - # bit moot. The same problems apply as for - # get_binary_stdin and friends from _compat. - msvcrt.setmode(f.fileno(), os.O_BINARY) - return func(f) + b = getattr(f, "buffer", None) + + if b is None: + return None + + return func(b) diff --git a/third_party/python/click/click/core.py b/third_party/python/click/click/core.py index f58bf26d2f98..cc65e896bf2d 100644 --- a/third_party/python/click/click/core.py +++ b/third_party/python/click/click/core.py @@ -1,17 +1,19 @@ +import enum import errno import inspect import os import sys +import typing as t +from collections import abc from contextlib import contextmanager +from contextlib import ExitStack from functools import update_wrapper +from gettext import gettext as _ +from gettext import ngettext from itertools import repeat +from types import TracebackType -from ._compat import isidentifier -from ._compat import iteritems -from ._compat import PY2 -from ._compat import string_types -from ._unicodefun import _check_for_unicode_literals -from ._unicodefun import _verify_python3_env +from . import types from .exceptions import Abort from .exceptions import BadParameter from .exceptions import ClickException @@ -22,58 +24,49 @@ from .formatting import HelpFormatter from .formatting import join_options from .globals import pop_context from .globals import push_context +from .parser import _flag_needs_value from .parser import OptionParser from .parser import split_opt from .termui import confirm from .termui import prompt from .termui import style -from .types import BOOL -from .types import convert_type -from .types import IntRange +from .utils import _detect_program_name +from .utils import _expand_args from .utils import echo -from .utils import get_os_args from .utils import make_default_short_help from .utils import make_str from .utils import PacifyFlushWrapper -_missing = object() +if t.TYPE_CHECKING: + import typing_extensions as te + from .shell_completion import CompletionItem -SUBCOMMAND_METAVAR = "COMMAND [ARGS]..." -SUBCOMMANDS_METAVAR = "COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]..." - -DEPRECATED_HELP_NOTICE = " (DEPRECATED)" -DEPRECATED_INVOKE_NOTICE = "DeprecationWarning: The command %(name)s is deprecated." +F = t.TypeVar("F", bound=t.Callable[..., t.Any]) +V = t.TypeVar("V") -def _maybe_show_deprecated_notice(cmd): - if cmd.deprecated: - echo(style(DEPRECATED_INVOKE_NOTICE % {"name": cmd.name}, fg="red"), err=True) +def _complete_visible_commands( + ctx: "Context", incomplete: str +) -> t.Iterator[t.Tuple[str, "Command"]]: + """List all the subcommands of a group that start with the + incomplete value and aren't hidden. - -def fast_exit(code): - """Exit without garbage collection, this speeds up exit by about 10ms for - things like bash completion. + :param ctx: Invocation context for the group. + :param incomplete: Value being completed. May be empty. """ - sys.stdout.flush() - sys.stderr.flush() - os._exit(code) + multi = t.cast(MultiCommand, ctx.command) + + for name in multi.list_commands(ctx): + if name.startswith(incomplete): + command = multi.get_command(ctx, name) + + if command is not None and not command.hidden: + yield name, command -def _bashcomplete(cmd, prog_name, complete_var=None): - """Internal handler for the bash completion support.""" - if complete_var is None: - complete_var = "_{}_COMPLETE".format(prog_name.replace("-", "_").upper()) - complete_instr = os.environ.get(complete_var) - if not complete_instr: - return - - from ._bashcomplete import bashcomplete - - if bashcomplete(cmd, prog_name, complete_var, complete_instr): - fast_exit(1) - - -def _check_multicommand(base_command, cmd_name, cmd, register=False): +def _check_multicommand( + base_command: "MultiCommand", cmd_name: str, cmd: "Command", register: bool = False +) -> None: if not base_command.chain or not isinstance(cmd, MultiCommand): return if register: @@ -87,44 +80,22 @@ def _check_multicommand(base_command, cmd_name, cmd, register=False): " that is in chain mode. This is not supported." ) raise RuntimeError( - "{}. Command '{}' is set to chain and '{}' was added as" - " subcommand but it in itself is a multi command. ('{}' is a {}" - " within a chained {} named '{}').".format( - hint, - base_command.name, - cmd_name, - cmd_name, - cmd.__class__.__name__, - base_command.__class__.__name__, - base_command.name, - ) + f"{hint}. Command {base_command.name!r} is set to chain and" + f" {cmd_name!r} was added as a subcommand but it in itself is a" + f" multi command. ({cmd_name!r} is a {type(cmd).__name__}" + f" within a chained {type(base_command).__name__} named" + f" {base_command.name!r})." ) -def batch(iterable, batch_size): +def batch(iterable: t.Iterable[V], batch_size: int) -> t.List[t.Tuple[V, ...]]: return list(zip(*repeat(iter(iterable), batch_size))) -def invoke_param_callback(callback, ctx, param, value): - code = getattr(callback, "__code__", None) - args = getattr(code, "co_argcount", 3) - - if args < 3: - from warnings import warn - - warn( - "Parameter callbacks take 3 args, (ctx, param, value). The" - " 2-arg style is deprecated and will be removed in 8.0.".format(callback), - DeprecationWarning, - stacklevel=3, - ) - return callback(ctx, value) - - return callback(ctx, param, value) - - @contextmanager -def augment_usage_errors(ctx, param=None): +def augment_usage_errors( + ctx: "Context", param: t.Optional["Parameter"] = None +) -> t.Iterator[None]: """Context manager that attaches extra information to exceptions.""" try: yield @@ -140,23 +111,53 @@ def augment_usage_errors(ctx, param=None): raise -def iter_params_for_processing(invocation_order, declaration_order): +def iter_params_for_processing( + invocation_order: t.Sequence["Parameter"], + declaration_order: t.Sequence["Parameter"], +) -> t.List["Parameter"]: """Given a sequence of parameters in the order as should be considered for processing and an iterable of parameters that exist, this returns a list in the correct order as they should be processed. """ - def sort_key(item): + def sort_key(item: "Parameter") -> t.Tuple[bool, float]: try: - idx = invocation_order.index(item) + idx: float = invocation_order.index(item) except ValueError: idx = float("inf") - return (not item.is_eager, idx) + + return not item.is_eager, idx return sorted(declaration_order, key=sort_key) -class Context(object): +class ParameterSource(enum.Enum): + """This is an :class:`~enum.Enum` that indicates the source of a + parameter's value. + + Use :meth:`click.Context.get_parameter_source` to get the + source for a parameter by name. + + .. versionchanged:: 8.0 + Use :class:`~enum.Enum` and drop the ``validate`` method. + + .. versionchanged:: 8.0 + Added the ``PROMPT`` value. + """ + + COMMANDLINE = enum.auto() + """The value was provided by the command line args.""" + ENVIRONMENT = enum.auto() + """The value was provided with an environment variable.""" + DEFAULT = enum.auto() + """Used the default specified by the parameter.""" + DEFAULT_MAP = enum.auto() + """Used a default provided by :attr:`Context.default_map`.""" + PROMPT = enum.auto() + """Used a prompt to confirm a default or provide a value.""" + + +class Context: """The context is a special internal object that holds state relevant for the script execution at every single level. It's normally invisible to commands unless they opt-in to getting access to it. @@ -168,21 +169,6 @@ class Context(object): A context can be used as context manager in which case it will call :meth:`close` on teardown. - .. versionadded:: 2.0 - Added the `resilient_parsing`, `help_option_names`, - `token_normalize_func` parameters. - - .. versionadded:: 3.0 - Added the `allow_extra_args` and `allow_interspersed_args` - parameters. - - .. versionadded:: 4.0 - Added the `color`, `ignore_unknown_options`, and - `max_content_width` parameters. - - .. versionadded:: 7.1 - Added the `show_default` parameter. - :param command: the command class for this context. :param parent: the parent context. :param info_name: the info name for this invocation. Generally this @@ -237,60 +223,95 @@ class Context(object): codes are used in texts that Click prints which is by default not the case. This for instance would affect help output. - :param show_default: if True, shows defaults for all options. - Even if an option is later created with show_default=False, - this command-level setting overrides it. + :param show_default: Show the default value for commands. If this + value is not set, it defaults to the value from the parent + context. ``Command.show_default`` overrides this default for the + specific command. + + .. versionchanged:: 8.1 + The ``show_default`` parameter is overridden by + ``Command.show_default``, instead of the other way around. + + .. versionchanged:: 8.0 + The ``show_default`` parameter defaults to the value from the + parent context. + + .. versionchanged:: 7.1 + Added the ``show_default`` parameter. + + .. versionchanged:: 4.0 + Added the ``color``, ``ignore_unknown_options``, and + ``max_content_width`` parameters. + + .. versionchanged:: 3.0 + Added the ``allow_extra_args`` and ``allow_interspersed_args`` + parameters. + + .. versionchanged:: 2.0 + Added the ``resilient_parsing``, ``help_option_names``, and + ``token_normalize_func`` parameters. """ + #: The formatter class to create with :meth:`make_formatter`. + #: + #: .. versionadded:: 8.0 + formatter_class: t.Type["HelpFormatter"] = HelpFormatter + def __init__( self, - command, - parent=None, - info_name=None, - obj=None, - auto_envvar_prefix=None, - default_map=None, - terminal_width=None, - max_content_width=None, - resilient_parsing=False, - allow_extra_args=None, - allow_interspersed_args=None, - ignore_unknown_options=None, - help_option_names=None, - token_normalize_func=None, - color=None, - show_default=None, - ): + command: "Command", + parent: t.Optional["Context"] = None, + info_name: t.Optional[str] = None, + obj: t.Optional[t.Any] = None, + auto_envvar_prefix: t.Optional[str] = None, + default_map: t.Optional[t.MutableMapping[str, t.Any]] = None, + terminal_width: t.Optional[int] = None, + max_content_width: t.Optional[int] = None, + resilient_parsing: bool = False, + allow_extra_args: t.Optional[bool] = None, + allow_interspersed_args: t.Optional[bool] = None, + ignore_unknown_options: t.Optional[bool] = None, + help_option_names: t.Optional[t.List[str]] = None, + token_normalize_func: t.Optional[t.Callable[[str], str]] = None, + color: t.Optional[bool] = None, + show_default: t.Optional[bool] = None, + ) -> None: #: the parent context or `None` if none exists. self.parent = parent #: the :class:`Command` for this context. self.command = command #: the descriptive information name self.info_name = info_name - #: the parsed parameters except if the value is hidden in which - #: case it's not remembered. - self.params = {} + #: Map of parameter names to their parsed values. Parameters + #: with ``expose_value=False`` are not stored. + self.params: t.Dict[str, t.Any] = {} #: the leftover arguments. - self.args = [] + self.args: t.List[str] = [] #: protected arguments. These are arguments that are prepended #: to `args` when certain parsing scenarios are encountered but #: must be never propagated to another arguments. This is used #: to implement nested parsing. - self.protected_args = [] + self.protected_args: t.List[str] = [] + #: the collected prefixes of the command's options. + self._opt_prefixes: t.Set[str] = set(parent._opt_prefixes) if parent else set() + if obj is None and parent is not None: obj = parent.obj + #: the user object stored. - self.obj = obj - self._meta = getattr(parent, "meta", {}) + self.obj: t.Any = obj + self._meta: t.Dict[str, t.Any] = getattr(parent, "meta", {}) #: A dictionary (-like object) with defaults for parameters. if ( default_map is None + and info_name is not None and parent is not None and parent.default_map is not None ): default_map = parent.default_map.get(info_name) - self.default_map = default_map + + self.default_map: t.Optional[t.MutableMapping[str, t.Any]] = default_map #: This flag indicates if a subcommand is going to be executed. A #: group callback can use this information to figure out if it's @@ -301,22 +322,25 @@ class Context(object): #: If chaining is enabled this will be set to ``'*'`` in case #: any commands are executed. It is however not possible to #: figure out which ones. If you require this knowledge you - #: should use a :func:`resultcallback`. - self.invoked_subcommand = None + #: should use a :func:`result_callback`. + self.invoked_subcommand: t.Optional[str] = None if terminal_width is None and parent is not None: terminal_width = parent.terminal_width + #: The width of the terminal (None is autodetection). - self.terminal_width = terminal_width + self.terminal_width: t.Optional[int] = terminal_width if max_content_width is None and parent is not None: max_content_width = parent.max_content_width + #: The maximum width of formatted content (None implies a sensible #: default which is 80 for most things). - self.max_content_width = max_content_width + self.max_content_width: t.Optional[int] = max_content_width if allow_extra_args is None: allow_extra_args = command.allow_extra_args + #: Indicates if the context allows extra args or if it should #: fail on parsing. #: @@ -325,14 +349,16 @@ class Context(object): if allow_interspersed_args is None: allow_interspersed_args = command.allow_interspersed_args + #: Indicates if the context allows mixing of arguments and #: options or not. #: #: .. versionadded:: 3.0 - self.allow_interspersed_args = allow_interspersed_args + self.allow_interspersed_args: bool = allow_interspersed_args if ignore_unknown_options is None: ignore_unknown_options = command.ignore_unknown_options + #: Instructs click to ignore options that a command does not #: understand and will store it on the context for later #: processing. This is primarily useful for situations where you @@ -341,7 +367,7 @@ class Context(object): #: forward all arguments. #: #: .. versionadded:: 4.0 - self.ignore_unknown_options = ignore_unknown_options + self.ignore_unknown_options: bool = ignore_unknown_options if help_option_names is None: if parent is not None: @@ -350,19 +376,21 @@ class Context(object): help_option_names = ["--help"] #: The names for the help options. - self.help_option_names = help_option_names + self.help_option_names: t.List[str] = help_option_names if token_normalize_func is None and parent is not None: token_normalize_func = parent.token_normalize_func #: An optional normalization function for tokens. This is #: options, choices, commands etc. - self.token_normalize_func = token_normalize_func + self.token_normalize_func: t.Optional[ + t.Callable[[str], str] + ] = token_normalize_func #: Indicates if resilient parsing is enabled. In that case Click #: will do its best to not cause any failures and default values #: will be ignored. Useful for completion. - self.resilient_parsing = resilient_parsing + self.resilient_parsing: bool = resilient_parsing # If there is no envvar prefix yet, but the parent has one and # the command on this level has a name, we can expand the envvar @@ -373,39 +401,73 @@ class Context(object): and parent.auto_envvar_prefix is not None and self.info_name is not None ): - auto_envvar_prefix = "{}_{}".format( - parent.auto_envvar_prefix, self.info_name.upper() + auto_envvar_prefix = ( + f"{parent.auto_envvar_prefix}_{self.info_name.upper()}" ) else: auto_envvar_prefix = auto_envvar_prefix.upper() + if auto_envvar_prefix is not None: auto_envvar_prefix = auto_envvar_prefix.replace("-", "_") - self.auto_envvar_prefix = auto_envvar_prefix + + self.auto_envvar_prefix: t.Optional[str] = auto_envvar_prefix if color is None and parent is not None: color = parent.color #: Controls if styling output is wanted or not. - self.color = color + self.color: t.Optional[bool] = color - self.show_default = show_default + if show_default is None and parent is not None: + show_default = parent.show_default - self._close_callbacks = [] + #: Show option default values when formatting help text. + self.show_default: t.Optional[bool] = show_default + + self._close_callbacks: t.List[t.Callable[[], t.Any]] = [] self._depth = 0 + self._parameter_source: t.Dict[str, ParameterSource] = {} + self._exit_stack = ExitStack() - def __enter__(self): + def to_info_dict(self) -> t.Dict[str, t.Any]: + """Gather information that could be useful for a tool generating + user-facing documentation. This traverses the entire CLI + structure. + + .. code-block:: python + + with Context(cli) as ctx: + info = ctx.to_info_dict() + + .. versionadded:: 8.0 + """ + return { + "command": self.command.to_info_dict(self), + "info_name": self.info_name, + "allow_extra_args": self.allow_extra_args, + "allow_interspersed_args": self.allow_interspersed_args, + "ignore_unknown_options": self.ignore_unknown_options, + "auto_envvar_prefix": self.auto_envvar_prefix, + } + + def __enter__(self) -> "Context": self._depth += 1 push_context(self) return self - def __exit__(self, exc_type, exc_value, tb): + def __exit__( + self, + exc_type: t.Optional[t.Type[BaseException]], + exc_value: t.Optional[BaseException], + tb: t.Optional[TracebackType], + ) -> None: self._depth -= 1 if self._depth == 0: self.close() pop_context() @contextmanager - def scope(self, cleanup=True): + def scope(self, cleanup: bool = True) -> t.Iterator["Context"]: """This helper method can be used with the context object to promote it to the current thread local (see :func:`get_current_context`). The default behavior of this is to invoke the cleanup functions which @@ -443,7 +505,7 @@ class Context(object): self._depth -= 1 @property - def meta(self): + def meta(self) -> t.Dict[str, t.Any]: """This is a dictionary which is shared with all the contexts that are nested. It exists so that click utilities can store some state here if they need to. It is however the responsibility of @@ -470,32 +532,72 @@ class Context(object): """ return self._meta - def make_formatter(self): - """Creates the formatter for the help and usage output.""" - return HelpFormatter( + def make_formatter(self) -> HelpFormatter: + """Creates the :class:`~click.HelpFormatter` for the help and + usage output. + + To quickly customize the formatter class used without overriding + this method, set the :attr:`formatter_class` attribute. + + .. versionchanged:: 8.0 + Added the :attr:`formatter_class` attribute. + """ + return self.formatter_class( width=self.terminal_width, max_width=self.max_content_width ) - def call_on_close(self, f): - """This decorator remembers a function as callback that should be - executed when the context tears down. This is most useful to bind - resource handling to the script execution. For instance, file objects - opened by the :class:`File` type will register their close callbacks - here. + def with_resource(self, context_manager: t.ContextManager[V]) -> V: + """Register a resource as if it were used in a ``with`` + statement. The resource will be cleaned up when the context is + popped. - :param f: the function to execute on teardown. + Uses :meth:`contextlib.ExitStack.enter_context`. It calls the + resource's ``__enter__()`` method and returns the result. When + the context is popped, it closes the stack, which calls the + resource's ``__exit__()`` method. + + To register a cleanup function for something that isn't a + context manager, use :meth:`call_on_close`. Or use something + from :mod:`contextlib` to turn it into a context manager first. + + .. code-block:: python + + @click.group() + @click.option("--name") + @click.pass_context + def cli(ctx): + ctx.obj = ctx.with_resource(connect_db(name)) + + :param context_manager: The context manager to enter. + :return: Whatever ``context_manager.__enter__()`` returns. + + .. versionadded:: 8.0 """ - self._close_callbacks.append(f) - return f + return self._exit_stack.enter_context(context_manager) - def close(self): - """Invokes all close callbacks.""" - for cb in self._close_callbacks: - cb() - self._close_callbacks = [] + def call_on_close(self, f: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]: + """Register a function to be called when the context tears down. + + This can be used to close resources opened during the script + execution. Resources that support Python's context manager + protocol which would be used in a ``with`` statement should be + registered with :meth:`with_resource` instead. + + :param f: The function to execute on teardown. + """ + return self._exit_stack.callback(f) + + def close(self) -> None: + """Invoke all close callbacks registered with + :meth:`call_on_close`, and exit all context managers entered + with :meth:`with_resource`. + """ + self._exit_stack.close() + # In case the context is reused, create a new exit stack. + self._exit_stack = ExitStack() @property - def command_path(self): + def command_path(self) -> str: """The computed command path. This is used for the ``usage`` information on the help page. It's automatically created by combining the info names of the chain of contexts to the root. @@ -504,25 +606,35 @@ class Context(object): if self.info_name is not None: rv = self.info_name if self.parent is not None: - rv = "{} {}".format(self.parent.command_path, rv) + parent_command_path = [self.parent.command_path] + + if isinstance(self.parent.command, Command): + for param in self.parent.command.get_params(self): + parent_command_path.extend(param.get_usage_pieces(self)) + + rv = f"{' '.join(parent_command_path)} {rv}" return rv.lstrip() - def find_root(self): + def find_root(self) -> "Context": """Finds the outermost context.""" node = self while node.parent is not None: node = node.parent return node - def find_object(self, object_type): + def find_object(self, object_type: t.Type[V]) -> t.Optional[V]: """Finds the closest object of a given type.""" - node = self + node: t.Optional["Context"] = self + while node is not None: if isinstance(node.obj, object_type): return node.obj + node = node.parent - def ensure_object(self, object_type): + return None + + def ensure_object(self, object_type: t.Type[V]) -> V: """Like :meth:`find_object` but sets the innermost object to a new instance of `object_type` if it does not exist. """ @@ -531,17 +643,39 @@ class Context(object): self.obj = rv = object_type() return rv - def lookup_default(self, name): - """Looks up the default for a parameter name. This by default - looks into the :attr:`default_map` if available. + @t.overload + def lookup_default( + self, name: str, call: "te.Literal[True]" = True + ) -> t.Optional[t.Any]: + ... + + @t.overload + def lookup_default( + self, name: str, call: "te.Literal[False]" = ... + ) -> t.Optional[t.Union[t.Any, t.Callable[[], t.Any]]]: + ... + + def lookup_default(self, name: str, call: bool = True) -> t.Optional[t.Any]: + """Get the default for a parameter from :attr:`default_map`. + + :param name: Name of the parameter. + :param call: If the default is a callable, call it. Disable to + return the callable instead. + + .. versionchanged:: 8.0 + Added the ``call`` parameter. """ if self.default_map is not None: - rv = self.default_map.get(name) - if callable(rv): - rv = rv() - return rv + value = self.default_map.get(name) - def fail(self, message): + if call and callable(value): + return value() + + return value + + return None + + def fail(self, message: str) -> "te.NoReturn": """Aborts the execution of the program with a specific error message. @@ -549,27 +683,58 @@ class Context(object): """ raise UsageError(message, self) - def abort(self): + def abort(self) -> "te.NoReturn": """Aborts the script.""" raise Abort() - def exit(self, code=0): + def exit(self, code: int = 0) -> "te.NoReturn": """Exits the application with a given exit code.""" raise Exit(code) - def get_usage(self): + def get_usage(self) -> str: """Helper method to get formatted usage string for the current context and command. """ return self.command.get_usage(self) - def get_help(self): + def get_help(self) -> str: """Helper method to get formatted help page for the current context and command. """ return self.command.get_help(self) - def invoke(*args, **kwargs): # noqa: B902 + def _make_sub_context(self, command: "Command") -> "Context": + """Create a new context of the same type as this context, but + for a new command. + + :meta private: + """ + return type(self)(command, info_name=command.name, parent=self) + + @t.overload + def invoke( + __self, # noqa: B902 + __callback: "t.Callable[..., V]", + *args: t.Any, + **kwargs: t.Any, + ) -> V: + ... + + @t.overload + def invoke( + __self, # noqa: B902 + __callback: "Command", + *args: t.Any, + **kwargs: t.Any, + ) -> t.Any: + ... + + def invoke( + __self, # noqa: B902 + __callback: t.Union["Command", "t.Callable[..., V]"], + *args: t.Any, + **kwargs: t.Any, + ) -> t.Union[t.Any, V]: """Invokes a command callback in exactly the way it expects. There are two ways to invoke this method: @@ -584,51 +749,89 @@ class Context(object): in against the intention of this code and no context was created. For more information about this change and why it was done in a bugfix release see :ref:`upgrade-to-3.2`. - """ - self, callback = args[:2] - ctx = self - # It's also possible to invoke another command which might or - # might not have a callback. In that case we also fill - # in defaults and make a new context for this command. - if isinstance(callback, Command): - other_cmd = callback - callback = other_cmd.callback - ctx = Context(other_cmd, info_name=other_cmd.name, parent=self) - if callback is None: + .. versionchanged:: 8.0 + All ``kwargs`` are tracked in :attr:`params` so they will be + passed if :meth:`forward` is called at multiple levels. + """ + if isinstance(__callback, Command): + other_cmd = __callback + + if other_cmd.callback is None: raise TypeError( "The given command does not have a callback that can be invoked." ) + else: + __callback = t.cast("t.Callable[..., V]", other_cmd.callback) + + ctx = __self._make_sub_context(other_cmd) for param in other_cmd.params: if param.name not in kwargs and param.expose_value: - kwargs[param.name] = param.get_default(ctx) + kwargs[param.name] = param.type_cast_value( # type: ignore + ctx, param.get_default(ctx) + ) - args = args[2:] - with augment_usage_errors(self): + # Track all kwargs as params, so that forward() will pass + # them on in subsequent calls. + ctx.params.update(kwargs) + else: + ctx = __self + + with augment_usage_errors(__self): with ctx: - return callback(*args, **kwargs) + return __callback(*args, **kwargs) - def forward(*args, **kwargs): # noqa: B902 + def forward( + __self, __cmd: "Command", *args: t.Any, **kwargs: t.Any # noqa: B902 + ) -> t.Any: """Similar to :meth:`invoke` but fills in default keyword arguments from the current context if the other command expects it. This cannot invoke callbacks directly, only other commands. - """ - self, cmd = args[:2] - # It's also possible to invoke another command which might or - # might not have a callback. - if not isinstance(cmd, Command): + .. versionchanged:: 8.0 + All ``kwargs`` are tracked in :attr:`params` so they will be + passed if ``forward`` is called at multiple levels. + """ + # Can only forward to other commands, not direct callbacks. + if not isinstance(__cmd, Command): raise TypeError("Callback is not a command.") - for param in self.params: + for param in __self.params: if param not in kwargs: - kwargs[param] = self.params[param] + kwargs[param] = __self.params[param] - return self.invoke(cmd, **kwargs) + return __self.invoke(__cmd, *args, **kwargs) + + def set_parameter_source(self, name: str, source: ParameterSource) -> None: + """Set the source of a parameter. This indicates the location + from which the value of the parameter was obtained. + + :param name: The name of the parameter. + :param source: A member of :class:`~click.core.ParameterSource`. + """ + self._parameter_source[name] = source + + def get_parameter_source(self, name: str) -> t.Optional[ParameterSource]: + """Get the source of a parameter. This indicates the location + from which the value of the parameter was obtained. + + This can be useful for determining when a user specified a value + on the command line that is the same as the default value. It + will be :attr:`~click.core.ParameterSource.DEFAULT` only if the + value was actually taken from the default. + + :param name: The name of the parameter. + :rtype: ParameterSource + + .. versionchanged:: 8.0 + Returns ``None`` if the parameter was not provided from any + source. + """ + return self._parameter_source.get(name) -class BaseCommand(object): +class BaseCommand: """The base command implements the minimal API contract of commands. Most code will never use this as it does not implement a lot of useful functionality but it can act as the direct subclass of alternative @@ -650,6 +853,10 @@ class BaseCommand(object): passed to the context object. """ + #: The context class to create with :meth:`make_context`. + #: + #: .. versionadded:: 8.0 + context_class: t.Type[Context] = Context #: the default for the :attr:`Context.allow_extra_args` flag. allow_extra_args = False #: the default for the :attr:`Context.allow_interspersed_args` flag. @@ -657,70 +864,158 @@ class BaseCommand(object): #: the default for the :attr:`Context.ignore_unknown_options` flag. ignore_unknown_options = False - def __init__(self, name, context_settings=None): + def __init__( + self, + name: t.Optional[str], + context_settings: t.Optional[t.MutableMapping[str, t.Any]] = None, + ) -> None: #: the name the command thinks it has. Upon registering a command #: on a :class:`Group` the group will default the command name #: with this information. You should instead use the #: :class:`Context`\'s :attr:`~Context.info_name` attribute. self.name = name + if context_settings is None: context_settings = {} + #: an optional dictionary with defaults passed to the context. - self.context_settings = context_settings + self.context_settings: t.MutableMapping[str, t.Any] = context_settings - def __repr__(self): - return "<{} {}>".format(self.__class__.__name__, self.name) + def to_info_dict(self, ctx: Context) -> t.Dict[str, t.Any]: + """Gather information that could be useful for a tool generating + user-facing documentation. This traverses the entire structure + below this command. - def get_usage(self, ctx): + Use :meth:`click.Context.to_info_dict` to traverse the entire + CLI structure. + + :param ctx: A :class:`Context` representing this command. + + .. versionadded:: 8.0 + """ + return {"name": self.name} + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + def get_usage(self, ctx: Context) -> str: raise NotImplementedError("Base commands cannot get usage") - def get_help(self, ctx): + def get_help(self, ctx: Context) -> str: raise NotImplementedError("Base commands cannot get help") - def make_context(self, info_name, args, parent=None, **extra): + def make_context( + self, + info_name: t.Optional[str], + args: t.List[str], + parent: t.Optional[Context] = None, + **extra: t.Any, + ) -> Context: """This function when given an info name and arguments will kick off the parsing and create a new :class:`Context`. It does not invoke the actual command callback though. - :param info_name: the info name for this invokation. Generally this + To quickly customize the context class used without overriding + this method, set the :attr:`context_class` attribute. + + :param info_name: the info name for this invocation. Generally this is the most descriptive name for the script or command. For the toplevel script it's usually - the name of the script, for commands below it it's - the name of the script. + the name of the script, for commands below it's + the name of the command. :param args: the arguments to parse as list of strings. :param parent: the parent context if available. :param extra: extra keyword arguments forwarded to the context constructor. + + .. versionchanged:: 8.0 + Added the :attr:`context_class` attribute. """ - for key, value in iteritems(self.context_settings): + for key, value in self.context_settings.items(): if key not in extra: extra[key] = value - ctx = Context(self, info_name=info_name, parent=parent, **extra) + + ctx = self.context_class( + self, info_name=info_name, parent=parent, **extra # type: ignore + ) + with ctx.scope(cleanup=False): self.parse_args(ctx, args) return ctx - def parse_args(self, ctx, args): + def parse_args(self, ctx: Context, args: t.List[str]) -> t.List[str]: """Given a context and a list of arguments this creates the parser and parses the arguments, then modifies the context as necessary. This is automatically invoked by :meth:`make_context`. """ raise NotImplementedError("Base commands do not know how to parse arguments.") - def invoke(self, ctx): + def invoke(self, ctx: Context) -> t.Any: """Given a context, this invokes the command. The default implementation is raising a not implemented error. """ - raise NotImplementedError("Base commands are not invokable by default") + raise NotImplementedError("Base commands are not invocable by default") + + def shell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]: + """Return a list of completions for the incomplete value. Looks + at the names of chained multi-commands. + + Any command could be part of a chained multi-command, so sibling + commands are valid at any point during command completion. Other + command classes will return more completions. + + :param ctx: Invocation context for this command. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + results: t.List["CompletionItem"] = [] + + while ctx.parent is not None: + ctx = ctx.parent + + if isinstance(ctx.command, MultiCommand) and ctx.command.chain: + results.extend( + CompletionItem(name, help=command.get_short_help_str()) + for name, command in _complete_visible_commands(ctx, incomplete) + if name not in ctx.protected_args + ) + + return results + + @t.overload + def main( + self, + args: t.Optional[t.Sequence[str]] = None, + prog_name: t.Optional[str] = None, + complete_var: t.Optional[str] = None, + standalone_mode: "te.Literal[True]" = True, + **extra: t.Any, + ) -> "te.NoReturn": + ... + + @t.overload + def main( + self, + args: t.Optional[t.Sequence[str]] = None, + prog_name: t.Optional[str] = None, + complete_var: t.Optional[str] = None, + standalone_mode: bool = ..., + **extra: t.Any, + ) -> t.Any: + ... def main( self, - args=None, - prog_name=None, - complete_var=None, - standalone_mode=True, - **extra - ): + args: t.Optional[t.Sequence[str]] = None, + prog_name: t.Optional[str] = None, + complete_var: t.Optional[str] = None, + standalone_mode: bool = True, + windows_expand_args: bool = True, + **extra: t.Any, + ) -> t.Any: """This is the way to invoke a script with all the bells and whistles as a command line application. This will always terminate the application after a call. If this is not wanted, ``SystemExit`` @@ -729,9 +1024,6 @@ class BaseCommand(object): This method is also available by directly calling the instance of a :class:`Command`. - .. versionadded:: 3.0 - Added the `standalone_mode` flag to control the standalone mode. - :param args: the arguments that should be used for parsing. If not provided, ``sys.argv[1:]`` is used. :param prog_name: the program name that should be used. By default @@ -750,31 +1042,35 @@ class BaseCommand(object): propagated to the caller and the return value of this function is the return value of :meth:`invoke`. + :param windows_expand_args: Expand glob patterns, user dir, and + env vars in command line args on Windows. :param extra: extra keyword arguments are forwarded to the context constructor. See :class:`Context` for more information. - """ - # If we are in Python 3, we will verify that the environment is - # sane at this point or reject further execution to avoid a - # broken script. - if not PY2: - _verify_python3_env() - else: - _check_for_unicode_literals() + .. versionchanged:: 8.0.1 + Added the ``windows_expand_args`` parameter to allow + disabling command line arg expansion on Windows. + + .. versionchanged:: 8.0 + When taking arguments from ``sys.argv`` on Windows, glob + patterns, user dir, and env vars are expanded. + + .. versionchanged:: 3.0 + Added the ``standalone_mode`` parameter. + """ if args is None: - args = get_os_args() + args = sys.argv[1:] + + if os.name == "nt" and windows_expand_args: + args = _expand_args(args) else: args = list(args) if prog_name is None: - prog_name = make_str( - os.path.basename(sys.argv[0] if sys.argv else __file__) - ) + prog_name = _detect_program_name() - # Hook for the Bash completion. This only activates if the Bash - # completion is actually enabled, otherwise this is quite a fast - # noop. - _bashcomplete(self, prog_name, complete_var) + # Process shell completion requests and exit early. + self._main_shell_completion(extra, prog_name, complete_var) try: try: @@ -790,18 +1086,18 @@ class BaseCommand(object): # even always obvious that `rv` indicates success/failure # by its truthiness/falsiness ctx.exit() - except (EOFError, KeyboardInterrupt): + except (EOFError, KeyboardInterrupt) as e: echo(file=sys.stderr) - raise Abort() + raise Abort() from e except ClickException as e: if not standalone_mode: raise e.show() sys.exit(e.exit_code) - except IOError as e: + except OSError as e: if e.errno == errno.EPIPE: - sys.stdout = PacifyFlushWrapper(sys.stdout) - sys.stderr = PacifyFlushWrapper(sys.stderr) + sys.stdout = t.cast(t.TextIO, PacifyFlushWrapper(sys.stdout)) + sys.stderr = t.cast(t.TextIO, PacifyFlushWrapper(sys.stderr)) sys.exit(1) else: raise @@ -821,10 +1117,42 @@ class BaseCommand(object): except Abort: if not standalone_mode: raise - echo("Aborted!", file=sys.stderr) + echo(_("Aborted!"), file=sys.stderr) sys.exit(1) - def __call__(self, *args, **kwargs): + def _main_shell_completion( + self, + ctx_args: t.MutableMapping[str, t.Any], + prog_name: str, + complete_var: t.Optional[str] = None, + ) -> None: + """Check if the shell is asking for tab completion, process + that, then exit early. Called from :meth:`main` before the + program is invoked. + + :param prog_name: Name of the executable in the shell. + :param complete_var: Name of the environment variable that holds + the completion instruction. Defaults to + ``_{PROG_NAME}_COMPLETE``. + + .. versionchanged:: 8.2.0 + Dots (``.``) in ``prog_name`` are replaced with underscores (``_``). + """ + if complete_var is None: + complete_name = prog_name.replace("-", "_").replace(".", "_") + complete_var = f"_{complete_name}_COMPLETE".upper() + + instruction = os.environ.get(complete_var) + + if not instruction: + return + + from .shell_completion import shell_complete + + rv = shell_complete(self, ctx_args, prog_name, complete_var, instruction) + sys.exit(rv) + + def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: """Alias for :meth:`main`.""" return self.main(*args, **kwargs) @@ -834,11 +1162,6 @@ class Command(BaseCommand): Click. A basic command handles command line parsing and might dispatch more parsing to commands nested below it. - .. versionchanged:: 2.0 - Added the `context_settings` parameter. - .. versionchanged:: 7.1 - Added the `no_args_is_help` parameter. - :param name: the name of the command to use unless a group overrides it. :param context_settings: an optional dictionary with defaults that are passed to the context object. @@ -860,35 +1183,45 @@ class Command(BaseCommand): :param deprecated: issues a message indicating that the command is deprecated. + + .. versionchanged:: 8.1 + ``help``, ``epilog``, and ``short_help`` are stored unprocessed, + all formatting is done when outputting help text, not at init, + and is done even if not using the ``@command`` decorator. + + .. versionchanged:: 8.0 + Added a ``repr`` showing the command name. + + .. versionchanged:: 7.1 + Added the ``no_args_is_help`` parameter. + + .. versionchanged:: 2.0 + Added the ``context_settings`` parameter. """ def __init__( self, - name, - context_settings=None, - callback=None, - params=None, - help=None, - epilog=None, - short_help=None, - options_metavar="[OPTIONS]", - add_help_option=True, - no_args_is_help=False, - hidden=False, - deprecated=False, - ): - BaseCommand.__init__(self, name, context_settings) + name: t.Optional[str], + context_settings: t.Optional[t.MutableMapping[str, t.Any]] = None, + callback: t.Optional[t.Callable[..., t.Any]] = None, + params: t.Optional[t.List["Parameter"]] = None, + help: t.Optional[str] = None, + epilog: t.Optional[str] = None, + short_help: t.Optional[str] = None, + options_metavar: t.Optional[str] = "[OPTIONS]", + add_help_option: bool = True, + no_args_is_help: bool = False, + hidden: bool = False, + deprecated: bool = False, + ) -> None: + super().__init__(name, context_settings) #: the callback to execute when the command fires. This might be #: `None` in which case nothing happens. self.callback = callback #: the list of parameters for this command in the order they #: should show up in the help page and execute. Eager parameters #: will automatically be handled before non eager ones. - self.params = params or [] - # if a form feed (page break) is found in the help text, truncate help - # text to the content preceding the first form feed - if help and "\f" in help: - help = help.split("\f", 1)[0] + self.params: t.List["Parameter"] = params or [] self.help = help self.epilog = epilog self.options_metavar = options_metavar @@ -898,7 +1231,19 @@ class Command(BaseCommand): self.hidden = hidden self.deprecated = deprecated - def get_usage(self, ctx): + def to_info_dict(self, ctx: Context) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict(ctx) + info_dict.update( + params=[param.to_info_dict() for param in self.get_params(ctx)], + help=self.help, + epilog=self.epilog, + short_help=self.short_help, + hidden=self.hidden, + deprecated=self.deprecated, + ) + return info_dict + + def get_usage(self, ctx: Context) -> str: """Formats the usage line into a string and returns it. Calls :meth:`format_usage` internally. @@ -907,14 +1252,16 @@ class Command(BaseCommand): self.format_usage(ctx, formatter) return formatter.getvalue().rstrip("\n") - def get_params(self, ctx): + def get_params(self, ctx: Context) -> t.List["Parameter"]: rv = self.params help_option = self.get_help_option(ctx) + if help_option is not None: - rv = rv + [help_option] + rv = [*rv, help_option] + return rv - def format_usage(self, ctx, formatter): + def format_usage(self, ctx: Context, formatter: HelpFormatter) -> None: """Writes the usage line into the formatter. This is a low-level method called by :meth:`get_usage`. @@ -922,30 +1269,33 @@ class Command(BaseCommand): pieces = self.collect_usage_pieces(ctx) formatter.write_usage(ctx.command_path, " ".join(pieces)) - def collect_usage_pieces(self, ctx): + def collect_usage_pieces(self, ctx: Context) -> t.List[str]: """Returns all the pieces that go into the usage line and returns it as a list of strings. """ - rv = [self.options_metavar] + rv = [self.options_metavar] if self.options_metavar else [] + for param in self.get_params(ctx): rv.extend(param.get_usage_pieces(ctx)) + return rv - def get_help_option_names(self, ctx): + def get_help_option_names(self, ctx: Context) -> t.List[str]: """Returns the names for the help option.""" all_names = set(ctx.help_option_names) for param in self.params: all_names.difference_update(param.opts) all_names.difference_update(param.secondary_opts) - return all_names + return list(all_names) - def get_help_option(self, ctx): + def get_help_option(self, ctx: Context) -> t.Optional["Option"]: """Returns the help option object.""" help_options = self.get_help_option_names(ctx) - if not help_options or not self.add_help_option: - return - def show_help(ctx, param, value): + if not help_options or not self.add_help_option: + return None + + def show_help(ctx: Context, param: "Parameter", value: str) -> None: if value and not ctx.resilient_parsing: echo(ctx.get_help(), color=ctx.color) ctx.exit() @@ -956,17 +1306,17 @@ class Command(BaseCommand): is_eager=True, expose_value=False, callback=show_help, - help="Show this message and exit.", + help=_("Show this message and exit."), ) - def make_parser(self, ctx): + def make_parser(self, ctx: Context) -> OptionParser: """Creates the underlying option parser for this command.""" parser = OptionParser(ctx) for param in self.get_params(ctx): param.add_to_parser(parser, ctx) return parser - def get_help(self, ctx): + def get_help(self, ctx: Context) -> str: """Formats the help into a string and returns it. Calls :meth:`format_help` internally. @@ -975,18 +1325,23 @@ class Command(BaseCommand): self.format_help(ctx, formatter) return formatter.getvalue().rstrip("\n") - def get_short_help_str(self, limit=45): + def get_short_help_str(self, limit: int = 45) -> str: """Gets short help for the command or makes it by shortening the long help string. """ - return ( - self.short_help - or self.help - and make_default_short_help(self.help, limit) - or "" - ) + if self.short_help: + text = inspect.cleandoc(self.short_help) + elif self.help: + text = make_default_short_help(self.help, limit) + else: + text = "" - def format_help(self, ctx, formatter): + if self.deprecated: + text = _("(Deprecated) {text}").format(text=text) + + return text.strip() + + def format_help(self, ctx: Context, formatter: HelpFormatter) -> None: """Writes the help into the formatter if it exists. This is a low-level method called by :meth:`get_help`. @@ -1003,21 +1358,24 @@ class Command(BaseCommand): self.format_options(ctx, formatter) self.format_epilog(ctx, formatter) - def format_help_text(self, ctx, formatter): + def format_help_text(self, ctx: Context, formatter: HelpFormatter) -> None: """Writes the help text to the formatter if it exists.""" - if self.help: - formatter.write_paragraph() - with formatter.indentation(): - help_text = self.help - if self.deprecated: - help_text += DEPRECATED_HELP_NOTICE - formatter.write_text(help_text) - elif self.deprecated: - formatter.write_paragraph() - with formatter.indentation(): - formatter.write_text(DEPRECATED_HELP_NOTICE) + if self.help is not None: + # truncate the help text to the first form feed + text = inspect.cleandoc(self.help).partition("\f")[0] + else: + text = "" - def format_options(self, ctx, formatter): + if self.deprecated: + text = _("(Deprecated) {text}").format(text=text) + + if text: + formatter.write_paragraph() + + with formatter.indentation(): + formatter.write_text(text) + + def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: """Writes all the options into the formatter if they exist.""" opts = [] for param in self.get_params(ctx): @@ -1026,17 +1384,19 @@ class Command(BaseCommand): opts.append(rv) if opts: - with formatter.section("Options"): + with formatter.section(_("Options")): formatter.write_dl(opts) - def format_epilog(self, ctx, formatter): + def format_epilog(self, ctx: Context, formatter: HelpFormatter) -> None: """Writes the epilog into the formatter if it exists.""" if self.epilog: + epilog = inspect.cleandoc(self.epilog) formatter.write_paragraph() - with formatter.indentation(): - formatter.write_text(self.epilog) - def parse_args(self, ctx, args): + with formatter.indentation(): + formatter.write_text(epilog) + + def parse_args(self, ctx: Context, args: t.List[str]) -> t.List[str]: if not args and self.no_args_is_help and not ctx.resilient_parsing: echo(ctx.get_help(), color=ctx.color) ctx.exit() @@ -1049,22 +1409,65 @@ class Command(BaseCommand): if args and not ctx.allow_extra_args and not ctx.resilient_parsing: ctx.fail( - "Got unexpected extra argument{} ({})".format( - "s" if len(args) != 1 else "", " ".join(map(make_str, args)) - ) + ngettext( + "Got unexpected extra argument ({args})", + "Got unexpected extra arguments ({args})", + len(args), + ).format(args=" ".join(map(str, args))) ) ctx.args = args + ctx._opt_prefixes.update(parser._opt_prefixes) return args - def invoke(self, ctx): + def invoke(self, ctx: Context) -> t.Any: """Given a context, this invokes the attached callback (if it exists) in the right way. """ - _maybe_show_deprecated_notice(self) + if self.deprecated: + message = _( + "DeprecationWarning: The command {name!r} is deprecated." + ).format(name=self.name) + echo(style(message, fg="red"), err=True) + if self.callback is not None: return ctx.invoke(self.callback, **ctx.params) + def shell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]: + """Return a list of completions for the incomplete value. Looks + at the names of options and chained multi-commands. + + :param ctx: Invocation context for this command. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + results: t.List["CompletionItem"] = [] + + if incomplete and not incomplete[0].isalnum(): + for param in self.get_params(ctx): + if ( + not isinstance(param, Option) + or param.hidden + or ( + not param.multiple + and ctx.get_parameter_source(param.name) # type: ignore + is ParameterSource.COMMANDLINE + ) + ): + continue + + results.extend( + CompletionItem(name, help=param.help) + for name in [*param.opts, *param.secondary_opts] + if name.startswith(incomplete) + ) + + results.extend(super().shell_complete(ctx, incomplete)) + return results + class MultiCommand(Command): """A multi command is the basic implementation of a command that @@ -1086,8 +1489,10 @@ class MultiCommand(Command): is enabled. This restricts the form of commands in that they cannot have optional arguments but it allows multiple commands to be chained together. - :param result_callback: the result callback to attach to this multi - command. + :param result_callback: The result callback to attach to this multi + command. This can be set or changed later with the + :meth:`result_callback` decorator. + :param attrs: Other command arguments described in :class:`Command`. """ allow_extra_args = True @@ -1095,29 +1500,33 @@ class MultiCommand(Command): def __init__( self, - name=None, - invoke_without_command=False, - no_args_is_help=None, - subcommand_metavar=None, - chain=False, - result_callback=None, - **attrs - ): - Command.__init__(self, name, **attrs) + name: t.Optional[str] = None, + invoke_without_command: bool = False, + no_args_is_help: t.Optional[bool] = None, + subcommand_metavar: t.Optional[str] = None, + chain: bool = False, + result_callback: t.Optional[t.Callable[..., t.Any]] = None, + **attrs: t.Any, + ) -> None: + super().__init__(name, **attrs) + if no_args_is_help is None: no_args_is_help = not invoke_without_command + self.no_args_is_help = no_args_is_help self.invoke_without_command = invoke_without_command + if subcommand_metavar is None: if chain: - subcommand_metavar = SUBCOMMANDS_METAVAR + subcommand_metavar = "COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]..." else: - subcommand_metavar = SUBCOMMAND_METAVAR + subcommand_metavar = "COMMAND [ARGS]..." + self.subcommand_metavar = subcommand_metavar self.chain = chain - #: The result callback that is stored. This can be set or - #: overridden with the :func:`resultcallback` decorator. - self.result_callback = result_callback + # The result callback that is stored. This can be set or + # overridden with the :func:`result_callback` decorator. + self._result_callback = result_callback if self.chain: for param in self.params: @@ -1127,17 +1536,35 @@ class MultiCommand(Command): " optional arguments." ) - def collect_usage_pieces(self, ctx): - rv = Command.collect_usage_pieces(self, ctx) + def to_info_dict(self, ctx: Context) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict(ctx) + commands = {} + + for name in self.list_commands(ctx): + command = self.get_command(ctx, name) + + if command is None: + continue + + sub_ctx = ctx._make_sub_context(command) + + with sub_ctx.scope(cleanup=False): + commands[name] = command.to_info_dict(sub_ctx) + + info_dict.update(commands=commands, chain=self.chain) + return info_dict + + def collect_usage_pieces(self, ctx: Context) -> t.List[str]: + rv = super().collect_usage_pieces(ctx) rv.append(self.subcommand_metavar) return rv - def format_options(self, ctx, formatter): - Command.format_options(self, ctx, formatter) + def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: + super().format_options(ctx, formatter) self.format_commands(ctx, formatter) - def resultcallback(self, replace=False): - """Adds a result callback to the chain command. By default if a + def result_callback(self, replace: bool = False) -> t.Callable[[F], F]: + """Adds a result callback to the command. By default if a result callback is already registered this will chain them but this can be disabled with the `replace` parameter. The result callback is invoked with the return value of the subcommand @@ -1152,31 +1579,36 @@ class MultiCommand(Command): def cli(input): return 42 - @cli.resultcallback() + @cli.result_callback() def process_result(result, input): return result + input - .. versionadded:: 3.0 - :param replace: if set to `True` an already existing result callback will be removed. + + .. versionchanged:: 8.0 + Renamed from ``resultcallback``. + + .. versionadded:: 3.0 """ - def decorator(f): - old_callback = self.result_callback + def decorator(f: F) -> F: + old_callback = self._result_callback + if old_callback is None or replace: - self.result_callback = f + self._result_callback = f return f - def function(__value, *args, **kwargs): - return f(old_callback(__value, *args, **kwargs), *args, **kwargs) + def function(__value, *args, **kwargs): # type: ignore + inner = old_callback(__value, *args, **kwargs) + return f(inner, *args, **kwargs) - self.result_callback = rv = update_wrapper(function, f) + self._result_callback = rv = update_wrapper(t.cast(F, function), f) return rv return decorator - def format_commands(self, ctx, formatter): + def format_commands(self, ctx: Context, formatter: HelpFormatter) -> None: """Extra format methods for multi methods that adds all the commands after the options. """ @@ -1201,15 +1633,16 @@ class MultiCommand(Command): rows.append((subcommand, help)) if rows: - with formatter.section("Commands"): + with formatter.section(_("Commands")): formatter.write_dl(rows) - def parse_args(self, ctx, args): + def parse_args(self, ctx: Context, args: t.List[str]) -> t.List[str]: if not args and self.no_args_is_help and not ctx.resilient_parsing: echo(ctx.get_help(), color=ctx.color) ctx.exit() - rest = Command.parse_args(self, ctx, args) + rest = super().parse_args(ctx, args) + if self.chain: ctx.protected_args = rest ctx.args = [] @@ -1218,29 +1651,24 @@ class MultiCommand(Command): return ctx.args - def invoke(self, ctx): - def _process_result(value): - if self.result_callback is not None: - value = ctx.invoke(self.result_callback, value, **ctx.params) + def invoke(self, ctx: Context) -> t.Any: + def _process_result(value: t.Any) -> t.Any: + if self._result_callback is not None: + value = ctx.invoke(self._result_callback, value, **ctx.params) return value if not ctx.protected_args: - # If we are invoked without command the chain flag controls - # how this happens. If we are not in chain mode, the return - # value here is the return value of the command. - # If however we are in chain mode, the return value is the - # return value of the result processor invoked with an empty - # list (which means that no subcommand actually was executed). if self.invoke_without_command: - if not self.chain: - return Command.invoke(self, ctx) + # No subcommand was invoked, so the result callback is + # invoked with the group return value for regular + # groups, or an empty list for chained groups. with ctx: - Command.invoke(self, ctx) - return _process_result([]) - ctx.fail("Missing command.") + rv = super().invoke(ctx) + return _process_result([] if self.chain else rv) + ctx.fail(_("Missing command.")) # Fetch args back out - args = ctx.protected_args + ctx.args + args = [*ctx.protected_args, *ctx.args] ctx.args = [] ctx.protected_args = [] @@ -1252,8 +1680,9 @@ class MultiCommand(Command): # resources until the result processor has worked. with ctx: cmd_name, cmd, args = self.resolve_command(ctx, args) + assert cmd is not None ctx.invoked_subcommand = cmd_name - Command.invoke(self, ctx) + super().invoke(ctx) sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) with sub_ctx: return _process_result(sub_ctx.command.invoke(sub_ctx)) @@ -1265,7 +1694,7 @@ class MultiCommand(Command): # but nothing else. with ctx: ctx.invoked_subcommand = "*" if args else None - Command.invoke(self, ctx) + super().invoke(ctx) # Otherwise we make every single context and invoke them in a # chain. In that case the return value to the result processor @@ -1273,6 +1702,7 @@ class MultiCommand(Command): contexts = [] while args: cmd_name, cmd, args = self.resolve_command(ctx, args) + assert cmd is not None sub_ctx = cmd.make_context( cmd_name, args, @@ -1289,7 +1719,9 @@ class MultiCommand(Command): rv.append(sub_ctx.command.invoke(sub_ctx)) return _process_result(rv) - def resolve_command(self, ctx, args): + def resolve_command( + self, ctx: Context, args: t.List[str] + ) -> t.Tuple[t.Optional[str], t.Optional[Command], t.List[str]]: cmd_name = make_str(args[0]) original_cmd_name = cmd_name @@ -1311,36 +1743,96 @@ class MultiCommand(Command): if cmd is None and not ctx.resilient_parsing: if split_opt(cmd_name)[0]: self.parse_args(ctx, ctx.args) - ctx.fail("No such command '{}'.".format(original_cmd_name)) + ctx.fail(_("No such command {name!r}.").format(name=original_cmd_name)) + return cmd_name if cmd else None, cmd, args[1:] - return cmd_name, cmd, args[1:] - - def get_command(self, ctx, cmd_name): + def get_command(self, ctx: Context, cmd_name: str) -> t.Optional[Command]: """Given a context and a command name, this returns a :class:`Command` object if it exists or returns `None`. """ - raise NotImplementedError() + raise NotImplementedError - def list_commands(self, ctx): + def list_commands(self, ctx: Context) -> t.List[str]: """Returns a list of subcommand names in the order they should appear. """ return [] + def shell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]: + """Return a list of completions for the incomplete value. Looks + at the names of options, subcommands, and chained + multi-commands. + + :param ctx: Invocation context for this command. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + results = [ + CompletionItem(name, help=command.get_short_help_str()) + for name, command in _complete_visible_commands(ctx, incomplete) + ] + results.extend(super().shell_complete(ctx, incomplete)) + return results + class Group(MultiCommand): - """A group allows a command to have subcommands attached. This is the - most common way to implement nesting in Click. + """A group allows a command to have subcommands attached. This is + the most common way to implement nesting in Click. - :param commands: a dictionary of commands. + :param name: The name of the group command. + :param commands: A dict mapping names to :class:`Command` objects. + Can also be a list of :class:`Command`, which will use + :attr:`Command.name` to create the dict. + :param attrs: Other command arguments described in + :class:`MultiCommand`, :class:`Command`, and + :class:`BaseCommand`. + + .. versionchanged:: 8.0 + The ``commands`` argument can be a list of command objects. """ - def __init__(self, name=None, commands=None, **attrs): - MultiCommand.__init__(self, name, **attrs) - #: the registered subcommands by their exported names. - self.commands = commands or {} + #: If set, this is used by the group's :meth:`command` decorator + #: as the default :class:`Command` class. This is useful to make all + #: subcommands use a custom command class. + #: + #: .. versionadded:: 8.0 + command_class: t.Optional[t.Type[Command]] = None - def add_command(self, cmd, name=None): + #: If set, this is used by the group's :meth:`group` decorator + #: as the default :class:`Group` class. This is useful to make all + #: subgroups use a custom group class. + #: + #: If set to the special value :class:`type` (literally + #: ``group_class = type``), this group's class will be used as the + #: default class. This makes a custom group class continue to make + #: custom groups. + #: + #: .. versionadded:: 8.0 + group_class: t.Optional[t.Union[t.Type["Group"], t.Type[type]]] = None + # Literal[type] isn't valid, so use Type[type] + + def __init__( + self, + name: t.Optional[str] = None, + commands: t.Optional[ + t.Union[t.MutableMapping[str, Command], t.Sequence[Command]] + ] = None, + **attrs: t.Any, + ) -> None: + super().__init__(name, **attrs) + + if commands is None: + commands = {} + elif isinstance(commands, abc.Sequence): + commands = {c.name: c for c in commands if c.name is not None} + + #: The registered subcommands by their exported names. + self.commands: t.MutableMapping[str, Command] = commands + + def add_command(self, cmd: Command, name: t.Optional[str] = None) -> None: """Registers another :class:`Command` with this group. If the name is not provided, the name of the command is used. """ @@ -1350,40 +1842,115 @@ class Group(MultiCommand): _check_multicommand(self, name, cmd, register=True) self.commands[name] = cmd - def command(self, *args, **kwargs): + @t.overload + def command(self, __func: t.Callable[..., t.Any]) -> Command: + ... + + @t.overload + def command( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Callable[[t.Callable[..., t.Any]], Command]: + ... + + def command( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Union[t.Callable[[t.Callable[..., t.Any]], Command], Command]: """A shortcut decorator for declaring and attaching a command to - the group. This takes the same arguments as :func:`command` but - immediately registers the created command with this instance by - calling into :meth:`add_command`. + the group. This takes the same arguments as :func:`command` and + immediately registers the created command with this group by + calling :meth:`add_command`. + + To customize the command class used, set the + :attr:`command_class` attribute. + + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. + + .. versionchanged:: 8.0 + Added the :attr:`command_class` attribute. """ from .decorators import command - def decorator(f): - cmd = command(*args, **kwargs)(f) + func: t.Optional[t.Callable[..., t.Any]] = None + + if args and callable(args[0]): + assert ( + len(args) == 1 and not kwargs + ), "Use 'command(**kwargs)(callable)' to provide arguments." + (func,) = args + args = () + + if self.command_class and kwargs.get("cls") is None: + kwargs["cls"] = self.command_class + + def decorator(f: t.Callable[..., t.Any]) -> Command: + cmd: Command = command(*args, **kwargs)(f) self.add_command(cmd) return cmd + if func is not None: + return decorator(func) + return decorator - def group(self, *args, **kwargs): + @t.overload + def group(self, __func: t.Callable[..., t.Any]) -> "Group": + ... + + @t.overload + def group( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Callable[[t.Callable[..., t.Any]], "Group"]: + ... + + def group( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Union[t.Callable[[t.Callable[..., t.Any]], "Group"], "Group"]: """A shortcut decorator for declaring and attaching a group to - the group. This takes the same arguments as :func:`group` but - immediately registers the created command with this instance by - calling into :meth:`add_command`. + the group. This takes the same arguments as :func:`group` and + immediately registers the created group with this group by + calling :meth:`add_command`. + + To customize the group class used, set the :attr:`group_class` + attribute. + + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. + + .. versionchanged:: 8.0 + Added the :attr:`group_class` attribute. """ from .decorators import group - def decorator(f): - cmd = group(*args, **kwargs)(f) + func: t.Optional[t.Callable[..., t.Any]] = None + + if args and callable(args[0]): + assert ( + len(args) == 1 and not kwargs + ), "Use 'group(**kwargs)(callable)' to provide arguments." + (func,) = args + args = () + + if self.group_class is not None and kwargs.get("cls") is None: + if self.group_class is type: + kwargs["cls"] = type(self) + else: + kwargs["cls"] = self.group_class + + def decorator(f: t.Callable[..., t.Any]) -> "Group": + cmd: Group = group(*args, **kwargs)(f) self.add_command(cmd) return cmd + if func is not None: + return decorator(func) + return decorator - def get_command(self, ctx, cmd_name): + def get_command(self, ctx: Context, cmd_name: str) -> t.Optional[Command]: return self.commands.get(cmd_name) - def list_commands(self, ctx): + def list_commands(self, ctx: Context) -> t.List[str]: return sorted(self.commands) @@ -1392,33 +1959,57 @@ class CommandCollection(MultiCommand): commands together into one. This is a straightforward implementation that accepts a list of different multi commands as sources and provides all the commands for each of them. + + See :class:`MultiCommand` and :class:`Command` for the description of + ``name`` and ``attrs``. """ - def __init__(self, name=None, sources=None, **attrs): - MultiCommand.__init__(self, name, **attrs) + def __init__( + self, + name: t.Optional[str] = None, + sources: t.Optional[t.List[MultiCommand]] = None, + **attrs: t.Any, + ) -> None: + super().__init__(name, **attrs) #: The list of registered multi commands. - self.sources = sources or [] + self.sources: t.List[MultiCommand] = sources or [] - def add_source(self, multi_cmd): + def add_source(self, multi_cmd: MultiCommand) -> None: """Adds a new multi command to the chain dispatcher.""" self.sources.append(multi_cmd) - def get_command(self, ctx, cmd_name): + def get_command(self, ctx: Context, cmd_name: str) -> t.Optional[Command]: for source in self.sources: rv = source.get_command(ctx, cmd_name) + if rv is not None: if self.chain: _check_multicommand(self, cmd_name, rv) + return rv - def list_commands(self, ctx): - rv = set() + return None + + def list_commands(self, ctx: Context) -> t.List[str]: + rv: t.Set[str] = set() + for source in self.sources: rv.update(source.list_commands(ctx)) + return sorted(rv) -class Parameter(object): +def _check_iter(value: t.Any) -> t.Iterator[t.Any]: + """Check if the value is iterable but not a string. Raises a type + error, or return an iterator over the value. + """ + if isinstance(value, str): + raise TypeError + + return iter(value) + + +class Parameter: r"""A parameter to a command comes in two versions: they are either :class:`Option`\s or :class:`Argument`\s. Other subclasses are currently not supported by design as some of the internals for parsing are @@ -1430,19 +2021,21 @@ class Parameter(object): argument. This is a list of flags or argument names. :param type: the type that should be used. Either a :class:`ParamType` - or a Python type. The later is converted into the former + or a Python type. The latter is converted into the former automatically if supported. :param required: controls if this is optional or not. :param default: the default value if omitted. This can also be a callable, in which case it's invoked when the default is needed without any arguments. - :param callback: a callback that should be executed after the parameter - was matched. This is called as ``fn(ctx, param, - value)`` and needs to return the value. + :param callback: A function to further process or validate the value + after type conversion. It is called as ``f(ctx, param, value)`` + and must return the value. It is called for all sources, + including prompts. :param nargs: the number of arguments to match. If not ``1`` the return value is a tuple instead of single value. The default for nargs is ``1`` (except if the type is a tuple, then it's - the arity of the tuple). + the arity of the tuple). If ``nargs=-1``, all remaining + parameters are collected. :param metavar: how the value is represented in the help page. :param expose_value: if this is `True` then the value is passed onwards to the command callback and stored on the context, @@ -1452,6 +2045,32 @@ class Parameter(object): order of processing. :param envvar: a string or list of strings that are environment variables that should be checked. + :param shell_complete: A function that returns custom shell + completions. Used instead of the param's type completion if + given. Takes ``ctx, param, incomplete`` and must return a list + of :class:`~click.shell_completion.CompletionItem` or a list of + strings. + + .. versionchanged:: 8.0 + ``process_value`` validates required parameters and bounded + ``nargs``, and invokes the parameter callback before returning + the value. This allows the callback to validate prompts. + ``full_process_value`` is removed. + + .. versionchanged:: 8.0 + ``autocompletion`` is renamed to ``shell_complete`` and has new + semantics described above. The old name is deprecated and will + be removed in 8.1, until then it will be wrapped to match the + new requirements. + + .. versionchanged:: 8.0 + For ``multiple=True, nargs>1``, the default must be a list of + tuples. + + .. versionchanged:: 8.0 + Setting a default is no longer required for ``nargs>1``, it will + default to ``None``. ``multiple=True`` or ``nargs=-1`` will + default to ``()``. .. versionchanged:: 7.1 Empty environment variables are ignored rather than taking the @@ -1463,27 +2082,36 @@ class Parameter(object): parameter. The old callback format will still work, but it will raise a warning to give you a chance to migrate the code easier. """ + param_type_name = "parameter" def __init__( self, - param_decls=None, - type=None, - required=False, - default=None, - callback=None, - nargs=None, - metavar=None, - expose_value=True, - is_eager=False, - envvar=None, - autocompletion=None, - ): + param_decls: t.Optional[t.Sequence[str]] = None, + type: t.Optional[t.Union[types.ParamType, t.Any]] = None, + required: bool = False, + default: t.Optional[t.Union[t.Any, t.Callable[[], t.Any]]] = None, + callback: t.Optional[t.Callable[[Context, "Parameter", t.Any], t.Any]] = None, + nargs: t.Optional[int] = None, + multiple: bool = False, + metavar: t.Optional[str] = None, + expose_value: bool = True, + is_eager: bool = False, + envvar: t.Optional[t.Union[str, t.Sequence[str]]] = None, + shell_complete: t.Optional[ + t.Callable[ + [Context, "Parameter", str], + t.Union[t.List["CompletionItem"], t.List[str]], + ] + ] = None, + ) -> None: + self.name: t.Optional[str] + self.opts: t.List[str] + self.secondary_opts: t.List[str] self.name, self.opts, self.secondary_opts = self._parse_decls( param_decls or (), expose_value ) - - self.type = convert_type(type, default) + self.type: types.ParamType = types.convert_type(type, default) # Default nargs to what the type tells us if we have that # information available. @@ -1496,158 +2124,326 @@ class Parameter(object): self.required = required self.callback = callback self.nargs = nargs - self.multiple = False + self.multiple = multiple self.expose_value = expose_value self.default = default self.is_eager = is_eager self.metavar = metavar self.envvar = envvar - self.autocompletion = autocompletion + self._custom_shell_complete = shell_complete - def __repr__(self): - return "<{} {}>".format(self.__class__.__name__, self.name) + if __debug__: + if self.type.is_composite and nargs != self.type.arity: + raise ValueError( + f"'nargs' must be {self.type.arity} (or None) for" + f" type {self.type!r}, but it was {nargs}." + ) + + # Skip no default or callable default. + check_default = default if not callable(default) else None + + if check_default is not None: + if multiple: + try: + # Only check the first value against nargs. + check_default = next(_check_iter(check_default), None) + except TypeError: + raise ValueError( + "'default' must be a list when 'multiple' is true." + ) from None + + # Can be None for multiple with empty default. + if nargs != 1 and check_default is not None: + try: + _check_iter(check_default) + except TypeError: + if multiple: + message = ( + "'default' must be a list of lists when 'multiple' is" + " true and 'nargs' != 1." + ) + else: + message = "'default' must be a list when 'nargs' != 1." + + raise ValueError(message) from None + + if nargs > 1 and len(check_default) != nargs: + subject = "item length" if multiple else "length" + raise ValueError( + f"'default' {subject} must match nargs={nargs}." + ) + + def to_info_dict(self) -> t.Dict[str, t.Any]: + """Gather information that could be useful for a tool generating + user-facing documentation. + + Use :meth:`click.Context.to_info_dict` to traverse the entire + CLI structure. + + .. versionadded:: 8.0 + """ + return { + "name": self.name, + "param_type_name": self.param_type_name, + "opts": self.opts, + "secondary_opts": self.secondary_opts, + "type": self.type.to_info_dict(), + "required": self.required, + "nargs": self.nargs, + "multiple": self.multiple, + "default": self.default, + "envvar": self.envvar, + } + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + def _parse_decls( + self, decls: t.Sequence[str], expose_value: bool + ) -> t.Tuple[t.Optional[str], t.List[str], t.List[str]]: + raise NotImplementedError() @property - def human_readable_name(self): + def human_readable_name(self) -> str: """Returns the human readable name of this parameter. This is the same as the name for options, but the metavar for arguments. """ - return self.name + return self.name # type: ignore - def make_metavar(self): + def make_metavar(self) -> str: if self.metavar is not None: return self.metavar + metavar = self.type.get_metavar(self) + if metavar is None: metavar = self.type.name.upper() + if self.nargs != 1: metavar += "..." + return metavar - def get_default(self, ctx): - """Given a context variable this calculates the default value.""" - # Otherwise go with the regular default. - if callable(self.default): - rv = self.default() - else: - rv = self.default - return self.type_cast_value(ctx, rv) + @t.overload + def get_default( + self, ctx: Context, call: "te.Literal[True]" = True + ) -> t.Optional[t.Any]: + ... - def add_to_parser(self, parser, ctx): - pass + @t.overload + def get_default( + self, ctx: Context, call: bool = ... + ) -> t.Optional[t.Union[t.Any, t.Callable[[], t.Any]]]: + ... + + def get_default( + self, ctx: Context, call: bool = True + ) -> t.Optional[t.Union[t.Any, t.Callable[[], t.Any]]]: + """Get the default for the parameter. Tries + :meth:`Context.lookup_default` first, then the local default. + + :param ctx: Current context. + :param call: If the default is a callable, call it. Disable to + return the callable instead. + + .. versionchanged:: 8.0.2 + Type casting is no longer performed when getting a default. + + .. versionchanged:: 8.0.1 + Type casting can fail in resilient parsing mode. Invalid + defaults will not prevent showing help text. + + .. versionchanged:: 8.0 + Looks at ``ctx.default_map`` first. + + .. versionchanged:: 8.0 + Added the ``call`` parameter. + """ + value = ctx.lookup_default(self.name, call=False) # type: ignore - def consume_value(self, ctx, opts): - value = opts.get(self.name) if value is None: - value = self.value_from_envvar(ctx) - if value is None: - value = ctx.lookup_default(self.name) + value = self.default + + if call and callable(value): + value = value() + return value - def type_cast_value(self, ctx, value): - """Given a value this runs it properly through the type system. - This automatically handles things like `nargs` and `multiple` as - well as composite types. + def add_to_parser(self, parser: OptionParser, ctx: Context) -> None: + raise NotImplementedError() + + def consume_value( + self, ctx: Context, opts: t.Mapping[str, t.Any] + ) -> t.Tuple[t.Any, ParameterSource]: + value = opts.get(self.name) # type: ignore + source = ParameterSource.COMMANDLINE + + if value is None: + value = self.value_from_envvar(ctx) + source = ParameterSource.ENVIRONMENT + + if value is None: + value = ctx.lookup_default(self.name) # type: ignore + source = ParameterSource.DEFAULT_MAP + + if value is None: + value = self.get_default(ctx) + source = ParameterSource.DEFAULT + + return value, source + + def type_cast_value(self, ctx: Context, value: t.Any) -> t.Any: + """Convert and validate a value against the option's + :attr:`type`, :attr:`multiple`, and :attr:`nargs`. """ - if self.type.is_composite: - if self.nargs <= 1: - raise TypeError( - "Attempted to invoke composite type but nargs has" - " been set to {}. This is not supported; nargs" - " needs to be set to a fixed value > 1.".format(self.nargs) - ) - if self.multiple: - return tuple(self.type(x or (), self, ctx) for x in value or ()) - return self.type(value or (), self, ctx) + if value is None: + return () if self.multiple or self.nargs == -1 else None - def _convert(value, level): - if level == 0: - return self.type(value, self, ctx) - return tuple(_convert(x, level - 1) for x in value or ()) + def check_iter(value: t.Any) -> t.Iterator[t.Any]: + try: + return _check_iter(value) + except TypeError: + # This should only happen when passing in args manually, + # the parser should construct an iterable when parsing + # the command line. + raise BadParameter( + _("Value must be an iterable."), ctx=ctx, param=self + ) from None - return _convert(value, (self.nargs != 1) + bool(self.multiple)) + if self.nargs == 1 or self.type.is_composite: - def process_value(self, ctx, value): - """Given a value and context this runs the logic to convert the - value as necessary. - """ - # If the value we were given is None we do nothing. This way - # code that calls this can easily figure out if something was - # not provided. Otherwise it would be converted into an empty - # tuple for multiple invocations which is inconvenient. - if value is not None: - return self.type_cast_value(ctx, value) + def convert(value: t.Any) -> t.Any: + return self.type(value, param=self, ctx=ctx) - def value_is_missing(self, value): + elif self.nargs == -1: + + def convert(value: t.Any) -> t.Any: # t.Tuple[t.Any, ...] + return tuple(self.type(x, self, ctx) for x in check_iter(value)) + + else: # nargs > 1 + + def convert(value: t.Any) -> t.Any: # t.Tuple[t.Any, ...] + value = tuple(check_iter(value)) + + if len(value) != self.nargs: + raise BadParameter( + ngettext( + "Takes {nargs} values but 1 was given.", + "Takes {nargs} values but {len} were given.", + len(value), + ).format(nargs=self.nargs, len=len(value)), + ctx=ctx, + param=self, + ) + + return tuple(self.type(x, self, ctx) for x in value) + + if self.multiple: + return tuple(convert(x) for x in check_iter(value)) + + return convert(value) + + def value_is_missing(self, value: t.Any) -> bool: if value is None: return True + if (self.nargs != 1 or self.multiple) and value == (): return True + return False - def full_process_value(self, ctx, value): - value = self.process_value(ctx, value) - - if value is None and not ctx.resilient_parsing: - value = self.get_default(ctx) + def process_value(self, ctx: Context, value: t.Any) -> t.Any: + value = self.type_cast_value(ctx, value) if self.required and self.value_is_missing(value): raise MissingParameter(ctx=ctx, param=self) + if self.callback is not None: + value = self.callback(ctx, self, value) + return value - def resolve_envvar_value(self, ctx): + def resolve_envvar_value(self, ctx: Context) -> t.Optional[str]: if self.envvar is None: - return - if isinstance(self.envvar, (tuple, list)): - for envvar in self.envvar: - rv = os.environ.get(envvar) - if rv is not None: - return rv - else: + return None + + if isinstance(self.envvar, str): rv = os.environ.get(self.envvar) - if rv != "": + if rv: return rv + else: + for envvar in self.envvar: + rv = os.environ.get(envvar) + + if rv: + return rv + + return None + + def value_from_envvar(self, ctx: Context) -> t.Optional[t.Any]: + rv: t.Optional[t.Any] = self.resolve_envvar_value(ctx) - def value_from_envvar(self, ctx): - rv = self.resolve_envvar_value(ctx) if rv is not None and self.nargs != 1: rv = self.type.split_envvar_value(rv) + return rv - def handle_parse_result(self, ctx, opts, args): + def handle_parse_result( + self, ctx: Context, opts: t.Mapping[str, t.Any], args: t.List[str] + ) -> t.Tuple[t.Any, t.List[str]]: with augment_usage_errors(ctx, param=self): - value = self.consume_value(ctx, opts) + value, source = self.consume_value(ctx, opts) + ctx.set_parameter_source(self.name, source) # type: ignore + try: - value = self.full_process_value(ctx, value) + value = self.process_value(ctx, value) except Exception: if not ctx.resilient_parsing: raise + value = None - if self.callback is not None: - try: - value = invoke_param_callback(self.callback, ctx, self, value) - except Exception: - if not ctx.resilient_parsing: - raise if self.expose_value: - ctx.params[self.name] = value + ctx.params[self.name] = value # type: ignore + return value, args - def get_help_record(self, ctx): + def get_help_record(self, ctx: Context) -> t.Optional[t.Tuple[str, str]]: pass - def get_usage_pieces(self, ctx): + def get_usage_pieces(self, ctx: Context) -> t.List[str]: return [] - def get_error_hint(self, ctx): + def get_error_hint(self, ctx: Context) -> str: """Get a stringified version of the param for use in error messages to indicate which param caused the error. """ hint_list = self.opts or [self.human_readable_name] - return " / ".join(repr(x) for x in hint_list) + return " / ".join(f"'{x}'" for x in hint_list) + + def shell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]: + """Return a list of completions for the incomplete value. If a + ``shell_complete`` function was given during init, it is used. + Otherwise, the :attr:`type` + :meth:`~click.types.ParamType.shell_complete` function is used. + + :param ctx: Invocation context for this command. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + if self._custom_shell_complete is not None: + results = self._custom_shell_complete(ctx, self, incomplete) + + if results and isinstance(results[0], str): + from click.shell_completion import CompletionItem + + results = [CompletionItem(c) for c in results] + + return t.cast(t.List["CompletionItem"], results) + + return self.type.shell_complete(ctx, self, incomplete) class Option(Parameter): @@ -1656,21 +2452,27 @@ class Option(Parameter): All other parameters are passed onwards to the parameter constructor. - :param show_default: controls if the default value should be shown on the - help page. Normally, defaults are not shown. If this - value is a string, it shows the string instead of the - value. This is particularly useful for dynamic options. - :param show_envvar: controls if an environment variable should be shown on - the help page. Normally, environment variables - are not shown. - :param prompt: if set to `True` or a non empty string then the user will be - prompted for input. If set to `True` the prompt will be the - option name capitalized. - :param confirmation_prompt: if set then the value will need to be confirmed - if it was prompted for. - :param hide_input: if this is `True` then the input on the prompt will be - hidden from the user. This is useful for password - input. + :param show_default: Show the default value for this option in its + help text. Values are not shown by default, unless + :attr:`Context.show_default` is ``True``. If this value is a + string, it shows that string in parentheses instead of the + actual value. This is particularly useful for dynamic options. + For single option boolean flags, the default remains hidden if + its value is ``False``. + :param show_envvar: Controls if an environment variable should be + shown on the help page. Normally, environment variables are not + shown. + :param prompt: If set to ``True`` or a non empty string then the + user will be prompted for input. If set to ``True`` the prompt + will be the option name capitalized. + :param confirmation_prompt: Prompt a second time to confirm the + value if it was prompted for. Can be set to a string instead of + ``True`` to customize the message. + :param prompt_required: If set to ``False``, the user will be + prompted for input only when the option was specified as a flag + without a value. + :param hide_input: If this is ``True`` then the input on the prompt + will be hidden from the user. This is useful for password input. :param is_flag: forces this option to act as a flag. The default is auto detection. :param flag_value: which value should be used for this flag if it's @@ -1687,106 +2489,168 @@ class Option(Parameter): context. :param help: the help string. :param hidden: hide this option from help outputs. + :param attrs: Other command arguments described in :class:`Parameter`. + + .. versionchanged:: 8.1.0 + Help text indentation is cleaned here instead of only in the + ``@option`` decorator. + + .. versionchanged:: 8.1.0 + The ``show_default`` parameter overrides + ``Context.show_default``. + + .. versionchanged:: 8.1.0 + The default of a single option boolean flag is not shown if the + default value is ``False``. + + .. versionchanged:: 8.0.1 + ``type`` is detected from ``flag_value`` if given. """ param_type_name = "option" def __init__( self, - param_decls=None, - show_default=False, - prompt=False, - confirmation_prompt=False, - hide_input=False, - is_flag=None, - flag_value=None, - multiple=False, - count=False, - allow_from_autoenv=True, - type=None, - help=None, - hidden=False, - show_choices=True, - show_envvar=False, - **attrs - ): - default_is_missing = attrs.get("default", _missing) is _missing - Parameter.__init__(self, param_decls, type=type, **attrs) + param_decls: t.Optional[t.Sequence[str]] = None, + show_default: t.Union[bool, str, None] = None, + prompt: t.Union[bool, str] = False, + confirmation_prompt: t.Union[bool, str] = False, + prompt_required: bool = True, + hide_input: bool = False, + is_flag: t.Optional[bool] = None, + flag_value: t.Optional[t.Any] = None, + multiple: bool = False, + count: bool = False, + allow_from_autoenv: bool = True, + type: t.Optional[t.Union[types.ParamType, t.Any]] = None, + help: t.Optional[str] = None, + hidden: bool = False, + show_choices: bool = True, + show_envvar: bool = False, + **attrs: t.Any, + ) -> None: + if help: + help = inspect.cleandoc(help) + + default_is_missing = "default" not in attrs + super().__init__(param_decls, type=type, multiple=multiple, **attrs) if prompt is True: - prompt_text = self.name.replace("_", " ").capitalize() + if self.name is None: + raise TypeError("'name' is required with 'prompt=True'.") + + prompt_text: t.Optional[str] = self.name.replace("_", " ").capitalize() elif prompt is False: prompt_text = None else: prompt_text = prompt + self.prompt = prompt_text self.confirmation_prompt = confirmation_prompt + self.prompt_required = prompt_required self.hide_input = hide_input self.hidden = hidden - # Flags + # If prompt is enabled but not required, then the option can be + # used as a flag to indicate using prompt or flag_value. + self._flag_needs_value = self.prompt is not None and not self.prompt_required + if is_flag is None: if flag_value is not None: + # Implicitly a flag because flag_value was set. is_flag = True + elif self._flag_needs_value: + # Not a flag, but when used as a flag it shows a prompt. + is_flag = False else: + # Implicitly a flag because flag options were given. is_flag = bool(self.secondary_opts) - if is_flag and default_is_missing: - self.default = False + elif is_flag is False and not self._flag_needs_value: + # Not a flag, and prompt is not enabled, can be used as a + # flag if flag_value is set. + self._flag_needs_value = flag_value is not None + + self.default: t.Union[t.Any, t.Callable[[], t.Any]] + + if is_flag and default_is_missing and not self.required: + if multiple: + self.default = () + else: + self.default = False + if flag_value is None: flag_value = not self.default - self.is_flag = is_flag - self.flag_value = flag_value - if self.is_flag and isinstance(self.flag_value, bool) and type in [None, bool]: - self.type = BOOL - self.is_bool_flag = True - else: - self.is_bool_flag = False + + self.type: types.ParamType + if is_flag and type is None: + # Re-guess the type from the flag value instead of the + # default. + self.type = types.convert_type(None, flag_value) + + self.is_flag: bool = is_flag + self.is_bool_flag: bool = is_flag and isinstance(self.type, types.BoolParamType) + self.flag_value: t.Any = flag_value # Counting self.count = count if count: if type is None: - self.type = IntRange(min=0) + self.type = types.IntRange(min=0) if default_is_missing: self.default = 0 - self.multiple = multiple self.allow_from_autoenv = allow_from_autoenv self.help = help self.show_default = show_default self.show_choices = show_choices self.show_envvar = show_envvar - # Sanity check for stuff we don't support if __debug__: - if self.nargs < 0: - raise TypeError("Options cannot have nargs < 0") + if self.nargs == -1: + raise TypeError("nargs=-1 is not supported for options.") + if self.prompt and self.is_flag and not self.is_bool_flag: - raise TypeError("Cannot prompt for flags that are not bools.") + raise TypeError("'prompt' is not valid for non-boolean flag.") + if not self.is_bool_flag and self.secondary_opts: - raise TypeError("Got secondary option for non boolean flag.") + raise TypeError("Secondary flag is not valid for non-boolean flag.") + if self.is_bool_flag and self.hide_input and self.prompt is not None: - raise TypeError("Hidden input does not work with boolean flag prompts.") + raise TypeError( + "'prompt' with 'hide_input' is not valid for boolean flag." + ) + if self.count: if self.multiple: - raise TypeError( - "Options cannot be multiple and count at the same time." - ) - elif self.is_flag: - raise TypeError( - "Options cannot be count and flags at the same time." - ) + raise TypeError("'count' is not valid with 'multiple'.") - def _parse_decls(self, decls, expose_value): + if self.is_flag: + raise TypeError("'count' is not valid with 'is_flag'.") + + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict.update( + help=self.help, + prompt=self.prompt, + is_flag=self.is_flag, + flag_value=self.flag_value, + count=self.count, + hidden=self.hidden, + ) + return info_dict + + def _parse_decls( + self, decls: t.Sequence[str], expose_value: bool + ) -> t.Tuple[t.Optional[str], t.List[str], t.List[str]]: opts = [] secondary_opts = [] name = None possible_names = [] for decl in decls: - if isidentifier(decl): + if decl.isidentifier(): if name is not None: - raise TypeError("Name defined twice") + raise TypeError(f"Name '{name}' defined twice") name = decl else: split_char = ";" if decl[:1] == "/" else "/" @@ -1799,6 +2663,11 @@ class Option(Parameter): second = second.lstrip() if second: secondary_opts.append(second.lstrip()) + if first == second: + raise ValueError( + f"Boolean option {decl!r} cannot use the" + " same flag for true/false." + ) else: possible_names.append(split_opt(decl)) opts.append(decl) @@ -1806,7 +2675,7 @@ class Option(Parameter): if name is None and possible_names: possible_names.sort(key=lambda x: -len(x[0])) # group long options first name = possible_names[0][1].replace("-", "_").lower() - if not isidentifier(name): + if not name.isidentifier(): name = None if name is None: @@ -1816,19 +2685,14 @@ class Option(Parameter): if not opts and not secondary_opts: raise TypeError( - "No options defined but a name was passed ({}). Did you" - " mean to declare an argument instead of an option?".format(name) + f"No options defined but a name was passed ({name})." + " Did you mean to declare an argument instead? Did" + f" you mean to pass '--{name}'?" ) return name, opts, secondary_opts - def add_to_parser(self, parser, ctx): - kwargs = { - "dest": self.name, - "nargs": self.nargs, - "obj": self, - } - + def add_to_parser(self, parser: OptionParser, ctx: Context) -> None: if self.multiple: action = "append" elif self.count: @@ -1837,91 +2701,180 @@ class Option(Parameter): action = "store" if self.is_flag: - kwargs.pop("nargs", None) - action_const = "{}_const".format(action) + action = f"{action}_const" + if self.is_bool_flag and self.secondary_opts: - parser.add_option(self.opts, action=action_const, const=True, **kwargs) parser.add_option( - self.secondary_opts, action=action_const, const=False, **kwargs + obj=self, opts=self.opts, dest=self.name, action=action, const=True + ) + parser.add_option( + obj=self, + opts=self.secondary_opts, + dest=self.name, + action=action, + const=False, ) else: parser.add_option( - self.opts, action=action_const, const=self.flag_value, **kwargs + obj=self, + opts=self.opts, + dest=self.name, + action=action, + const=self.flag_value, ) else: - kwargs["action"] = action - parser.add_option(self.opts, **kwargs) + parser.add_option( + obj=self, + opts=self.opts, + dest=self.name, + action=action, + nargs=self.nargs, + ) - def get_help_record(self, ctx): + def get_help_record(self, ctx: Context) -> t.Optional[t.Tuple[str, str]]: if self.hidden: - return - any_prefix_is_slash = [] + return None + + any_prefix_is_slash = False + + def _write_opts(opts: t.Sequence[str]) -> str: + nonlocal any_prefix_is_slash - def _write_opts(opts): rv, any_slashes = join_options(opts) + if any_slashes: - any_prefix_is_slash[:] = [True] + any_prefix_is_slash = True + if not self.is_flag and not self.count: - rv += " {}".format(self.make_metavar()) + rv += f" {self.make_metavar()}" + return rv rv = [_write_opts(self.opts)] + if self.secondary_opts: rv.append(_write_opts(self.secondary_opts)) help = self.help or "" extra = [] + if self.show_envvar: envvar = self.envvar + if envvar is None: - if self.allow_from_autoenv and ctx.auto_envvar_prefix is not None: - envvar = "{}_{}".format(ctx.auto_envvar_prefix, self.name.upper()) + if ( + self.allow_from_autoenv + and ctx.auto_envvar_prefix is not None + and self.name is not None + ): + envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" + if envvar is not None: - extra.append( - "env var: {}".format( - ", ".join(str(d) for d in envvar) - if isinstance(envvar, (list, tuple)) - else envvar - ) + var_str = ( + envvar + if isinstance(envvar, str) + else ", ".join(str(d) for d in envvar) ) - if self.default is not None and (self.show_default or ctx.show_default): - if isinstance(self.show_default, string_types): - default_string = "({})".format(self.show_default) - elif isinstance(self.default, (list, tuple)): - default_string = ", ".join(str(d) for d in self.default) - elif inspect.isfunction(self.default): - default_string = "(dynamic)" + extra.append(_("env var: {var}").format(var=var_str)) + + # Temporarily enable resilient parsing to avoid type casting + # failing for the default. Might be possible to extend this to + # help formatting in general. + resilient = ctx.resilient_parsing + ctx.resilient_parsing = True + + try: + default_value = self.get_default(ctx, call=False) + finally: + ctx.resilient_parsing = resilient + + show_default = False + show_default_is_str = False + + if self.show_default is not None: + if isinstance(self.show_default, str): + show_default_is_str = show_default = True else: - default_string = self.default - extra.append("default: {}".format(default_string)) + show_default = self.show_default + elif ctx.show_default is not None: + show_default = ctx.show_default + + if show_default_is_str or (show_default and (default_value is not None)): + if show_default_is_str: + default_string = f"({self.show_default})" + elif isinstance(default_value, (list, tuple)): + default_string = ", ".join(str(d) for d in default_value) + elif inspect.isfunction(default_value): + default_string = _("(dynamic)") + elif self.is_bool_flag and self.secondary_opts: + # For boolean flags that have distinct True/False opts, + # use the opt without prefix instead of the value. + default_string = split_opt( + (self.opts if self.default else self.secondary_opts)[0] + )[1] + elif self.is_bool_flag and not self.secondary_opts and not default_value: + default_string = "" + else: + default_string = str(default_value) + + if default_string: + extra.append(_("default: {default}").format(default=default_string)) + + if ( + isinstance(self.type, types._NumberRangeBase) + # skip count with default range type + and not (self.count and self.type.min == 0 and self.type.max is None) + ): + range_str = self.type._describe_range() + + if range_str: + extra.append(range_str) if self.required: - extra.append("required") + extra.append(_("required")) + if extra: - help = "{}[{}]".format( - "{} ".format(help) if help else "", "; ".join(extra) - ) + extra_str = "; ".join(extra) + help = f"{help} [{extra_str}]" if help else f"[{extra_str}]" return ("; " if any_prefix_is_slash else " / ").join(rv), help - def get_default(self, ctx): + @t.overload + def get_default( + self, ctx: Context, call: "te.Literal[True]" = True + ) -> t.Optional[t.Any]: + ... + + @t.overload + def get_default( + self, ctx: Context, call: bool = ... + ) -> t.Optional[t.Union[t.Any, t.Callable[[], t.Any]]]: + ... + + def get_default( + self, ctx: Context, call: bool = True + ) -> t.Optional[t.Union[t.Any, t.Callable[[], t.Any]]]: # If we're a non boolean flag our default is more complex because # we need to look at all flags in the same group to figure out - # if we're the the default one in which case we return the flag + # if we're the default one in which case we return the flag # value as default. if self.is_flag and not self.is_bool_flag: for param in ctx.command.params: if param.name == self.name and param.default: - return param.flag_value - return None - return Parameter.get_default(self, ctx) + return t.cast(Option, param).flag_value - def prompt_for_value(self, ctx): + return None + + return super().get_default(ctx, call=call) + + def prompt_for_value(self, ctx: Context) -> t.Any: """This is an alternative flow that can be activated in the full value processing if a value does not exist. It will prompt the user until a valid value exists and then returns the processed value as result. """ + assert self.prompt is not None + # Calculate the default before prompting anything to be stable. default = self.get_default(ctx) @@ -1940,29 +2893,77 @@ class Option(Parameter): value_proc=lambda x: self.process_value(ctx, x), ) - def resolve_envvar_value(self, ctx): - rv = Parameter.resolve_envvar_value(self, ctx) + def resolve_envvar_value(self, ctx: Context) -> t.Optional[str]: + rv = super().resolve_envvar_value(ctx) + if rv is not None: return rv - if self.allow_from_autoenv and ctx.auto_envvar_prefix is not None: - envvar = "{}_{}".format(ctx.auto_envvar_prefix, self.name.upper()) - return os.environ.get(envvar) - def value_from_envvar(self, ctx): - rv = self.resolve_envvar_value(ctx) + if ( + self.allow_from_autoenv + and ctx.auto_envvar_prefix is not None + and self.name is not None + ): + envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" + rv = os.environ.get(envvar) + + if rv: + return rv + + return None + + def value_from_envvar(self, ctx: Context) -> t.Optional[t.Any]: + rv: t.Optional[t.Any] = self.resolve_envvar_value(ctx) + if rv is None: return None + value_depth = (self.nargs != 1) + bool(self.multiple) - if value_depth > 0 and rv is not None: + + if value_depth > 0: rv = self.type.split_envvar_value(rv) + if self.multiple and self.nargs != 1: rv = batch(rv, self.nargs) + return rv - def full_process_value(self, ctx, value): - if value is None and self.prompt is not None and not ctx.resilient_parsing: - return self.prompt_for_value(ctx) - return Parameter.full_process_value(self, ctx, value) + def consume_value( + self, ctx: Context, opts: t.Mapping[str, "Parameter"] + ) -> t.Tuple[t.Any, ParameterSource]: + value, source = super().consume_value(ctx, opts) + + # The parser will emit a sentinel value if the option can be + # given as a flag without a value. This is different from None + # to distinguish from the flag not being given at all. + if value is _flag_needs_value: + if self.prompt is not None and not ctx.resilient_parsing: + value = self.prompt_for_value(ctx) + source = ParameterSource.PROMPT + else: + value = self.flag_value + source = ParameterSource.COMMANDLINE + + elif ( + self.multiple + and value is not None + and any(v is _flag_needs_value for v in value) + ): + value = [self.flag_value if v is _flag_needs_value else v for v in value] + source = ParameterSource.COMMANDLINE + + # The value wasn't set, or used the param's default, prompt if + # prompting is enabled. + elif ( + source in {None, ParameterSource.DEFAULT} + and self.prompt is not None + and (self.required or self.prompt_required) + and not ctx.resilient_parsing + ): + value = self.prompt_for_value(ctx) + source = ParameterSource.PROMPT + + return value, source class Argument(Parameter): @@ -1970,42 +2971,53 @@ class Argument(Parameter): provide fewer features than options but can have infinite ``nargs`` and are required by default. - All parameters are passed onwards to the parameter constructor. + All parameters are passed onwards to the constructor of :class:`Parameter`. """ param_type_name = "argument" - def __init__(self, param_decls, required=None, **attrs): + def __init__( + self, + param_decls: t.Sequence[str], + required: t.Optional[bool] = None, + **attrs: t.Any, + ) -> None: if required is None: if attrs.get("default") is not None: required = False else: required = attrs.get("nargs", 1) > 0 - Parameter.__init__(self, param_decls, required=required, **attrs) - if self.default is not None and self.nargs < 0: - raise TypeError( - "nargs=-1 in combination with a default value is not supported." - ) + + if "multiple" in attrs: + raise TypeError("__init__() got an unexpected keyword argument 'multiple'.") + + super().__init__(param_decls, required=required, **attrs) + + if __debug__: + if self.default is not None and self.nargs == -1: + raise TypeError("'default' is not supported for nargs=-1.") @property - def human_readable_name(self): + def human_readable_name(self) -> str: if self.metavar is not None: return self.metavar - return self.name.upper() + return self.name.upper() # type: ignore - def make_metavar(self): + def make_metavar(self) -> str: if self.metavar is not None: return self.metavar var = self.type.get_metavar(self) if not var: - var = self.name.upper() + var = self.name.upper() # type: ignore if not self.required: - var = "[{}]".format(var) + var = f"[{var}]" if self.nargs != 1: var += "..." return var - def _parse_decls(self, decls, expose_value): + def _parse_decls( + self, decls: t.Sequence[str], expose_value: bool + ) -> t.Tuple[t.Optional[str], t.List[str], t.List[str]]: if not decls: if not expose_value: return None, [], [] @@ -2016,15 +3028,15 @@ class Argument(Parameter): else: raise TypeError( "Arguments take exactly one parameter declaration, got" - " {}".format(len(decls)) + f" {len(decls)}." ) return name, [arg], [] - def get_usage_pieces(self, ctx): + def get_usage_pieces(self, ctx: Context) -> t.List[str]: return [self.make_metavar()] - def get_error_hint(self, ctx): - return repr(self.make_metavar()) + def get_error_hint(self, ctx: Context) -> str: + return f"'{self.make_metavar()}'" - def add_to_parser(self, parser, ctx): + def add_to_parser(self, parser: OptionParser, ctx: Context) -> None: parser.add_argument(dest=self.name, nargs=self.nargs, obj=self) diff --git a/third_party/python/click/click/decorators.py b/third_party/python/click/click/decorators.py index c7b5af6cc57f..d9bba9502ca3 100644 --- a/third_party/python/click/click/decorators.py +++ b/third_party/python/click/click/decorators.py @@ -1,41 +1,55 @@ import inspect -import sys +import types +import typing as t from functools import update_wrapper +from gettext import gettext as _ -from ._compat import iteritems -from ._unicodefun import _check_for_unicode_literals from .core import Argument from .core import Command +from .core import Context from .core import Group from .core import Option +from .core import Parameter from .globals import get_current_context from .utils import echo +if t.TYPE_CHECKING: + import typing_extensions as te -def pass_context(f): + P = te.ParamSpec("P") + +R = t.TypeVar("R") +T = t.TypeVar("T") +_AnyCallable = t.Callable[..., t.Any] +FC = t.TypeVar("FC", bound=t.Union[_AnyCallable, Command]) + + +def pass_context(f: "t.Callable[te.Concatenate[Context, P], R]") -> "t.Callable[P, R]": """Marks a callback as wanting to receive the current context object as first argument. """ - def new_func(*args, **kwargs): + def new_func(*args: "P.args", **kwargs: "P.kwargs") -> "R": return f(get_current_context(), *args, **kwargs) return update_wrapper(new_func, f) -def pass_obj(f): +def pass_obj(f: "t.Callable[te.Concatenate[t.Any, P], R]") -> "t.Callable[P, R]": """Similar to :func:`pass_context`, but only pass the object on the context onwards (:attr:`Context.obj`). This is useful if that object represents the state of a nested system. """ - def new_func(*args, **kwargs): + def new_func(*args: "P.args", **kwargs: "P.kwargs") -> "R": return f(get_current_context().obj, *args, **kwargs) return update_wrapper(new_func, f) -def make_pass_decorator(object_type, ensure=False): +def make_pass_decorator( + object_type: t.Type[T], ensure: bool = False +) -> t.Callable[["t.Callable[te.Concatenate[T, P], R]"], "t.Callable[P, R]"]: """Given an object type this creates a decorator that will work similar to :func:`pass_obj` but instead of passing the object of the current context, it will find the innermost context of type @@ -58,52 +72,107 @@ def make_pass_decorator(object_type, ensure=False): remembered on the context if it's not there yet. """ - def decorator(f): - def new_func(*args, **kwargs): + def decorator(f: "t.Callable[te.Concatenate[T, P], R]") -> "t.Callable[P, R]": + def new_func(*args: "P.args", **kwargs: "P.kwargs") -> "R": ctx = get_current_context() + + obj: t.Optional[T] if ensure: obj = ctx.ensure_object(object_type) else: obj = ctx.find_object(object_type) + if obj is None: raise RuntimeError( "Managed to invoke callback without a context" - " object of type '{}' existing".format(object_type.__name__) + f" object of type {object_type.__name__!r}" + " existing." ) + return ctx.invoke(f, obj, *args, **kwargs) return update_wrapper(new_func, f) - return decorator + return decorator # type: ignore[return-value] -def _make_command(f, name, attrs, cls): - if isinstance(f, Command): - raise TypeError("Attempted to convert a callback into a command twice.") - try: - params = f.__click_params__ - params.reverse() - del f.__click_params__ - except AttributeError: - params = [] - help = attrs.get("help") - if help is None: - help = inspect.getdoc(f) - if isinstance(help, bytes): - help = help.decode("utf-8") - else: - help = inspect.cleandoc(help) - attrs["help"] = help - _check_for_unicode_literals() - return cls( - name=name or f.__name__.lower().replace("_", "-"), - callback=f, - params=params, - **attrs +def pass_meta_key( + key: str, *, doc_description: t.Optional[str] = None +) -> "t.Callable[[t.Callable[te.Concatenate[t.Any, P], R]], t.Callable[P, R]]": + """Create a decorator that passes a key from + :attr:`click.Context.meta` as the first argument to the decorated + function. + + :param key: Key in ``Context.meta`` to pass. + :param doc_description: Description of the object being passed, + inserted into the decorator's docstring. Defaults to "the 'key' + key from Context.meta". + + .. versionadded:: 8.0 + """ + + def decorator(f: "t.Callable[te.Concatenate[t.Any, P], R]") -> "t.Callable[P, R]": + def new_func(*args: "P.args", **kwargs: "P.kwargs") -> R: + ctx = get_current_context() + obj = ctx.meta[key] + return ctx.invoke(f, obj, *args, **kwargs) + + return update_wrapper(new_func, f) + + if doc_description is None: + doc_description = f"the {key!r} key from :attr:`click.Context.meta`" + + decorator.__doc__ = ( + f"Decorator that passes {doc_description} as the first argument" + " to the decorated function." ) + return decorator # type: ignore[return-value] -def command(name=None, cls=None, **attrs): +CmdType = t.TypeVar("CmdType", bound=Command) + + +# variant: no call, directly as decorator for a function. +@t.overload +def command(name: _AnyCallable) -> Command: + ... + + +# variant: with positional name and with positional or keyword cls argument: +# @command(namearg, CommandCls, ...) or @command(namearg, cls=CommandCls, ...) +@t.overload +def command( + name: t.Optional[str], + cls: t.Type[CmdType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], CmdType]: + ... + + +# variant: name omitted, cls _must_ be a keyword argument, @command(cls=CommandCls, ...) +@t.overload +def command( + name: None = None, + *, + cls: t.Type[CmdType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], CmdType]: + ... + + +# variant: with optional string name, no cls argument provided. +@t.overload +def command( + name: t.Optional[str] = ..., cls: None = None, **attrs: t.Any +) -> t.Callable[[_AnyCallable], Command]: + ... + + +def command( + name: t.Union[t.Optional[str], _AnyCallable] = None, + cls: t.Optional[t.Type[CmdType]] = None, + **attrs: t.Any, +) -> t.Union[Command, t.Callable[[_AnyCallable], t.Union[Command, CmdType]]]: r"""Creates a new :class:`Command` and uses the decorated function as callback. This will also automatically attach all decorated :func:`option`\s and :func:`argument`\s as parameters to the command. @@ -113,6 +182,8 @@ def command(name=None, cls=None, **attrs): pass the intended name as the first argument. All keyword arguments are forwarded to the underlying command class. + For the ``params`` argument, any decorated params are appended to + the end of the list. Once decorated the function turns into a :class:`Command` instance that can be invoked as a command line utility or be attached to a @@ -122,212 +193,369 @@ def command(name=None, cls=None, **attrs): name with underscores replaced by dashes. :param cls: the command class to instantiate. This defaults to :class:`Command`. - """ - if cls is None: - cls = Command - def decorator(f): - cmd = _make_command(f, name, attrs, cls) + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. + + .. versionchanged:: 8.1 + The ``params`` argument can be used. Decorated params are + appended to the end of the list. + """ + + func: t.Optional[t.Callable[[_AnyCallable], t.Any]] = None + + if callable(name): + func = name + name = None + assert cls is None, "Use 'command(cls=cls)(callable)' to specify a class." + assert not attrs, "Use 'command(**kwargs)(callable)' to provide arguments." + + if cls is None: + cls = t.cast(t.Type[CmdType], Command) + + def decorator(f: _AnyCallable) -> CmdType: + if isinstance(f, Command): + raise TypeError("Attempted to convert a callback into a command twice.") + + attr_params = attrs.pop("params", None) + params = attr_params if attr_params is not None else [] + + try: + decorator_params = f.__click_params__ # type: ignore + except AttributeError: + pass + else: + del f.__click_params__ # type: ignore + params.extend(reversed(decorator_params)) + + if attrs.get("help") is None: + attrs["help"] = f.__doc__ + + if t.TYPE_CHECKING: + assert cls is not None + assert not callable(name) + + cmd = cls( + name=name or f.__name__.lower().replace("_", "-"), + callback=f, + params=params, + **attrs, + ) cmd.__doc__ = f.__doc__ return cmd + if func is not None: + return decorator(func) + return decorator -def group(name=None, **attrs): +GrpType = t.TypeVar("GrpType", bound=Group) + + +# variant: no call, directly as decorator for a function. +@t.overload +def group(name: _AnyCallable) -> Group: + ... + + +# variant: with positional name and with positional or keyword cls argument: +# @group(namearg, GroupCls, ...) or @group(namearg, cls=GroupCls, ...) +@t.overload +def group( + name: t.Optional[str], + cls: t.Type[GrpType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], GrpType]: + ... + + +# variant: name omitted, cls _must_ be a keyword argument, @group(cmd=GroupCls, ...) +@t.overload +def group( + name: None = None, + *, + cls: t.Type[GrpType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], GrpType]: + ... + + +# variant: with optional string name, no cls argument provided. +@t.overload +def group( + name: t.Optional[str] = ..., cls: None = None, **attrs: t.Any +) -> t.Callable[[_AnyCallable], Group]: + ... + + +def group( + name: t.Union[str, _AnyCallable, None] = None, + cls: t.Optional[t.Type[GrpType]] = None, + **attrs: t.Any, +) -> t.Union[Group, t.Callable[[_AnyCallable], t.Union[Group, GrpType]]]: """Creates a new :class:`Group` with a function as callback. This works otherwise the same as :func:`command` just that the `cls` parameter is set to :class:`Group`. + + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. """ - attrs.setdefault("cls", Group) - return command(name, **attrs) + if cls is None: + cls = t.cast(t.Type[GrpType], Group) + + if callable(name): + return command(cls=cls, **attrs)(name) + + return command(name, cls, **attrs) -def _param_memo(f, param): +def _param_memo(f: t.Callable[..., t.Any], param: Parameter) -> None: if isinstance(f, Command): f.params.append(param) else: if not hasattr(f, "__click_params__"): - f.__click_params__ = [] - f.__click_params__.append(param) + f.__click_params__ = [] # type: ignore + + f.__click_params__.append(param) # type: ignore -def argument(*param_decls, **attrs): +def argument( + *param_decls: str, cls: t.Optional[t.Type[Argument]] = None, **attrs: t.Any +) -> t.Callable[[FC], FC]: """Attaches an argument to the command. All positional arguments are passed as parameter declarations to :class:`Argument`; all keyword arguments are forwarded unchanged (except ``cls``). This is equivalent to creating an :class:`Argument` instance manually and attaching it to the :attr:`Command.params` list. + For the default argument class, refer to :class:`Argument` and + :class:`Parameter` for descriptions of parameters. + :param cls: the argument class to instantiate. This defaults to :class:`Argument`. + :param param_decls: Passed as positional arguments to the constructor of + ``cls``. + :param attrs: Passed as keyword arguments to the constructor of ``cls``. """ + if cls is None: + cls = Argument - def decorator(f): - ArgumentClass = attrs.pop("cls", Argument) - _param_memo(f, ArgumentClass(param_decls, **attrs)) + def decorator(f: FC) -> FC: + _param_memo(f, cls(param_decls, **attrs)) return f return decorator -def option(*param_decls, **attrs): +def option( + *param_decls: str, cls: t.Optional[t.Type[Option]] = None, **attrs: t.Any +) -> t.Callable[[FC], FC]: """Attaches an option to the command. All positional arguments are passed as parameter declarations to :class:`Option`; all keyword arguments are forwarded unchanged (except ``cls``). This is equivalent to creating an :class:`Option` instance manually and attaching it to the :attr:`Command.params` list. + For the default option class, refer to :class:`Option` and + :class:`Parameter` for descriptions of parameters. + :param cls: the option class to instantiate. This defaults to :class:`Option`. + :param param_decls: Passed as positional arguments to the constructor of + ``cls``. + :param attrs: Passed as keyword arguments to the constructor of ``cls``. """ + if cls is None: + cls = Option - def decorator(f): - # Issue 926, copy attrs, so pre-defined options can re-use the same cls= - option_attrs = attrs.copy() - - if "help" in option_attrs: - option_attrs["help"] = inspect.cleandoc(option_attrs["help"]) - OptionClass = option_attrs.pop("cls", Option) - _param_memo(f, OptionClass(param_decls, **option_attrs)) + def decorator(f: FC) -> FC: + _param_memo(f, cls(param_decls, **attrs)) return f return decorator -def confirmation_option(*param_decls, **attrs): - """Shortcut for confirmation prompts that can be ignored by passing - ``--yes`` as parameter. +def confirmation_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: + """Add a ``--yes`` option which shows a prompt before continuing if + not passed. If the prompt is declined, the program will exit. - This is equivalent to decorating a function with :func:`option` with - the following parameters:: - - def callback(ctx, param, value): - if not value: - ctx.abort() - - @click.command() - @click.option('--yes', is_flag=True, callback=callback, - expose_value=False, prompt='Do you want to continue?') - def dropdb(): - pass + :param param_decls: One or more option names. Defaults to the single + value ``"--yes"``. + :param kwargs: Extra arguments are passed to :func:`option`. """ - def decorator(f): - def callback(ctx, param, value): - if not value: - ctx.abort() + def callback(ctx: Context, param: Parameter, value: bool) -> None: + if not value: + ctx.abort() - attrs.setdefault("is_flag", True) - attrs.setdefault("callback", callback) - attrs.setdefault("expose_value", False) - attrs.setdefault("prompt", "Do you want to continue?") - attrs.setdefault("help", "Confirm the action without prompting.") - return option(*(param_decls or ("--yes",)), **attrs)(f) + if not param_decls: + param_decls = ("--yes",) - return decorator + kwargs.setdefault("is_flag", True) + kwargs.setdefault("callback", callback) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("prompt", "Do you want to continue?") + kwargs.setdefault("help", "Confirm the action without prompting.") + return option(*param_decls, **kwargs) -def password_option(*param_decls, **attrs): - """Shortcut for password prompts. +def password_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: + """Add a ``--password`` option which prompts for a password, hiding + input and asking to enter the value again for confirmation. - This is equivalent to decorating a function with :func:`option` with - the following parameters:: + :param param_decls: One or more option names. Defaults to the single + value ``"--password"``. + :param kwargs: Extra arguments are passed to :func:`option`. + """ + if not param_decls: + param_decls = ("--password",) - @click.command() - @click.option('--password', prompt=True, confirmation_prompt=True, - hide_input=True) - def changeadmin(password): - pass + kwargs.setdefault("prompt", True) + kwargs.setdefault("confirmation_prompt", True) + kwargs.setdefault("hide_input", True) + return option(*param_decls, **kwargs) + + +def version_option( + version: t.Optional[str] = None, + *param_decls: str, + package_name: t.Optional[str] = None, + prog_name: t.Optional[str] = None, + message: t.Optional[str] = None, + **kwargs: t.Any, +) -> t.Callable[[FC], FC]: + """Add a ``--version`` option which immediately prints the version + number and exits the program. + + If ``version`` is not provided, Click will try to detect it using + :func:`importlib.metadata.version` to get the version for the + ``package_name``. On Python < 3.8, the ``importlib_metadata`` + backport must be installed. + + If ``package_name`` is not provided, Click will try to detect it by + inspecting the stack frames. This will be used to detect the + version, so it must match the name of the installed package. + + :param version: The version number to show. If not provided, Click + will try to detect it. + :param param_decls: One or more option names. Defaults to the single + value ``"--version"``. + :param package_name: The package name to detect the version from. If + not provided, Click will try to detect it. + :param prog_name: The name of the CLI to show in the message. If not + provided, it will be detected from the command. + :param message: The message to show. The values ``%(prog)s``, + ``%(package)s``, and ``%(version)s`` are available. Defaults to + ``"%(prog)s, version %(version)s"``. + :param kwargs: Extra arguments are passed to :func:`option`. + :raise RuntimeError: ``version`` could not be detected. + + .. versionchanged:: 8.0 + Add the ``package_name`` parameter, and the ``%(package)s`` + value for messages. + + .. versionchanged:: 8.0 + Use :mod:`importlib.metadata` instead of ``pkg_resources``. The + version is detected based on the package name, not the entry + point name. The Python package name must match the installed + package name, or be passed with ``package_name=``. + """ + if message is None: + message = _("%(prog)s, version %(version)s") + + if version is None and package_name is None: + frame = inspect.currentframe() + f_back = frame.f_back if frame is not None else None + f_globals = f_back.f_globals if f_back is not None else None + # break reference cycle + # https://docs.python.org/3/library/inspect.html#the-interpreter-stack + del frame + + if f_globals is not None: + package_name = f_globals.get("__name__") + + if package_name == "__main__": + package_name = f_globals.get("__package__") + + if package_name: + package_name = package_name.partition(".")[0] + + def callback(ctx: Context, param: Parameter, value: bool) -> None: + if not value or ctx.resilient_parsing: + return + + nonlocal prog_name + nonlocal version + + if prog_name is None: + prog_name = ctx.find_root().info_name + + if version is None and package_name is not None: + metadata: t.Optional[types.ModuleType] + + try: + from importlib import metadata # type: ignore + except ImportError: + # Python < 3.8 + import importlib_metadata as metadata # type: ignore + + try: + version = metadata.version(package_name) # type: ignore + except metadata.PackageNotFoundError: # type: ignore + raise RuntimeError( + f"{package_name!r} is not installed. Try passing" + " 'package_name' instead." + ) from None + + if version is None: + raise RuntimeError( + f"Could not determine the version for {package_name!r} automatically." + ) + + echo( + message % {"prog": prog_name, "package": package_name, "version": version}, + color=ctx.color, + ) + ctx.exit() + + if not param_decls: + param_decls = ("--version",) + + kwargs.setdefault("is_flag", True) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("is_eager", True) + kwargs.setdefault("help", _("Show the version and exit.")) + kwargs["callback"] = callback + return option(*param_decls, **kwargs) + + +def help_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: + """Add a ``--help`` option which immediately prints the help page + and exits the program. + + This is usually unnecessary, as the ``--help`` option is added to + each command automatically unless ``add_help_option=False`` is + passed. + + :param param_decls: One or more option names. Defaults to the single + value ``"--help"``. + :param kwargs: Extra arguments are passed to :func:`option`. """ - def decorator(f): - attrs.setdefault("prompt", True) - attrs.setdefault("confirmation_prompt", True) - attrs.setdefault("hide_input", True) - return option(*(param_decls or ("--password",)), **attrs)(f) + def callback(ctx: Context, param: Parameter, value: bool) -> None: + if not value or ctx.resilient_parsing: + return - return decorator + echo(ctx.get_help(), color=ctx.color) + ctx.exit() + if not param_decls: + param_decls = ("--help",) -def version_option(version=None, *param_decls, **attrs): - """Adds a ``--version`` option which immediately ends the program - printing out the version number. This is implemented as an eager - option that prints the version and exits the program in the callback. - - :param version: the version number to show. If not provided Click - attempts an auto discovery via setuptools. - :param prog_name: the name of the program (defaults to autodetection) - :param message: custom message to show instead of the default - (``'%(prog)s, version %(version)s'``) - :param others: everything else is forwarded to :func:`option`. - """ - if version is None: - if hasattr(sys, "_getframe"): - module = sys._getframe(1).f_globals.get("__name__") - else: - module = "" - - def decorator(f): - prog_name = attrs.pop("prog_name", None) - message = attrs.pop("message", "%(prog)s, version %(version)s") - - def callback(ctx, param, value): - if not value or ctx.resilient_parsing: - return - prog = prog_name - if prog is None: - prog = ctx.find_root().info_name - ver = version - if ver is None: - try: - import pkg_resources - except ImportError: - pass - else: - for dist in pkg_resources.working_set: - scripts = dist.get_entry_map().get("console_scripts") or {} - for _, entry_point in iteritems(scripts): - if entry_point.module_name == module: - ver = dist.version - break - if ver is None: - raise RuntimeError("Could not determine version") - echo(message % {"prog": prog, "version": ver}, color=ctx.color) - ctx.exit() - - attrs.setdefault("is_flag", True) - attrs.setdefault("expose_value", False) - attrs.setdefault("is_eager", True) - attrs.setdefault("help", "Show the version and exit.") - attrs["callback"] = callback - return option(*(param_decls or ("--version",)), **attrs)(f) - - return decorator - - -def help_option(*param_decls, **attrs): - """Adds a ``--help`` option which immediately ends the program - printing out the help page. This is usually unnecessary to add as - this is added by default to all commands unless suppressed. - - Like :func:`version_option`, this is implemented as eager option that - prints in the callback and exits. - - All arguments are forwarded to :func:`option`. - """ - - def decorator(f): - def callback(ctx, param, value): - if value and not ctx.resilient_parsing: - echo(ctx.get_help(), color=ctx.color) - ctx.exit() - - attrs.setdefault("is_flag", True) - attrs.setdefault("expose_value", False) - attrs.setdefault("help", "Show this message and exit.") - attrs.setdefault("is_eager", True) - attrs["callback"] = callback - return option(*(param_decls or ("--help",)), **attrs)(f) - - return decorator + kwargs.setdefault("is_flag", True) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("is_eager", True) + kwargs.setdefault("help", _("Show this message and exit.")) + kwargs["callback"] = callback + return option(*param_decls, **kwargs) diff --git a/third_party/python/click/click/exceptions.py b/third_party/python/click/click/exceptions.py index 592ee38f0dec..fe68a3613f74 100644 --- a/third_party/python/click/click/exceptions.py +++ b/third_party/python/click/click/exceptions.py @@ -1,45 +1,47 @@ -from ._compat import filename_to_ui +import typing as t +from gettext import gettext as _ +from gettext import ngettext + from ._compat import get_text_stderr -from ._compat import PY2 from .utils import echo +from .utils import format_filename + +if t.TYPE_CHECKING: + from .core import Command + from .core import Context + from .core import Parameter -def _join_param_hints(param_hint): - if isinstance(param_hint, (tuple, list)): +def _join_param_hints( + param_hint: t.Optional[t.Union[t.Sequence[str], str]] +) -> t.Optional[str]: + if param_hint is not None and not isinstance(param_hint, str): return " / ".join(repr(x) for x in param_hint) + return param_hint class ClickException(Exception): """An exception that Click can handle and show to the user.""" - #: The exit code for this exception + #: The exit code for this exception. exit_code = 1 - def __init__(self, message): - ctor_msg = message - if PY2: - if ctor_msg is not None: - ctor_msg = ctor_msg.encode("utf-8") - Exception.__init__(self, ctor_msg) + def __init__(self, message: str) -> None: + super().__init__(message) self.message = message - def format_message(self): + def format_message(self) -> str: return self.message - def __str__(self): + def __str__(self) -> str: return self.message - if PY2: - __unicode__ = __str__ - - def __str__(self): - return self.message.encode("utf-8") - - def show(self, file=None): + def show(self, file: t.Optional[t.IO[t.Any]] = None) -> None: if file is None: file = get_text_stderr() - echo("Error: {}".format(self.format_message()), file=file) + + echo(_("Error: {message}").format(message=self.format_message()), file=file) class UsageError(ClickException): @@ -53,24 +55,32 @@ class UsageError(ClickException): exit_code = 2 - def __init__(self, message, ctx=None): - ClickException.__init__(self, message) + def __init__(self, message: str, ctx: t.Optional["Context"] = None) -> None: + super().__init__(message) self.ctx = ctx - self.cmd = self.ctx.command if self.ctx else None + self.cmd: t.Optional["Command"] = self.ctx.command if self.ctx else None - def show(self, file=None): + def show(self, file: t.Optional[t.IO[t.Any]] = None) -> None: if file is None: file = get_text_stderr() color = None hint = "" - if self.cmd is not None and self.cmd.get_help_option(self.ctx) is not None: - hint = "Try '{} {}' for help.\n".format( - self.ctx.command_path, self.ctx.help_option_names[0] + if ( + self.ctx is not None + and self.ctx.command.get_help_option(self.ctx) is not None + ): + hint = _("Try '{command} {option}' for help.").format( + command=self.ctx.command_path, option=self.ctx.help_option_names[0] ) + hint = f"{hint}\n" if self.ctx is not None: color = self.ctx.color - echo("{}\n{}".format(self.ctx.get_usage(), hint), file=file, color=color) - echo("Error: {}".format(self.format_message()), file=file, color=color) + echo(f"{self.ctx.get_usage()}\n{hint}", file=file, color=color) + echo( + _("Error: {message}").format(message=self.format_message()), + file=file, + color=color, + ) class BadParameter(UsageError): @@ -91,21 +101,28 @@ class BadParameter(UsageError): each item is quoted and separated. """ - def __init__(self, message, ctx=None, param=None, param_hint=None): - UsageError.__init__(self, message, ctx) + def __init__( + self, + message: str, + ctx: t.Optional["Context"] = None, + param: t.Optional["Parameter"] = None, + param_hint: t.Optional[str] = None, + ) -> None: + super().__init__(message, ctx) self.param = param self.param_hint = param_hint - def format_message(self): + def format_message(self) -> str: if self.param_hint is not None: param_hint = self.param_hint elif self.param is not None: - param_hint = self.param.get_error_hint(self.ctx) + param_hint = self.param.get_error_hint(self.ctx) # type: ignore else: - return "Invalid value: {}".format(self.message) - param_hint = _join_param_hints(param_hint) + return _("Invalid value: {message}").format(message=self.message) - return "Invalid value for {}: {}".format(param_hint, self.message) + return _("Invalid value for {param_hint}: {message}").format( + param_hint=_join_param_hints(param_hint), message=self.message + ) class MissingParameter(BadParameter): @@ -121,19 +138,26 @@ class MissingParameter(BadParameter): """ def __init__( - self, message=None, ctx=None, param=None, param_hint=None, param_type=None - ): - BadParameter.__init__(self, message, ctx, param, param_hint) + self, + message: t.Optional[str] = None, + ctx: t.Optional["Context"] = None, + param: t.Optional["Parameter"] = None, + param_hint: t.Optional[str] = None, + param_type: t.Optional[str] = None, + ) -> None: + super().__init__(message or "", ctx, param, param_hint) self.param_type = param_type - def format_message(self): + def format_message(self) -> str: if self.param_hint is not None: - param_hint = self.param_hint + param_hint: t.Optional[str] = self.param_hint elif self.param is not None: - param_hint = self.param.get_error_hint(self.ctx) + param_hint = self.param.get_error_hint(self.ctx) # type: ignore else: param_hint = None + param_hint = _join_param_hints(param_hint) + param_hint = f" {param_hint}" if param_hint else "" param_type = self.param_type if param_type is None and self.param is not None: @@ -144,30 +168,31 @@ class MissingParameter(BadParameter): msg_extra = self.param.type.get_missing_message(self.param) if msg_extra: if msg: - msg += ". {}".format(msg_extra) + msg += f". {msg_extra}" else: msg = msg_extra - return "Missing {}{}{}{}".format( - param_type, - " {}".format(param_hint) if param_hint else "", - ". " if msg else ".", - msg or "", - ) + msg = f" {msg}" if msg else "" - def __str__(self): - if self.message is None: + # Translate param_type for known types. + if param_type == "argument": + missing = _("Missing argument") + elif param_type == "option": + missing = _("Missing option") + elif param_type == "parameter": + missing = _("Missing parameter") + else: + missing = _("Missing {param_type}").format(param_type=param_type) + + return f"{missing}{param_hint}.{msg}" + + def __str__(self) -> str: + if not self.message: param_name = self.param.name if self.param else None - return "missing parameter: {}".format(param_name) + return _("Missing parameter: {param_name}").format(param_name=param_name) else: return self.message - if PY2: - __unicode__ = __str__ - - def __str__(self): - return self.__unicode__().encode("utf-8") - class NoSuchOption(UsageError): """Raised if click attempted to handle an option that does not @@ -176,22 +201,31 @@ class NoSuchOption(UsageError): .. versionadded:: 4.0 """ - def __init__(self, option_name, message=None, possibilities=None, ctx=None): + def __init__( + self, + option_name: str, + message: t.Optional[str] = None, + possibilities: t.Optional[t.Sequence[str]] = None, + ctx: t.Optional["Context"] = None, + ) -> None: if message is None: - message = "no such option: {}".format(option_name) - UsageError.__init__(self, message, ctx) + message = _("No such option: {name}").format(name=option_name) + + super().__init__(message, ctx) self.option_name = option_name self.possibilities = possibilities - def format_message(self): - bits = [self.message] - if self.possibilities: - if len(self.possibilities) == 1: - bits.append("Did you mean {}?".format(self.possibilities[0])) - else: - possibilities = sorted(self.possibilities) - bits.append("(Possible options: {})".format(", ".join(possibilities))) - return " ".join(bits) + def format_message(self) -> str: + if not self.possibilities: + return self.message + + possibility_str = ", ".join(sorted(self.possibilities)) + suggest = ngettext( + "Did you mean {possibility}?", + "(Possible options: {possibilities})", + len(self.possibilities), + ).format(possibility=possibility_str, possibilities=possibility_str) + return f"{self.message} {suggest}" class BadOptionUsage(UsageError): @@ -204,8 +238,10 @@ class BadOptionUsage(UsageError): :param option_name: the name of the option being used incorrectly. """ - def __init__(self, option_name, message, ctx=None): - UsageError.__init__(self, message, ctx) + def __init__( + self, option_name: str, message: str, ctx: t.Optional["Context"] = None + ) -> None: + super().__init__(message, ctx) self.option_name = option_name @@ -217,23 +253,22 @@ class BadArgumentUsage(UsageError): .. versionadded:: 6.0 """ - def __init__(self, message, ctx=None): - UsageError.__init__(self, message, ctx) - class FileError(ClickException): """Raised if a file cannot be opened.""" - def __init__(self, filename, hint=None): - ui_filename = filename_to_ui(filename) + def __init__(self, filename: str, hint: t.Optional[str] = None) -> None: if hint is None: - hint = "unknown error" - ClickException.__init__(self, hint) - self.ui_filename = ui_filename + hint = _("unknown error") + + super().__init__(hint) + self.ui_filename: str = format_filename(filename) self.filename = filename - def format_message(self): - return "Could not open file {}: {}".format(self.ui_filename, self.message) + def format_message(self) -> str: + return _("Could not open file {filename!r}: {message}").format( + filename=self.ui_filename, message=self.message + ) class Abort(RuntimeError): @@ -249,5 +284,5 @@ class Exit(RuntimeError): __slots__ = ("exit_code",) - def __init__(self, code=0): - self.exit_code = code + def __init__(self, code: int = 0) -> None: + self.exit_code: int = code diff --git a/third_party/python/click/click/formatting.py b/third_party/python/click/click/formatting.py index 319c7f6163e2..ddd2a2f825f2 100644 --- a/third_party/python/click/click/formatting.py +++ b/third_party/python/click/click/formatting.py @@ -1,30 +1,38 @@ +import typing as t from contextlib import contextmanager +from gettext import gettext as _ from ._compat import term_len from .parser import split_opt -from .termui import get_terminal_size # Can force a width. This is used by the test system -FORCED_WIDTH = None +FORCED_WIDTH: t.Optional[int] = None -def measure_table(rows): - widths = {} +def measure_table(rows: t.Iterable[t.Tuple[str, str]]) -> t.Tuple[int, ...]: + widths: t.Dict[int, int] = {} + for row in rows: for idx, col in enumerate(row): widths[idx] = max(widths.get(idx, 0), term_len(col)) + return tuple(y for x, y in sorted(widths.items())) -def iter_rows(rows, col_count): +def iter_rows( + rows: t.Iterable[t.Tuple[str, str]], col_count: int +) -> t.Iterator[t.Tuple[str, ...]]: for row in rows: - row = tuple(row) yield row + ("",) * (col_count - len(row)) def wrap_text( - text, width=78, initial_indent="", subsequent_indent="", preserve_paragraphs=False -): + text: str, + width: int = 78, + initial_indent: str = "", + subsequent_indent: str = "", + preserve_paragraphs: bool = False, +) -> str: """A helper function that intelligently wraps text. By default, it assumes that it operates on a single paragraph of text but if the `preserve_paragraphs` parameter is provided it will intelligently @@ -55,11 +63,11 @@ def wrap_text( if not preserve_paragraphs: return wrapper.fill(text) - p = [] - buf = [] + p: t.List[t.Tuple[int, bool, str]] = [] + buf: t.List[str] = [] indent = None - def _flush_par(): + def _flush_par() -> None: if not buf: return if buf[0].strip() == "\b": @@ -91,7 +99,7 @@ def wrap_text( return "\n\n".join(rv) -class HelpFormatter(object): +class HelpFormatter: """This class helps with formatting text-based help pages. It's usually just needed for very special internal cases, but it's also exposed so that developers can write their own fancy outputs. @@ -103,38 +111,51 @@ class HelpFormatter(object): width clamped to a maximum of 78. """ - def __init__(self, indent_increment=2, width=None, max_width=None): + def __init__( + self, + indent_increment: int = 2, + width: t.Optional[int] = None, + max_width: t.Optional[int] = None, + ) -> None: + import shutil + self.indent_increment = indent_increment if max_width is None: max_width = 80 if width is None: width = FORCED_WIDTH if width is None: - width = max(min(get_terminal_size()[0], max_width) - 2, 50) + width = max(min(shutil.get_terminal_size().columns, max_width) - 2, 50) self.width = width self.current_indent = 0 - self.buffer = [] + self.buffer: t.List[str] = [] - def write(self, string): + def write(self, string: str) -> None: """Writes a unicode string into the internal buffer.""" self.buffer.append(string) - def indent(self): + def indent(self) -> None: """Increases the indentation.""" self.current_indent += self.indent_increment - def dedent(self): + def dedent(self) -> None: """Decreases the indentation.""" self.current_indent -= self.indent_increment - def write_usage(self, prog, args="", prefix="Usage: "): + def write_usage( + self, prog: str, args: str = "", prefix: t.Optional[str] = None + ) -> None: """Writes a usage line into the buffer. :param prog: the program name. :param args: whitespace separated list of arguments. - :param prefix: the prefix for the first line. + :param prefix: The prefix for the first line. Defaults to + ``"Usage: "``. """ - usage_prefix = "{:>{w}}{} ".format(prefix, prog, w=self.current_indent) + if prefix is None: + prefix = f"{_('Usage:')} " + + usage_prefix = f"{prefix:>{self.current_indent}}{prog} " text_width = self.width - self.current_indent if text_width >= (term_len(usage_prefix) + 20): @@ -161,25 +182,24 @@ class HelpFormatter(object): self.write("\n") - def write_heading(self, heading): + def write_heading(self, heading: str) -> None: """Writes a heading into the buffer.""" - self.write("{:>{w}}{}:\n".format("", heading, w=self.current_indent)) + self.write(f"{'':>{self.current_indent}}{heading}:\n") - def write_paragraph(self): + def write_paragraph(self) -> None: """Writes a paragraph into the buffer.""" if self.buffer: self.write("\n") - def write_text(self, text): + def write_text(self, text: str) -> None: """Writes re-indented text into the buffer. This rewraps and preserves paragraphs. """ - text_width = max(self.width - self.current_indent, 11) indent = " " * self.current_indent self.write( wrap_text( text, - text_width, + self.width, initial_indent=indent, subsequent_indent=indent, preserve_paragraphs=True, @@ -187,7 +207,12 @@ class HelpFormatter(object): ) self.write("\n") - def write_dl(self, rows, col_max=30, col_spacing=2): + def write_dl( + self, + rows: t.Sequence[t.Tuple[str, str]], + col_max: int = 30, + col_spacing: int = 2, + ) -> None: """Writes a definition list into the buffer. This is how options and commands are usually formatted. @@ -204,7 +229,7 @@ class HelpFormatter(object): first_col = min(widths[0], col_max) + col_spacing for first, second in iter_rows(rows, len(widths)): - self.write("{:>{w}}{}".format("", first, w=self.current_indent)) + self.write(f"{'':>{self.current_indent}}{first}") if not second: self.write("\n") continue @@ -219,23 +244,15 @@ class HelpFormatter(object): lines = wrapped_text.splitlines() if lines: - self.write("{}\n".format(lines[0])) + self.write(f"{lines[0]}\n") for line in lines[1:]: - self.write( - "{:>{w}}{}\n".format( - "", line, w=first_col + self.current_indent - ) - ) - - if len(lines) > 1: - # separate long help from next option - self.write("\n") + self.write(f"{'':>{first_col + self.current_indent}}{line}\n") else: self.write("\n") @contextmanager - def section(self, name): + def section(self, name: str) -> t.Iterator[None]: """Helpful context manager that writes a paragraph, a heading, and the indents. @@ -250,7 +267,7 @@ class HelpFormatter(object): self.dedent() @contextmanager - def indentation(self): + def indentation(self) -> t.Iterator[None]: """A context manager that increases the indentation.""" self.indent() try: @@ -258,12 +275,12 @@ class HelpFormatter(object): finally: self.dedent() - def getvalue(self): + def getvalue(self) -> str: """Returns the buffer contents.""" return "".join(self.buffer) -def join_options(options): +def join_options(options: t.Sequence[str]) -> t.Tuple[str, bool]: """Given a list of option strings this joins them in the most appropriate way and returns them in the form ``(formatted_string, any_prefix_is_slash)`` where the second item in the tuple is a flag that @@ -271,13 +288,14 @@ def join_options(options): """ rv = [] any_prefix_is_slash = False + for opt in options: prefix = split_opt(opt)[0] + if prefix == "/": any_prefix_is_slash = True + rv.append((len(prefix), opt)) rv.sort(key=lambda x: x[0]) - - rv = ", ".join(x[1] for x in rv) - return rv, any_prefix_is_slash + return ", ".join(x[1] for x in rv), any_prefix_is_slash diff --git a/third_party/python/click/click/globals.py b/third_party/python/click/click/globals.py index 1649f9a0bfbe..480058f10dd6 100644 --- a/third_party/python/click/click/globals.py +++ b/third_party/python/click/click/globals.py @@ -1,9 +1,24 @@ +import typing as t from threading import local +if t.TYPE_CHECKING: + import typing_extensions as te + from .core import Context + _local = local() -def get_current_context(silent=False): +@t.overload +def get_current_context(silent: "te.Literal[False]" = False) -> "Context": + ... + + +@t.overload +def get_current_context(silent: bool = ...) -> t.Optional["Context"]: + ... + + +def get_current_context(silent: bool = False) -> t.Optional["Context"]: """Returns the current click context. This can be used as a way to access the current context object from anywhere. This is a more implicit alternative to the :func:`pass_context` decorator. This function is @@ -19,29 +34,35 @@ def get_current_context(silent=False): :exc:`RuntimeError`. """ try: - return _local.stack[-1] - except (AttributeError, IndexError): + return t.cast("Context", _local.stack[-1]) + except (AttributeError, IndexError) as e: if not silent: - raise RuntimeError("There is no active click context.") + raise RuntimeError("There is no active click context.") from e + + return None -def push_context(ctx): +def push_context(ctx: "Context") -> None: """Pushes a new context to the current stack.""" _local.__dict__.setdefault("stack", []).append(ctx) -def pop_context(): +def pop_context() -> None: """Removes the top level from the stack.""" _local.stack.pop() -def resolve_color_default(color=None): - """"Internal helper to get the default value of the color flag. If a +def resolve_color_default(color: t.Optional[bool] = None) -> t.Optional[bool]: + """Internal helper to get the default value of the color flag. If a value is passed it's returned unchanged, otherwise it's looked up from the current context. """ if color is not None: return color + ctx = get_current_context(silent=True) + if ctx is not None: return ctx.color + + return None diff --git a/third_party/python/click/click/parser.py b/third_party/python/click/click/parser.py index f43ebfe9fc09..5fa7adfac842 100644 --- a/third_party/python/click/click/parser.py +++ b/third_party/python/click/click/parser.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This module started out as largely a copy paste from the stdlib's optparse module with the features removed that we do not need from @@ -18,16 +17,38 @@ by the Python Software Foundation. This is limited to code in parser.py. Copyright 2001-2006 Gregory P. Ward. All rights reserved. Copyright 2002-2006 Python Software Foundation. All rights reserved. """ -import re +# This code uses parts of optparse written by Gregory P. Ward and +# maintained by the Python Software Foundation. +# Copyright 2001-2006 Gregory P. Ward +# Copyright 2002-2006 Python Software Foundation +import typing as t from collections import deque +from gettext import gettext as _ +from gettext import ngettext from .exceptions import BadArgumentUsage from .exceptions import BadOptionUsage from .exceptions import NoSuchOption from .exceptions import UsageError +if t.TYPE_CHECKING: + import typing_extensions as te + from .core import Argument as CoreArgument + from .core import Context + from .core import Option as CoreOption + from .core import Parameter as CoreParameter -def _unpack_args(args, nargs_spec): +V = t.TypeVar("V") + +# Sentinel value that indicates an option was passed as a flag without a +# value but is not a flag option. Option.consume_value uses this to +# prompt or use the flag_value. +_flag_needs_value = object() + + +def _unpack_args( + args: t.Sequence[str], nargs_spec: t.Sequence[int] +) -> t.Tuple[t.Sequence[t.Union[str, t.Sequence[t.Optional[str]], None]], t.List[str]]: """Given an iterable of arguments and an iterable of nargs specifications, it returns a tuple with all the unpacked arguments at the first index and all remaining arguments as the second. @@ -39,10 +60,10 @@ def _unpack_args(args, nargs_spec): """ args = deque(args) nargs_spec = deque(nargs_spec) - rv = [] - spos = None + rv: t.List[t.Union[str, t.Tuple[t.Optional[str], ...], None]] = [] + spos: t.Optional[int] = None - def _fetch(c): + def _fetch(c: "te.Deque[V]") -> t.Optional[V]: try: if spos is None: return c.popleft() @@ -53,18 +74,25 @@ def _unpack_args(args, nargs_spec): while nargs_spec: nargs = _fetch(nargs_spec) + + if nargs is None: + continue + if nargs == 1: rv.append(_fetch(args)) elif nargs > 1: x = [_fetch(args) for _ in range(nargs)] + # If we're reversed, we're pulling in the arguments in reverse, # so we need to turn them around. if spos is not None: x.reverse() + rv.append(tuple(x)) elif nargs < 0: if spos is not None: raise TypeError("Cannot have two nargs < 0") + spos = len(rv) rv.append(None) @@ -78,13 +106,7 @@ def _unpack_args(args, nargs_spec): return tuple(rv), list(args) -def _error_opt_args(nargs, opt): - if nargs == 1: - raise BadOptionUsage(opt, "{} option requires an argument".format(opt)) - raise BadOptionUsage(opt, "{} option requires {} arguments".format(opt, nargs)) - - -def split_opt(opt): +def split_opt(opt: str) -> t.Tuple[str, str]: first = opt[:1] if first.isalnum(): return "", opt @@ -93,42 +115,65 @@ def split_opt(opt): return first, opt[1:] -def normalize_opt(opt, ctx): +def normalize_opt(opt: str, ctx: t.Optional["Context"]) -> str: if ctx is None or ctx.token_normalize_func is None: return opt prefix, opt = split_opt(opt) - return prefix + ctx.token_normalize_func(opt) + return f"{prefix}{ctx.token_normalize_func(opt)}" -def split_arg_string(string): - """Given an argument string this attempts to split it into small parts.""" - rv = [] - for match in re.finditer( - r"('([^'\\]*(?:\\.[^'\\]*)*)'|\"([^\"\\]*(?:\\.[^\"\\]*)*)\"|\S+)\s*", - string, - re.S, +def split_arg_string(string: str) -> t.List[str]: + """Split an argument string as with :func:`shlex.split`, but don't + fail if the string is incomplete. Ignores a missing closing quote or + incomplete escape sequence and uses the partial token as-is. + + .. code-block:: python + + split_arg_string("example 'my file") + ["example", "my file"] + + split_arg_string("example my\\") + ["example", "my"] + + :param string: String to split. + """ + import shlex + + lex = shlex.shlex(string, posix=True) + lex.whitespace_split = True + lex.commenters = "" + out = [] + + try: + for token in lex: + out.append(token) + except ValueError: + # Raised when end-of-string is reached in an invalid state. Use + # the partial token as-is. The quote or escape character is in + # lex.state, not lex.token. + out.append(lex.token) + + return out + + +class Option: + def __init__( + self, + obj: "CoreOption", + opts: t.Sequence[str], + dest: t.Optional[str], + action: t.Optional[str] = None, + nargs: int = 1, + const: t.Optional[t.Any] = None, ): - arg = match.group().strip() - if arg[:1] == arg[-1:] and arg[:1] in "\"'": - arg = arg[1:-1].encode("ascii", "backslashreplace").decode("unicode-escape") - try: - arg = type(string)(arg) - except UnicodeError: - pass - rv.append(arg) - return rv - - -class Option(object): - def __init__(self, opts, dest, action=None, nargs=1, const=None, obj=None): self._short_opts = [] self._long_opts = [] - self.prefixes = set() + self.prefixes: t.Set[str] = set() for opt in opts: prefix, value = split_opt(opt) if not prefix: - raise ValueError("Invalid start character for option ({})".format(opt)) + raise ValueError(f"Invalid start character for option ({opt})") self.prefixes.add(prefix[0]) if len(prefix) == 1 and len(value) == 1: self._short_opts.append(opt) @@ -146,53 +191,66 @@ class Option(object): self.obj = obj @property - def takes_value(self): + def takes_value(self) -> bool: return self.action in ("store", "append") - def process(self, value, state): + def process(self, value: t.Any, state: "ParsingState") -> None: if self.action == "store": - state.opts[self.dest] = value + state.opts[self.dest] = value # type: ignore elif self.action == "store_const": - state.opts[self.dest] = self.const + state.opts[self.dest] = self.const # type: ignore elif self.action == "append": - state.opts.setdefault(self.dest, []).append(value) + state.opts.setdefault(self.dest, []).append(value) # type: ignore elif self.action == "append_const": - state.opts.setdefault(self.dest, []).append(self.const) + state.opts.setdefault(self.dest, []).append(self.const) # type: ignore elif self.action == "count": - state.opts[self.dest] = state.opts.get(self.dest, 0) + 1 + state.opts[self.dest] = state.opts.get(self.dest, 0) + 1 # type: ignore else: - raise ValueError("unknown action '{}'".format(self.action)) + raise ValueError(f"unknown action '{self.action}'") state.order.append(self.obj) -class Argument(object): - def __init__(self, dest, nargs=1, obj=None): +class Argument: + def __init__(self, obj: "CoreArgument", dest: t.Optional[str], nargs: int = 1): self.dest = dest self.nargs = nargs self.obj = obj - def process(self, value, state): + def process( + self, + value: t.Union[t.Optional[str], t.Sequence[t.Optional[str]]], + state: "ParsingState", + ) -> None: if self.nargs > 1: + assert value is not None holes = sum(1 for x in value if x is None) if holes == len(value): value = None elif holes != 0: raise BadArgumentUsage( - "argument {} takes {} values".format(self.dest, self.nargs) + _("Argument {name!r} takes {nargs} values.").format( + name=self.dest, nargs=self.nargs + ) ) - state.opts[self.dest] = value + + if self.nargs == -1 and self.obj.envvar is not None and value == (): + # Replace empty tuple with None so that a value from the + # environment may be tried. + value = None + + state.opts[self.dest] = value # type: ignore state.order.append(self.obj) -class ParsingState(object): - def __init__(self, rargs): - self.opts = {} - self.largs = [] +class ParsingState: + def __init__(self, rargs: t.List[str]) -> None: + self.opts: t.Dict[str, t.Any] = {} + self.largs: t.List[str] = [] self.rargs = rargs - self.order = [] + self.order: t.List["CoreParameter"] = [] -class OptionParser(object): +class OptionParser: """The option parser is an internal class that is ultimately used to parse options and arguments. It's modelled after optparse and brings a similar but vastly simplified API. It should generally not be used @@ -206,7 +264,7 @@ class OptionParser(object): should go with. """ - def __init__(self, ctx=None): + def __init__(self, ctx: t.Optional["Context"] = None) -> None: #: The :class:`~click.Context` for this parser. This might be #: `None` for some advanced use cases. self.ctx = ctx @@ -214,50 +272,60 @@ class OptionParser(object): #: If this is set to `False`, the parser will stop on the first #: non-option. Click uses this to implement nested subcommands #: safely. - self.allow_interspersed_args = True + self.allow_interspersed_args: bool = True #: This tells the parser how to deal with unknown options. By #: default it will error out (which is sensible), but there is a #: second mode where it will ignore it and continue processing #: after shifting all the unknown options into the resulting args. - self.ignore_unknown_options = False + self.ignore_unknown_options: bool = False + if ctx is not None: self.allow_interspersed_args = ctx.allow_interspersed_args self.ignore_unknown_options = ctx.ignore_unknown_options - self._short_opt = {} - self._long_opt = {} - self._opt_prefixes = {"-", "--"} - self._args = [] - def add_option(self, opts, dest, action=None, nargs=1, const=None, obj=None): + self._short_opt: t.Dict[str, Option] = {} + self._long_opt: t.Dict[str, Option] = {} + self._opt_prefixes = {"-", "--"} + self._args: t.List[Argument] = [] + + def add_option( + self, + obj: "CoreOption", + opts: t.Sequence[str], + dest: t.Optional[str], + action: t.Optional[str] = None, + nargs: int = 1, + const: t.Optional[t.Any] = None, + ) -> None: """Adds a new option named `dest` to the parser. The destination is not inferred (unlike with optparse) and needs to be explicitly provided. Action can be any of ``store``, ``store_const``, - ``append``, ``appnd_const`` or ``count``. + ``append``, ``append_const`` or ``count``. The `obj` can be used to identify the option in the order list that is returned from the parser. """ - if obj is None: - obj = dest opts = [normalize_opt(opt, self.ctx) for opt in opts] - option = Option(opts, dest, action=action, nargs=nargs, const=const, obj=obj) + option = Option(obj, opts, dest, action=action, nargs=nargs, const=const) self._opt_prefixes.update(option.prefixes) for opt in option._short_opts: self._short_opt[opt] = option for opt in option._long_opts: self._long_opt[opt] = option - def add_argument(self, dest, nargs=1, obj=None): + def add_argument( + self, obj: "CoreArgument", dest: t.Optional[str], nargs: int = 1 + ) -> None: """Adds a positional argument named `dest` to the parser. The `obj` can be used to identify the option in the order list that is returned from the parser. """ - if obj is None: - obj = dest - self._args.append(Argument(dest=dest, nargs=nargs, obj=obj)) + self._args.append(Argument(obj, dest=dest, nargs=nargs)) - def parse_args(self, args): + def parse_args( + self, args: t.List[str] + ) -> t.Tuple[t.Dict[str, t.Any], t.List[str], t.List["CoreParameter"]]: """Parses positional arguments and returns ``(values, args, order)`` for the parsed options and arguments as well as the leftover arguments if there are any. The order is a list of objects as they @@ -273,7 +341,7 @@ class OptionParser(object): raise return state.opts, state.largs, state.order - def _process_args_for_args(self, state): + def _process_args_for_args(self, state: ParsingState) -> None: pargs, args = _unpack_args( state.largs + state.rargs, [x.nargs for x in self._args] ) @@ -284,7 +352,7 @@ class OptionParser(object): state.largs = args state.rargs = [] - def _process_args_for_options(self, state): + def _process_args_for_options(self, state: ParsingState) -> None: while state.rargs: arg = state.rargs.pop(0) arglen = len(arg) @@ -320,9 +388,13 @@ class OptionParser(object): # *empty* -- still a subset of [arg0, ..., arg(i-1)], but # not a very interesting subset! - def _match_long_opt(self, opt, explicit_value, state): + def _match_long_opt( + self, opt: str, explicit_value: t.Optional[str], state: ParsingState + ) -> None: if opt not in self._long_opt: - possibilities = [word for word in self._long_opt if word.startswith(opt)] + from difflib import get_close_matches + + possibilities = get_close_matches(opt, self._long_opt) raise NoSuchOption(opt, possibilities=possibilities, ctx=self.ctx) option = self._long_opt[opt] @@ -334,31 +406,26 @@ class OptionParser(object): if explicit_value is not None: state.rargs.insert(0, explicit_value) - nargs = option.nargs - if len(state.rargs) < nargs: - _error_opt_args(nargs, opt) - elif nargs == 1: - value = state.rargs.pop(0) - else: - value = tuple(state.rargs[:nargs]) - del state.rargs[:nargs] + value = self._get_value_from_state(opt, option, state) elif explicit_value is not None: - raise BadOptionUsage(opt, "{} option does not take a value".format(opt)) + raise BadOptionUsage( + opt, _("Option {name!r} does not take a value.").format(name=opt) + ) else: value = None option.process(value, state) - def _match_short_opt(self, arg, state): + def _match_short_opt(self, arg: str, state: ParsingState) -> None: stop = False i = 1 prefix = arg[0] unknown_options = [] for ch in arg[1:]: - opt = normalize_opt(prefix + ch, self.ctx) + opt = normalize_opt(f"{prefix}{ch}", self.ctx) option = self._short_opt.get(opt) i += 1 @@ -374,14 +441,7 @@ class OptionParser(object): state.rargs.insert(0, arg[i:]) stop = True - nargs = option.nargs - if len(state.rargs) < nargs: - _error_opt_args(nargs, opt) - elif nargs == 1: - value = state.rargs.pop(0) - else: - value = tuple(state.rargs[:nargs]) - del state.rargs[:nargs] + value = self._get_value_from_state(opt, option, state) else: value = None @@ -391,14 +451,52 @@ class OptionParser(object): if stop: break - # If we got any unknown options we re-combinate the string of the + # If we got any unknown options we recombine the string of the # remaining options and re-attach the prefix, then report that # to the state as new larg. This way there is basic combinatorics # that can be achieved while still ignoring unknown arguments. if self.ignore_unknown_options and unknown_options: - state.largs.append("{}{}".format(prefix, "".join(unknown_options))) + state.largs.append(f"{prefix}{''.join(unknown_options)}") - def _process_opts(self, arg, state): + def _get_value_from_state( + self, option_name: str, option: Option, state: ParsingState + ) -> t.Any: + nargs = option.nargs + + if len(state.rargs) < nargs: + if option.obj._flag_needs_value: + # Option allows omitting the value. + value = _flag_needs_value + else: + raise BadOptionUsage( + option_name, + ngettext( + "Option {name!r} requires an argument.", + "Option {name!r} requires {nargs} arguments.", + nargs, + ).format(name=option_name, nargs=nargs), + ) + elif nargs == 1: + next_rarg = state.rargs[0] + + if ( + option.obj._flag_needs_value + and isinstance(next_rarg, str) + and next_rarg[:1] in self._opt_prefixes + and len(next_rarg) > 1 + ): + # The next arg looks like the start of an option, don't + # use it as the value if omitting the value is allowed. + value = _flag_needs_value + else: + value = state.rargs.pop(0) + else: + value = tuple(state.rargs[:nargs]) + del state.rargs[:nargs] + + return value + + def _process_opts(self, arg: str, state: ParsingState) -> None: explicit_value = None # Long option handling happens in two parts. The first part is # supporting explicitly attached values. In any case, we will try @@ -422,7 +520,10 @@ class OptionParser(object): # short option code and will instead raise the no option # error. if arg[:2] not in self._opt_prefixes: - return self._match_short_opt(arg, state) + self._match_short_opt(arg, state) + return + if not self.ignore_unknown_options: raise + state.largs.append(arg) diff --git a/third_party/python/click/click/py.typed b/third_party/python/click/click/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/third_party/python/click/click/shell_completion.py b/third_party/python/click/click/shell_completion.py new file mode 100644 index 000000000000..5de124702ec7 --- /dev/null +++ b/third_party/python/click/click/shell_completion.py @@ -0,0 +1,593 @@ +import os +import re +import typing as t +from gettext import gettext as _ + +from .core import Argument +from .core import BaseCommand +from .core import Context +from .core import MultiCommand +from .core import Option +from .core import Parameter +from .core import ParameterSource +from .parser import split_arg_string +from .utils import echo + + +def shell_complete( + cli: BaseCommand, + ctx_args: t.MutableMapping[str, t.Any], + prog_name: str, + complete_var: str, + instruction: str, +) -> int: + """Perform shell completion for the given CLI program. + + :param cli: Command being called. + :param ctx_args: Extra arguments to pass to + ``cli.make_context``. + :param prog_name: Name of the executable in the shell. + :param complete_var: Name of the environment variable that holds + the completion instruction. + :param instruction: Value of ``complete_var`` with the completion + instruction and shell, in the form ``instruction_shell``. + :return: Status code to exit with. + """ + shell, _, instruction = instruction.partition("_") + comp_cls = get_completion_class(shell) + + if comp_cls is None: + return 1 + + comp = comp_cls(cli, ctx_args, prog_name, complete_var) + + if instruction == "source": + echo(comp.source()) + return 0 + + if instruction == "complete": + echo(comp.complete()) + return 0 + + return 1 + + +class CompletionItem: + """Represents a completion value and metadata about the value. The + default metadata is ``type`` to indicate special shell handling, + and ``help`` if a shell supports showing a help string next to the + value. + + Arbitrary parameters can be passed when creating the object, and + accessed using ``item.attr``. If an attribute wasn't passed, + accessing it returns ``None``. + + :param value: The completion suggestion. + :param type: Tells the shell script to provide special completion + support for the type. Click uses ``"dir"`` and ``"file"``. + :param help: String shown next to the value if supported. + :param kwargs: Arbitrary metadata. The built-in implementations + don't use this, but custom type completions paired with custom + shell support could use it. + """ + + __slots__ = ("value", "type", "help", "_info") + + def __init__( + self, + value: t.Any, + type: str = "plain", + help: t.Optional[str] = None, + **kwargs: t.Any, + ) -> None: + self.value: t.Any = value + self.type: str = type + self.help: t.Optional[str] = help + self._info = kwargs + + def __getattr__(self, name: str) -> t.Any: + return self._info.get(name) + + +# Only Bash >= 4.4 has the nosort option. +_SOURCE_BASH = """\ +%(complete_func)s() { + local IFS=$'\\n' + local response + + response=$(env COMP_WORDS="${COMP_WORDS[*]}" COMP_CWORD=$COMP_CWORD \ +%(complete_var)s=bash_complete $1) + + for completion in $response; do + IFS=',' read type value <<< "$completion" + + if [[ $type == 'dir' ]]; then + COMPREPLY=() + compopt -o dirnames + elif [[ $type == 'file' ]]; then + COMPREPLY=() + compopt -o default + elif [[ $type == 'plain' ]]; then + COMPREPLY+=($value) + fi + done + + return 0 +} + +%(complete_func)s_setup() { + complete -o nosort -F %(complete_func)s %(prog_name)s +} + +%(complete_func)s_setup; +""" + +_SOURCE_ZSH = """\ +#compdef %(prog_name)s + +%(complete_func)s() { + local -a completions + local -a completions_with_descriptions + local -a response + (( ! $+commands[%(prog_name)s] )) && return 1 + + response=("${(@f)$(env COMP_WORDS="${words[*]}" COMP_CWORD=$((CURRENT-1)) \ +%(complete_var)s=zsh_complete %(prog_name)s)}") + + for type key descr in ${response}; do + if [[ "$type" == "plain" ]]; then + if [[ "$descr" == "_" ]]; then + completions+=("$key") + else + completions_with_descriptions+=("$key":"$descr") + fi + elif [[ "$type" == "dir" ]]; then + _path_files -/ + elif [[ "$type" == "file" ]]; then + _path_files -f + fi + done + + if [ -n "$completions_with_descriptions" ]; then + _describe -V unsorted completions_with_descriptions -U + fi + + if [ -n "$completions" ]; then + compadd -U -V unsorted -a completions + fi +} + +if [[ $zsh_eval_context[-1] == loadautofunc ]]; then + # autoload from fpath, call function directly + %(complete_func)s "$@" +else + # eval/source/. command, register function for later + compdef %(complete_func)s %(prog_name)s +fi +""" + +_SOURCE_FISH = """\ +function %(complete_func)s + set -l response (env %(complete_var)s=fish_complete COMP_WORDS=(commandline -cp) \ +COMP_CWORD=(commandline -t) %(prog_name)s) + + for completion in $response + set -l metadata (string split "," $completion) + + if test $metadata[1] = "dir" + __fish_complete_directories $metadata[2] + else if test $metadata[1] = "file" + __fish_complete_path $metadata[2] + else if test $metadata[1] = "plain" + echo $metadata[2] + end + end +end + +complete --no-files --command %(prog_name)s --arguments \ +"(%(complete_func)s)" +""" + + +class ShellComplete: + """Base class for providing shell completion support. A subclass for + a given shell will override attributes and methods to implement the + completion instructions (``source`` and ``complete``). + + :param cli: Command being called. + :param prog_name: Name of the executable in the shell. + :param complete_var: Name of the environment variable that holds + the completion instruction. + + .. versionadded:: 8.0 + """ + + name: t.ClassVar[str] + """Name to register the shell as with :func:`add_completion_class`. + This is used in completion instructions (``{name}_source`` and + ``{name}_complete``). + """ + + source_template: t.ClassVar[str] + """Completion script template formatted by :meth:`source`. This must + be provided by subclasses. + """ + + def __init__( + self, + cli: BaseCommand, + ctx_args: t.MutableMapping[str, t.Any], + prog_name: str, + complete_var: str, + ) -> None: + self.cli = cli + self.ctx_args = ctx_args + self.prog_name = prog_name + self.complete_var = complete_var + + @property + def func_name(self) -> str: + """The name of the shell function defined by the completion + script. + """ + safe_name = re.sub(r"\W*", "", self.prog_name.replace("-", "_"), re.ASCII) + return f"_{safe_name}_completion" + + def source_vars(self) -> t.Dict[str, t.Any]: + """Vars for formatting :attr:`source_template`. + + By default this provides ``complete_func``, ``complete_var``, + and ``prog_name``. + """ + return { + "complete_func": self.func_name, + "complete_var": self.complete_var, + "prog_name": self.prog_name, + } + + def source(self) -> str: + """Produce the shell script that defines the completion + function. By default this ``%``-style formats + :attr:`source_template` with the dict returned by + :meth:`source_vars`. + """ + return self.source_template % self.source_vars() + + def get_completion_args(self) -> t.Tuple[t.List[str], str]: + """Use the env vars defined by the shell script to return a + tuple of ``args, incomplete``. This must be implemented by + subclasses. + """ + raise NotImplementedError + + def get_completions( + self, args: t.List[str], incomplete: str + ) -> t.List[CompletionItem]: + """Determine the context and last complete command or parameter + from the complete args. Call that object's ``shell_complete`` + method to get the completions for the incomplete value. + + :param args: List of complete args before the incomplete value. + :param incomplete: Value being completed. May be empty. + """ + ctx = _resolve_context(self.cli, self.ctx_args, self.prog_name, args) + obj, incomplete = _resolve_incomplete(ctx, args, incomplete) + return obj.shell_complete(ctx, incomplete) + + def format_completion(self, item: CompletionItem) -> str: + """Format a completion item into the form recognized by the + shell script. This must be implemented by subclasses. + + :param item: Completion item to format. + """ + raise NotImplementedError + + def complete(self) -> str: + """Produce the completion data to send back to the shell. + + By default this calls :meth:`get_completion_args`, gets the + completions, then calls :meth:`format_completion` for each + completion. + """ + args, incomplete = self.get_completion_args() + completions = self.get_completions(args, incomplete) + out = [self.format_completion(item) for item in completions] + return "\n".join(out) + + +class BashComplete(ShellComplete): + """Shell completion for Bash.""" + + name = "bash" + source_template = _SOURCE_BASH + + def _check_version(self) -> None: + import subprocess + + output = subprocess.run( + ["bash", "-c", 'echo "${BASH_VERSION}"'], stdout=subprocess.PIPE + ) + match = re.search(r"^(\d+)\.(\d+)\.\d+", output.stdout.decode()) + + if match is not None: + major, minor = match.groups() + + if major < "4" or major == "4" and minor < "4": + raise RuntimeError( + _( + "Shell completion is not supported for Bash" + " versions older than 4.4." + ) + ) + else: + raise RuntimeError( + _("Couldn't detect Bash version, shell completion is not supported.") + ) + + def source(self) -> str: + self._check_version() + return super().source() + + def get_completion_args(self) -> t.Tuple[t.List[str], str]: + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) + args = cwords[1:cword] + + try: + incomplete = cwords[cword] + except IndexError: + incomplete = "" + + return args, incomplete + + def format_completion(self, item: CompletionItem) -> str: + return f"{item.type},{item.value}" + + +class ZshComplete(ShellComplete): + """Shell completion for Zsh.""" + + name = "zsh" + source_template = _SOURCE_ZSH + + def get_completion_args(self) -> t.Tuple[t.List[str], str]: + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) + args = cwords[1:cword] + + try: + incomplete = cwords[cword] + except IndexError: + incomplete = "" + + return args, incomplete + + def format_completion(self, item: CompletionItem) -> str: + return f"{item.type}\n{item.value}\n{item.help if item.help else '_'}" + + +class FishComplete(ShellComplete): + """Shell completion for Fish.""" + + name = "fish" + source_template = _SOURCE_FISH + + def get_completion_args(self) -> t.Tuple[t.List[str], str]: + cwords = split_arg_string(os.environ["COMP_WORDS"]) + incomplete = os.environ["COMP_CWORD"] + args = cwords[1:] + + # Fish stores the partial word in both COMP_WORDS and + # COMP_CWORD, remove it from complete args. + if incomplete and args and args[-1] == incomplete: + args.pop() + + return args, incomplete + + def format_completion(self, item: CompletionItem) -> str: + if item.help: + return f"{item.type},{item.value}\t{item.help}" + + return f"{item.type},{item.value}" + + +ShellCompleteType = t.TypeVar("ShellCompleteType", bound=t.Type[ShellComplete]) + + +_available_shells: t.Dict[str, t.Type[ShellComplete]] = { + "bash": BashComplete, + "fish": FishComplete, + "zsh": ZshComplete, +} + + +def add_completion_class( + cls: ShellCompleteType, name: t.Optional[str] = None +) -> ShellCompleteType: + """Register a :class:`ShellComplete` subclass under the given name. + The name will be provided by the completion instruction environment + variable during completion. + + :param cls: The completion class that will handle completion for the + shell. + :param name: Name to register the class under. Defaults to the + class's ``name`` attribute. + """ + if name is None: + name = cls.name + + _available_shells[name] = cls + + return cls + + +def get_completion_class(shell: str) -> t.Optional[t.Type[ShellComplete]]: + """Look up a registered :class:`ShellComplete` subclass by the name + provided by the completion instruction environment variable. If the + name isn't registered, returns ``None``. + + :param shell: Name the class is registered under. + """ + return _available_shells.get(shell) + + +def _is_incomplete_argument(ctx: Context, param: Parameter) -> bool: + """Determine if the given parameter is an argument that can still + accept values. + + :param ctx: Invocation context for the command represented by the + parsed complete args. + :param param: Argument object being checked. + """ + if not isinstance(param, Argument): + return False + + assert param.name is not None + # Will be None if expose_value is False. + value = ctx.params.get(param.name) + return ( + param.nargs == -1 + or ctx.get_parameter_source(param.name) is not ParameterSource.COMMANDLINE + or ( + param.nargs > 1 + and isinstance(value, (tuple, list)) + and len(value) < param.nargs + ) + ) + + +def _start_of_option(ctx: Context, value: str) -> bool: + """Check if the value looks like the start of an option.""" + if not value: + return False + + c = value[0] + return c in ctx._opt_prefixes + + +def _is_incomplete_option(ctx: Context, args: t.List[str], param: Parameter) -> bool: + """Determine if the given parameter is an option that needs a value. + + :param args: List of complete args before the incomplete value. + :param param: Option object being checked. + """ + if not isinstance(param, Option): + return False + + if param.is_flag or param.count: + return False + + last_option = None + + for index, arg in enumerate(reversed(args)): + if index + 1 > param.nargs: + break + + if _start_of_option(ctx, arg): + last_option = arg + + return last_option is not None and last_option in param.opts + + +def _resolve_context( + cli: BaseCommand, + ctx_args: t.MutableMapping[str, t.Any], + prog_name: str, + args: t.List[str], +) -> Context: + """Produce the context hierarchy starting with the command and + traversing the complete arguments. This only follows the commands, + it doesn't trigger input prompts or callbacks. + + :param cli: Command being called. + :param prog_name: Name of the executable in the shell. + :param args: List of complete args before the incomplete value. + """ + ctx_args["resilient_parsing"] = True + ctx = cli.make_context(prog_name, args.copy(), **ctx_args) + args = ctx.protected_args + ctx.args + + while args: + command = ctx.command + + if isinstance(command, MultiCommand): + if not command.chain: + name, cmd, args = command.resolve_command(ctx, args) + + if cmd is None: + return ctx + + ctx = cmd.make_context(name, args, parent=ctx, resilient_parsing=True) + args = ctx.protected_args + ctx.args + else: + sub_ctx = ctx + + while args: + name, cmd, args = command.resolve_command(ctx, args) + + if cmd is None: + return ctx + + sub_ctx = cmd.make_context( + name, + args, + parent=ctx, + allow_extra_args=True, + allow_interspersed_args=False, + resilient_parsing=True, + ) + args = sub_ctx.args + + ctx = sub_ctx + args = [*sub_ctx.protected_args, *sub_ctx.args] + else: + break + + return ctx + + +def _resolve_incomplete( + ctx: Context, args: t.List[str], incomplete: str +) -> t.Tuple[t.Union[BaseCommand, Parameter], str]: + """Find the Click object that will handle the completion of the + incomplete value. Return the object and the incomplete value. + + :param ctx: Invocation context for the command represented by + the parsed complete args. + :param args: List of complete args before the incomplete value. + :param incomplete: Value being completed. May be empty. + """ + # Different shells treat an "=" between a long option name and + # value differently. Might keep the value joined, return the "=" + # as a separate item, or return the split name and value. Always + # split and discard the "=" to make completion easier. + if incomplete == "=": + incomplete = "" + elif "=" in incomplete and _start_of_option(ctx, incomplete): + name, _, incomplete = incomplete.partition("=") + args.append(name) + + # The "--" marker tells Click to stop treating values as options + # even if they start with the option character. If it hasn't been + # given and the incomplete arg looks like an option, the current + # command will provide option name completions. + if "--" not in args and _start_of_option(ctx, incomplete): + return ctx.command, incomplete + + params = ctx.command.get_params(ctx) + + # If the last complete arg is an option name with an incomplete + # value, the option will provide value completions. + for param in params: + if _is_incomplete_option(ctx, args, param): + return param, incomplete + + # It's not an option name or value. The first argument without a + # parsed value will provide value completions. + for param in params: + if _is_incomplete_argument(ctx, param): + return param, incomplete + + # There were no unparsed arguments, the command may be a group that + # will provide command name completions. + return ctx.command, incomplete diff --git a/third_party/python/click/click/termui.py b/third_party/python/click/click/termui.py index 02ef9e9f045c..db7a4b286174 100644 --- a/third_party/python/click/click/termui.py +++ b/third_party/python/click/click/termui.py @@ -1,30 +1,29 @@ import inspect import io import itertools -import os -import struct import sys +import typing as t +from gettext import gettext as _ -from ._compat import DEFAULT_COLUMNS -from ._compat import get_winterm_size from ._compat import isatty -from ._compat import raw_input -from ._compat import string_types from ._compat import strip_ansi -from ._compat import text_type -from ._compat import WIN from .exceptions import Abort from .exceptions import UsageError from .globals import resolve_color_default from .types import Choice from .types import convert_type -from .types import Path +from .types import ParamType from .utils import echo from .utils import LazyFile +if t.TYPE_CHECKING: + from ._termui_impl import ProgressBar + +V = t.TypeVar("V") + # The prompt functions to use. The doc tools currently override these # functions to customize how they work. -visible_prompt_func = raw_input +visible_prompt_func: t.Callable[[str], str] = input _ansi_colors = { "black": 30, @@ -48,24 +47,29 @@ _ansi_colors = { _ansi_reset_all = "\033[0m" -def hidden_prompt_func(prompt): +def hidden_prompt_func(prompt: str) -> str: import getpass return getpass.getpass(prompt) def _build_prompt( - text, suffix, show_default=False, default=None, show_choices=True, type=None -): + text: str, + suffix: str, + show_default: bool = False, + default: t.Optional[t.Any] = None, + show_choices: bool = True, + type: t.Optional[ParamType] = None, +) -> str: prompt = text if type is not None and show_choices and isinstance(type, Choice): - prompt += " ({})".format(", ".join(map(str, type.choices))) + prompt += f" ({', '.join(map(str, type.choices))})" if default is not None and show_default: - prompt = "{} [{}]".format(prompt, _format_default(default)) - return prompt + suffix + prompt = f"{prompt} [{_format_default(default)}]" + return f"{prompt}{suffix}" -def _format_default(default): +def _format_default(default: t.Any) -> t.Any: if isinstance(default, (io.IOBase, LazyFile)) and hasattr(default, "name"): return default.name @@ -73,38 +77,31 @@ def _format_default(default): def prompt( - text, - default=None, - hide_input=False, - confirmation_prompt=False, - type=None, - value_proc=None, - prompt_suffix=": ", - show_default=True, - err=False, - show_choices=True, -): + text: str, + default: t.Optional[t.Any] = None, + hide_input: bool = False, + confirmation_prompt: t.Union[bool, str] = False, + type: t.Optional[t.Union[ParamType, t.Any]] = None, + value_proc: t.Optional[t.Callable[[str], t.Any]] = None, + prompt_suffix: str = ": ", + show_default: bool = True, + err: bool = False, + show_choices: bool = True, +) -> t.Any: """Prompts a user for input. This is a convenience function that can be used to prompt a user for input later. - If the user aborts the input by sending a interrupt signal, this + If the user aborts the input by sending an interrupt signal, this function will catch it and raise a :exc:`Abort` exception. - .. versionadded:: 7.0 - Added the show_choices parameter. - - .. versionadded:: 6.0 - Added unicode support for cmd.exe on Windows. - - .. versionadded:: 4.0 - Added the `err` parameter. - :param text: the text to show for the prompt. :param default: the default value to use if no input happens. If this is not given it will prompt until it's aborted. :param hide_input: if this is set to true then the input value will be hidden. - :param confirmation_prompt: asks for confirmation for the value. + :param confirmation_prompt: Prompt a second time to confirm the + value. Can be set to a string instead of ``True`` to customize + the message. :param type: the type to use to check the value against. :param value_proc: if this parameter is provided it's a function that is invoked instead of the type conversion to @@ -117,23 +114,37 @@ def prompt( For example if type is a Choice of either day or week, show_choices is true and text is "Group by" then the prompt will be "Group by (day, week): ". - """ - result = None - def prompt_func(text): + .. versionadded:: 8.0 + ``confirmation_prompt`` can be a custom string. + + .. versionadded:: 7.0 + Added the ``show_choices`` parameter. + + .. versionadded:: 6.0 + Added unicode support for cmd.exe on Windows. + + .. versionadded:: 4.0 + Added the `err` parameter. + + """ + + def prompt_func(text: str) -> str: f = hidden_prompt_func if hide_input else visible_prompt_func try: # Write the prompt separately so that we get nice # coloring through colorama on Windows - echo(text, nl=False, err=err) - return f("") + echo(text.rstrip(" "), nl=False, err=err) + # Echo a space to stdout to work around an issue where + # readline causes backspace to clear the whole line. + return f(" ") except (KeyboardInterrupt, EOFError): # getpass doesn't print a newline if the user aborts input with ^C. # Allegedly this behavior is inherited from getpass(3). # A doc bug has been filed at https://bugs.python.org/issue24711 if hide_input: echo(None, err=err) - raise Abort() + raise Abort() from None if value_proc is None: value_proc = convert_type(type, default) @@ -142,72 +153,94 @@ def prompt( text, prompt_suffix, show_default, default, show_choices, type ) - while 1: - while 1: + if confirmation_prompt: + if confirmation_prompt is True: + confirmation_prompt = _("Repeat for confirmation") + + confirmation_prompt = _build_prompt(confirmation_prompt, prompt_suffix) + + while True: + while True: value = prompt_func(prompt) if value: break elif default is not None: - if isinstance(value_proc, Path): - # validate Path default value(exists, dir_okay etc.) - value = default - break - return default + value = default + break try: result = value_proc(value) except UsageError as e: - echo("Error: {}".format(e.message), err=err) # noqa: B306 + if hide_input: + echo(_("Error: The value you entered was invalid."), err=err) + else: + echo(_("Error: {e.message}").format(e=e), err=err) # noqa: B306 continue if not confirmation_prompt: return result - while 1: - value2 = prompt_func("Repeat for confirmation: ") - if value2: + while True: + value2 = prompt_func(confirmation_prompt) + is_empty = not value and not value2 + if value2 or is_empty: break if value == value2: return result - echo("Error: the two entered values do not match", err=err) + echo(_("Error: The two entered values do not match."), err=err) def confirm( - text, default=False, abort=False, prompt_suffix=": ", show_default=True, err=False -): + text: str, + default: t.Optional[bool] = False, + abort: bool = False, + prompt_suffix: str = ": ", + show_default: bool = True, + err: bool = False, +) -> bool: """Prompts for confirmation (yes/no question). If the user aborts the input by sending a interrupt signal this function will catch it and raise a :exc:`Abort` exception. - .. versionadded:: 4.0 - Added the `err` parameter. - :param text: the question to ask. - :param default: the default for the prompt. + :param default: The default value to use when no input is given. If + ``None``, repeat until input is given. :param abort: if this is set to `True` a negative answer aborts the exception by raising :exc:`Abort`. :param prompt_suffix: a suffix that should be added to the prompt. :param show_default: shows or hides the default value in the prompt. :param err: if set to true the file defaults to ``stderr`` instead of ``stdout``, the same as with echo. + + .. versionchanged:: 8.0 + Repeat until input is given if ``default`` is ``None``. + + .. versionadded:: 4.0 + Added the ``err`` parameter. """ prompt = _build_prompt( - text, prompt_suffix, show_default, "Y/n" if default else "y/N" + text, + prompt_suffix, + show_default, + "y/n" if default is None else ("Y/n" if default else "y/N"), ) - while 1: + + while True: try: # Write the prompt separately so that we get nice # coloring through colorama on Windows - echo(prompt, nl=False, err=err) - value = visible_prompt_func("").lower().strip() + echo(prompt.rstrip(" "), nl=False, err=err) + # Echo a space to stdout to work around an issue where + # readline causes backspace to clear the whole line. + value = visible_prompt_func(" ").lower().strip() except (KeyboardInterrupt, EOFError): - raise Abort() + raise Abort() from None if value in ("y", "yes"): rv = True elif value in ("n", "no"): rv = False - elif value == "": + elif default is not None and value == "": rv = default else: - echo("Error: invalid input", err=err) + echo(_("Error: invalid input"), err=err) continue break if abort and not rv: @@ -215,54 +248,10 @@ def confirm( return rv -def get_terminal_size(): - """Returns the current size of the terminal as tuple in the form - ``(width, height)`` in columns and rows. - """ - # If shutil has get_terminal_size() (Python 3.3 and later) use that - if sys.version_info >= (3, 3): - import shutil - - shutil_get_terminal_size = getattr(shutil, "get_terminal_size", None) - if shutil_get_terminal_size: - sz = shutil_get_terminal_size() - return sz.columns, sz.lines - - # We provide a sensible default for get_winterm_size() when being invoked - # inside a subprocess. Without this, it would not provide a useful input. - if get_winterm_size is not None: - size = get_winterm_size() - if size == (0, 0): - return (79, 24) - else: - return size - - def ioctl_gwinsz(fd): - try: - import fcntl - import termios - - cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234")) - except Exception: - return - return cr - - cr = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - try: - cr = ioctl_gwinsz(fd) - finally: - os.close(fd) - except Exception: - pass - if not cr or not cr[0] or not cr[1]: - cr = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", DEFAULT_COLUMNS)) - return int(cr[1]), int(cr[0]) - - -def echo_via_pager(text_or_generator, color=None): +def echo_via_pager( + text_or_generator: t.Union[t.Iterable[str], t.Callable[[], t.Iterable[str]], str], + color: t.Optional[bool] = None, +) -> None: """This function takes a text and shows it via an environment specific pager on stdout. @@ -277,14 +266,14 @@ def echo_via_pager(text_or_generator, color=None): color = resolve_color_default(color) if inspect.isgeneratorfunction(text_or_generator): - i = text_or_generator() - elif isinstance(text_or_generator, string_types): + i = t.cast(t.Callable[[], t.Iterable[str]], text_or_generator)() + elif isinstance(text_or_generator, str): i = [text_or_generator] else: - i = iter(text_or_generator) + i = iter(t.cast(t.Iterable[str], text_or_generator)) # convert every element of i to a text type if necessary - text_generator = (el if isinstance(el, string_types) else text_type(el) for el in i) + text_generator = (el if isinstance(el, str) else str(el) for el in i) from ._termui_impl import pager @@ -292,21 +281,22 @@ def echo_via_pager(text_or_generator, color=None): def progressbar( - iterable=None, - length=None, - label=None, - show_eta=True, - show_percent=None, - show_pos=False, - item_show_func=None, - fill_char="#", - empty_char="-", - bar_template="%(label)s [%(bar)s] %(info)s", - info_sep=" ", - width=36, - file=None, - color=None, -): + iterable: t.Optional[t.Iterable[V]] = None, + length: t.Optional[int] = None, + label: t.Optional[str] = None, + show_eta: bool = True, + show_percent: t.Optional[bool] = None, + show_pos: bool = False, + item_show_func: t.Optional[t.Callable[[t.Optional[V]], t.Optional[str]]] = None, + fill_char: str = "#", + empty_char: str = "-", + bar_template: str = "%(label)s [%(bar)s] %(info)s", + info_sep: str = " ", + width: int = 36, + file: t.Optional[t.TextIO] = None, + color: t.Optional[bool] = None, + update_min_steps: int = 1, +) -> "ProgressBar[V]": """This function creates an iterable context manager that can be used to iterate over something while showing a progress bar. It will either iterate over the `iterable` or `length` items (that are counted @@ -346,11 +336,19 @@ def progressbar( process_chunk(chunk) bar.update(chunks.bytes) - .. versionadded:: 2.0 + The ``update()`` method also takes an optional value specifying the + ``current_item`` at the new position. This is useful when used + together with ``item_show_func`` to customize the output for each + manual step:: - .. versionadded:: 4.0 - Added the `color` parameter. Added a `update` method to the - progressbar object. + with click.progressbar( + length=total_size, + label='Unzipping archive', + item_show_func=lambda a: a.filename + ) as bar: + for archive in zip_file: + archive.extract() + bar.update(archive.size, archive) :param iterable: an iterable to iterate over. If not provided the length is required. @@ -369,10 +367,10 @@ def progressbar( `False` if not. :param show_pos: enables or disables the absolute position display. The default is `False`. - :param item_show_func: a function called with the current item which - can return a string to show the current item - next to the progress bar. Note that the current - item can be `None`! + :param item_show_func: A function called with the current item which + can return a string to show next to the progress bar. If the + function returns ``None`` nothing is shown. The current item can + be ``None``, such as when entering and exiting the bar. :param fill_char: the character to use to show the filled part of the progress bar. :param empty_char: the character to use to show the non-filled part of @@ -384,12 +382,33 @@ def progressbar( :param info_sep: the separator between multiple info items (eta etc.) :param width: the width of the progress bar in characters, 0 means full terminal width - :param file: the file to write to. If this is not a terminal then - only the label is printed. + :param file: The file to write to. If this is not a terminal then + only the label is printed. :param color: controls if the terminal supports ANSI colors or not. The default is autodetection. This is only needed if ANSI codes are included anywhere in the progress bar output which is not the case by default. + :param update_min_steps: Render only when this many updates have + completed. This allows tuning for very fast iterators. + + .. versionchanged:: 8.0 + Output is shown even if execution time is less than 0.5 seconds. + + .. versionchanged:: 8.0 + ``item_show_func`` shows the current item, not the previous one. + + .. versionchanged:: 8.0 + Labels are echoed if the output is not a TTY. Reverts a change + in 7.0 that removed all output. + + .. versionadded:: 8.0 + Added the ``update_min_steps`` parameter. + + .. versionchanged:: 4.0 + Added the ``color`` parameter. Added the ``update`` method to + the object. + + .. versionadded:: 2.0 """ from ._termui_impl import ProgressBar @@ -409,10 +428,11 @@ def progressbar( label=label, width=width, color=color, + update_min_steps=update_min_steps, ) -def clear(): +def clear() -> None: """Clears the terminal screen. This will have the effect of clearing the whole visible space of the terminal and moving the cursor to the top left. This does not do anything if not connected to a terminal. @@ -421,26 +441,38 @@ def clear(): """ if not isatty(sys.stdout): return - # If we're on Windows and we don't have colorama available, then we - # clear the screen by shelling out. Otherwise we can use an escape - # sequence. - if WIN: - os.system("cls") - else: - sys.stdout.write("\033[2J\033[1;1H") + + # ANSI escape \033[2J clears the screen, \033[1;1H moves the cursor + echo("\033[2J\033[1;1H", nl=False) + + +def _interpret_color( + color: t.Union[int, t.Tuple[int, int, int], str], offset: int = 0 +) -> str: + if isinstance(color, int): + return f"{38 + offset};5;{color:d}" + + if isinstance(color, (tuple, list)): + r, g, b = color + return f"{38 + offset};2;{r:d};{g:d};{b:d}" + + return str(_ansi_colors[color] + offset) def style( - text, - fg=None, - bg=None, - bold=None, - dim=None, - underline=None, - blink=None, - reverse=None, - reset=True, -): + text: t.Any, + fg: t.Optional[t.Union[int, t.Tuple[int, int, int], str]] = None, + bg: t.Optional[t.Union[int, t.Tuple[int, int, int], str]] = None, + bold: t.Optional[bool] = None, + dim: t.Optional[bool] = None, + underline: t.Optional[bool] = None, + overline: t.Optional[bool] = None, + italic: t.Optional[bool] = None, + blink: t.Optional[bool] = None, + reverse: t.Optional[bool] = None, + strikethrough: t.Optional[bool] = None, + reset: bool = True, +) -> str: """Styles a text with ANSI styles and returns the new string. By default the styling is self contained which means that at the end of the string a reset code is issued. This can be prevented by @@ -451,6 +483,7 @@ def style( click.echo(click.style('Hello World!', fg='green')) click.echo(click.style('ATTENTION!', blink=True)) click.echo(click.style('Some things', reverse=True, fg='cyan')) + click.echo(click.style('More colors', fg=(255, 12, 128), bg=117)) Supported color names: @@ -472,10 +505,15 @@ def style( * ``bright_white`` * ``reset`` (reset the color code only) - .. versionadded:: 2.0 + If the terminal supports it, color may also be specified as: - .. versionadded:: 7.0 - Added support for bright colors. + - An integer in the interval [0, 255]. The terminal must support + 8-bit/256-color mode. + - An RGB tuple of three integers in [0, 255]. The terminal must + support 24-bit/true-color mode. + + See https://en.wikipedia.org/wiki/ANSI_color and + https://gist.github.com/XVilka/8346728 for more information. :param text: the string to style with ansi codes. :param fg: if provided this will become the foreground color. @@ -484,42 +522,73 @@ def style( :param dim: if provided this will enable or disable dim mode. This is badly supported. :param underline: if provided this will enable or disable underline. + :param overline: if provided this will enable or disable overline. + :param italic: if provided this will enable or disable italic. :param blink: if provided this will enable or disable blinking. :param reverse: if provided this will enable or disable inverse rendering (foreground becomes background and the other way round). + :param strikethrough: if provided this will enable or disable + striking through text. :param reset: by default a reset-all code is added at the end of the string which means that styles do not carry over. This can be disabled to compose styles. + + .. versionchanged:: 8.0 + A non-string ``message`` is converted to a string. + + .. versionchanged:: 8.0 + Added support for 256 and RGB color codes. + + .. versionchanged:: 8.0 + Added the ``strikethrough``, ``italic``, and ``overline`` + parameters. + + .. versionchanged:: 7.0 + Added support for bright colors. + + .. versionadded:: 2.0 """ + if not isinstance(text, str): + text = str(text) + bits = [] + if fg: try: - bits.append("\033[{}m".format(_ansi_colors[fg])) + bits.append(f"\033[{_interpret_color(fg)}m") except KeyError: - raise TypeError("Unknown color '{}'".format(fg)) + raise TypeError(f"Unknown color {fg!r}") from None + if bg: try: - bits.append("\033[{}m".format(_ansi_colors[bg] + 10)) + bits.append(f"\033[{_interpret_color(bg, 10)}m") except KeyError: - raise TypeError("Unknown color '{}'".format(bg)) + raise TypeError(f"Unknown color {bg!r}") from None + if bold is not None: - bits.append("\033[{}m".format(1 if bold else 22)) + bits.append(f"\033[{1 if bold else 22}m") if dim is not None: - bits.append("\033[{}m".format(2 if dim else 22)) + bits.append(f"\033[{2 if dim else 22}m") if underline is not None: - bits.append("\033[{}m".format(4 if underline else 24)) + bits.append(f"\033[{4 if underline else 24}m") + if overline is not None: + bits.append(f"\033[{53 if overline else 55}m") + if italic is not None: + bits.append(f"\033[{3 if italic else 23}m") if blink is not None: - bits.append("\033[{}m".format(5 if blink else 25)) + bits.append(f"\033[{5 if blink else 25}m") if reverse is not None: - bits.append("\033[{}m".format(7 if reverse else 27)) + bits.append(f"\033[{7 if reverse else 27}m") + if strikethrough is not None: + bits.append(f"\033[{9 if strikethrough else 29}m") bits.append(text) if reset: bits.append(_ansi_reset_all) return "".join(bits) -def unstyle(text): +def unstyle(text: str) -> str: """Removes ANSI styling information from a string. Usually it's not necessary to use this function as Click's echo function will automatically remove styling if necessary. @@ -531,7 +600,14 @@ def unstyle(text): return strip_ansi(text) -def secho(message=None, file=None, nl=True, err=False, color=None, **styles): +def secho( + message: t.Optional[t.Any] = None, + file: t.Optional[t.IO[t.AnyStr]] = None, + nl: bool = True, + err: bool = False, + color: t.Optional[bool] = None, + **styles: t.Any, +) -> None: """This function combines :func:`echo` and :func:`style` into one call. As such the following two calls are the same:: @@ -541,16 +617,31 @@ def secho(message=None, file=None, nl=True, err=False, color=None, **styles): All keyword arguments are forwarded to the underlying functions depending on which one they go with. + Non-string types will be converted to :class:`str`. However, + :class:`bytes` are passed directly to :meth:`echo` without applying + style. If you want to style bytes that represent text, call + :meth:`bytes.decode` first. + + .. versionchanged:: 8.0 + A non-string ``message`` is converted to a string. Bytes are + passed through without style applied. + .. versionadded:: 2.0 """ - if message is not None: + if message is not None and not isinstance(message, (bytes, bytearray)): message = style(message, **styles) + return echo(message, file=file, nl=nl, err=err, color=color) def edit( - text=None, editor=None, env=None, require_save=True, extension=".txt", filename=None -): + text: t.Optional[t.AnyStr] = None, + editor: t.Optional[str] = None, + env: t.Optional[t.Mapping[str, str]] = None, + require_save: bool = True, + extension: str = ".txt", + filename: t.Optional[str] = None, +) -> t.Optional[t.AnyStr]: r"""Edits the given text in the defined editor. If an editor is given (should be the full path to the executable but the regular operating system search path is used for finding the executable) it overrides @@ -580,15 +671,16 @@ def edit( """ from ._termui_impl import Editor - editor = Editor( - editor=editor, env=env, require_save=require_save, extension=extension - ) + ed = Editor(editor=editor, env=env, require_save=require_save, extension=extension) + if filename is None: - return editor.edit(text) - editor.edit_file(filename) + return ed.edit(text) + + ed.edit_file(filename) + return None -def launch(url, wait=False, locate=False): +def launch(url: str, wait: bool = False, locate: bool = False) -> int: """This function launches the given URL (or filename) in the default viewer application for this file type. If this is an executable, it might launch the executable in a new session. The return value is @@ -603,7 +695,9 @@ def launch(url, wait=False, locate=False): .. versionadded:: 2.0 :param url: URL or filename of the thing to launch. - :param wait: waits for the program to stop. + :param wait: Wait for the program to exit before returning. This + only works if the launched program blocks. In particular, + ``xdg-open`` on Linux does not block. :param locate: if this is set to `True` then instead of launching the application associated with the URL it will attempt to launch a file manager with the file located. This @@ -617,10 +711,10 @@ def launch(url, wait=False, locate=False): # If this is provided, getchar() calls into this instead. This is used # for unittesting purposes. -_getchar = None +_getchar: t.Optional[t.Callable[[bool], str]] = None -def getchar(echo=False): +def getchar(echo: bool = False) -> str: """Fetches a single character from the terminal and returns it. This will always return a unicode character and under certain rare circumstances this might return more than one character. The @@ -640,19 +734,23 @@ def getchar(echo=False): :param echo: if set to `True`, the character read will also show up on the terminal. The default is to not show it. """ - f = _getchar - if f is None: + global _getchar + + if _getchar is None: from ._termui_impl import getchar as f - return f(echo) + + _getchar = f + + return _getchar(echo) -def raw_terminal(): +def raw_terminal() -> t.ContextManager[int]: from ._termui_impl import raw_terminal as f return f() -def pause(info="Press any key to continue ...", err=False): +def pause(info: t.Optional[str] = None, err: bool = False) -> None: """This command stops execution and waits for the user to press any key to continue. This is similar to the Windows batch "pause" command. If the program is not run through a terminal, this command @@ -663,12 +761,17 @@ def pause(info="Press any key to continue ...", err=False): .. versionadded:: 4.0 Added the `err` parameter. - :param info: the info string to print before pausing. + :param info: The message to print before pausing. Defaults to + ``"Press any key to continue..."``. :param err: if set to message goes to ``stderr`` instead of ``stdout``, the same as with echo. """ if not isatty(sys.stdin) or not isatty(sys.stdout): return + + if info is None: + info = _("Press any key to continue...") + try: if info: echo(info, nl=False, err=err) diff --git a/third_party/python/click/click/testing.py b/third_party/python/click/click/testing.py index a3dba3b3014b..e0df0d2a657f 100644 --- a/third_party/python/click/click/testing.py +++ b/third_party/python/click/click/testing.py @@ -1,77 +1,117 @@ import contextlib +import io import os import shlex import shutil import sys import tempfile +import typing as t +from types import TracebackType from . import formatting from . import termui from . import utils -from ._compat import iteritems -from ._compat import PY2 -from ._compat import string_types +from ._compat import _find_binary_reader + +if t.TYPE_CHECKING: + from .core import BaseCommand -if PY2: - from cStringIO import StringIO -else: - import io - from ._compat import _find_binary_reader - - -class EchoingStdin(object): - def __init__(self, input, output): +class EchoingStdin: + def __init__(self, input: t.BinaryIO, output: t.BinaryIO) -> None: self._input = input self._output = output + self._paused = False - def __getattr__(self, x): + def __getattr__(self, x: str) -> t.Any: return getattr(self._input, x) - def _echo(self, rv): - self._output.write(rv) + def _echo(self, rv: bytes) -> bytes: + if not self._paused: + self._output.write(rv) + return rv - def read(self, n=-1): + def read(self, n: int = -1) -> bytes: return self._echo(self._input.read(n)) - def readline(self, n=-1): + def read1(self, n: int = -1) -> bytes: + return self._echo(self._input.read1(n)) # type: ignore + + def readline(self, n: int = -1) -> bytes: return self._echo(self._input.readline(n)) - def readlines(self): + def readlines(self) -> t.List[bytes]: return [self._echo(x) for x in self._input.readlines()] - def __iter__(self): + def __iter__(self) -> t.Iterator[bytes]: return iter(self._echo(x) for x in self._input) - def __repr__(self): + def __repr__(self) -> str: return repr(self._input) -def make_input_stream(input, charset): +@contextlib.contextmanager +def _pause_echo(stream: t.Optional[EchoingStdin]) -> t.Iterator[None]: + if stream is None: + yield + else: + stream._paused = True + yield + stream._paused = False + + +class _NamedTextIOWrapper(io.TextIOWrapper): + def __init__( + self, buffer: t.BinaryIO, name: str, mode: str, **kwargs: t.Any + ) -> None: + super().__init__(buffer, **kwargs) + self._name = name + self._mode = mode + + @property + def name(self) -> str: + return self._name + + @property + def mode(self) -> str: + return self._mode + + +def make_input_stream( + input: t.Optional[t.Union[str, bytes, t.IO[t.Any]]], charset: str +) -> t.BinaryIO: # Is already an input stream. if hasattr(input, "read"): - if PY2: - return input - rv = _find_binary_reader(input) + rv = _find_binary_reader(t.cast(t.IO[t.Any], input)) + if rv is not None: return rv + raise TypeError("Could not find binary reader for input stream.") if input is None: input = b"" - elif not isinstance(input, bytes): + elif isinstance(input, str): input = input.encode(charset) - if PY2: - return StringIO(input) + return io.BytesIO(input) -class Result(object): +class Result: """Holds the captured result of an invoked CLI script.""" def __init__( - self, runner, stdout_bytes, stderr_bytes, exit_code, exception, exc_info=None + self, + runner: "CliRunner", + stdout_bytes: bytes, + stderr_bytes: t.Optional[bytes], + return_value: t.Any, + exit_code: int, + exception: t.Optional[BaseException], + exc_info: t.Optional[ + t.Tuple[t.Type[BaseException], BaseException, TracebackType] + ] = None, ): #: The runner that created the result self.runner = runner @@ -79,6 +119,10 @@ class Result(object): self.stdout_bytes = stdout_bytes #: The standard error as bytes, or None if not available self.stderr_bytes = stderr_bytes + #: The value returned from the invoked command. + #: + #: .. versionadded:: 8.0 + self.return_value = return_value #: The exit code as integer. self.exit_code = exit_code #: The exception that happened if one did. @@ -87,19 +131,19 @@ class Result(object): self.exc_info = exc_info @property - def output(self): + def output(self) -> str: """The (standard) output as unicode string.""" return self.stdout @property - def stdout(self): + def stdout(self) -> str: """The standard output as unicode string.""" return self.stdout_bytes.decode(self.runner.charset, "replace").replace( "\r\n", "\n" ) @property - def stderr(self): + def stderr(self) -> str: """The standard error as unicode string.""" if self.stderr_bytes is None: raise ValueError("stderr not separately captured") @@ -107,21 +151,18 @@ class Result(object): "\r\n", "\n" ) - def __repr__(self): - return "<{} {}>".format( - type(self).__name__, repr(self.exception) if self.exception else "okay" - ) + def __repr__(self) -> str: + exc_str = repr(self.exception) if self.exception else "okay" + return f"<{type(self).__name__} {exc_str}>" -class CliRunner(object): +class CliRunner: """The CLI runner provides functionality to invoke a Click command line script for unittesting purposes in a isolated environment. This only works in single-threaded systems without any concurrency as it changes the global interpreter state. - :param charset: the character set for the input and output data. This is - UTF-8 by default and should not be changed currently as - the reporting to Click only works in Python 2 properly. + :param charset: the character set for the input and output data. :param env: a dictionary with environment variables for overriding. :param echo_stdin: if this is set to `True`, then reading from stdin writes to stdout. This is useful for showing examples in @@ -134,22 +175,28 @@ class CliRunner(object): independently """ - def __init__(self, charset=None, env=None, echo_stdin=False, mix_stderr=True): - if charset is None: - charset = "utf-8" + def __init__( + self, + charset: str = "utf-8", + env: t.Optional[t.Mapping[str, t.Optional[str]]] = None, + echo_stdin: bool = False, + mix_stderr: bool = True, + ) -> None: self.charset = charset - self.env = env or {} + self.env: t.Mapping[str, t.Optional[str]] = env or {} self.echo_stdin = echo_stdin self.mix_stderr = mix_stderr - def get_default_prog_name(self, cli): + def get_default_prog_name(self, cli: "BaseCommand") -> str: """Given a command object it will return the default program name for it. The default is the `name` attribute or ``"root"`` if not set. """ return cli.name or "root" - def make_env(self, overrides=None): + def make_env( + self, overrides: t.Optional[t.Mapping[str, t.Optional[str]]] = None + ) -> t.Mapping[str, t.Optional[str]]: """Returns the environment overrides for invoking a script.""" rv = dict(self.env) if overrides: @@ -157,7 +204,12 @@ class CliRunner(object): return rv @contextlib.contextmanager - def isolation(self, input=None, env=None, color=False): + def isolation( + self, + input: t.Optional[t.Union[str, bytes, t.IO[t.Any]]] = None, + env: t.Optional[t.Mapping[str, t.Optional[str]]] = None, + color: bool = False, + ) -> t.Iterator[t.Tuple[io.BytesIO, t.Optional[io.BytesIO]]]: """A context manager that sets up the isolation for invoking of a command line tool. This sets up stdin with the given input data and `os.environ` with the overrides from the given dictionary. @@ -166,15 +218,20 @@ class CliRunner(object): This is automatically done in the :meth:`invoke` method. - .. versionadded:: 4.0 - The ``color`` parameter was added. - :param input: the input stream to put into sys.stdin. :param env: the environment overrides as dictionary. :param color: whether the output should contain color codes. The application can still override this explicitly. + + .. versionchanged:: 8.0 + ``stderr`` is opened with ``errors="backslashreplace"`` + instead of the default ``"strict"``. + + .. versionchanged:: 4.0 + Added the ``color`` parameter. """ - input = make_input_stream(input, self.charset) + bytes_input = make_input_stream(input, self.charset) + echo_input = None old_stdin = sys.stdin old_stdout = sys.stdout @@ -184,51 +241,68 @@ class CliRunner(object): env = self.make_env(env) - if PY2: - bytes_output = StringIO() - if self.echo_stdin: - input = EchoingStdin(input, bytes_output) - sys.stdout = bytes_output - if not self.mix_stderr: - bytes_error = StringIO() - sys.stderr = bytes_error - else: - bytes_output = io.BytesIO() - if self.echo_stdin: - input = EchoingStdin(input, bytes_output) - input = io.TextIOWrapper(input, encoding=self.charset) - sys.stdout = io.TextIOWrapper(bytes_output, encoding=self.charset) - if not self.mix_stderr: - bytes_error = io.BytesIO() - sys.stderr = io.TextIOWrapper(bytes_error, encoding=self.charset) + bytes_output = io.BytesIO() + if self.echo_stdin: + bytes_input = echo_input = t.cast( + t.BinaryIO, EchoingStdin(bytes_input, bytes_output) + ) + + sys.stdin = text_input = _NamedTextIOWrapper( + bytes_input, encoding=self.charset, name="", mode="r" + ) + + if self.echo_stdin: + # Force unbuffered reads, otherwise TextIOWrapper reads a + # large chunk which is echoed early. + text_input._CHUNK_SIZE = 1 # type: ignore + + sys.stdout = _NamedTextIOWrapper( + bytes_output, encoding=self.charset, name="", mode="w" + ) + + bytes_error = None if self.mix_stderr: sys.stderr = sys.stdout + else: + bytes_error = io.BytesIO() + sys.stderr = _NamedTextIOWrapper( + bytes_error, + encoding=self.charset, + name="", + mode="w", + errors="backslashreplace", + ) - sys.stdin = input - - def visible_input(prompt=None): + @_pause_echo(echo_input) # type: ignore + def visible_input(prompt: t.Optional[str] = None) -> str: sys.stdout.write(prompt or "") - val = input.readline().rstrip("\r\n") - sys.stdout.write("{}\n".format(val)) + val = text_input.readline().rstrip("\r\n") + sys.stdout.write(f"{val}\n") sys.stdout.flush() return val - def hidden_input(prompt=None): - sys.stdout.write("{}\n".format(prompt or "")) + @_pause_echo(echo_input) # type: ignore + def hidden_input(prompt: t.Optional[str] = None) -> str: + sys.stdout.write(f"{prompt or ''}\n") sys.stdout.flush() - return input.readline().rstrip("\r\n") + return text_input.readline().rstrip("\r\n") - def _getchar(echo): + @_pause_echo(echo_input) # type: ignore + def _getchar(echo: bool) -> str: char = sys.stdin.read(1) + if echo: sys.stdout.write(char) - sys.stdout.flush() + + sys.stdout.flush() return char default_color = color - def should_strip_ansi(stream=None, color=None): + def should_strip_ansi( + stream: t.Optional[t.IO[t.Any]] = None, color: t.Optional[bool] = None + ) -> bool: if color is None: return not default_color return not color @@ -236,15 +310,15 @@ class CliRunner(object): old_visible_prompt_func = termui.visible_prompt_func old_hidden_prompt_func = termui.hidden_prompt_func old__getchar_func = termui._getchar - old_should_strip_ansi = utils.should_strip_ansi + old_should_strip_ansi = utils.should_strip_ansi # type: ignore termui.visible_prompt_func = visible_input termui.hidden_prompt_func = hidden_input termui._getchar = _getchar - utils.should_strip_ansi = should_strip_ansi + utils.should_strip_ansi = should_strip_ansi # type: ignore old_env = {} try: - for key, value in iteritems(env): + for key, value in env.items(): old_env[key] = os.environ.get(key) if value is None: try: @@ -253,9 +327,9 @@ class CliRunner(object): pass else: os.environ[key] = value - yield (bytes_output, not self.mix_stderr and bytes_error) + yield (bytes_output, bytes_error) finally: - for key, value in iteritems(old_env): + for key, value in old_env.items(): if value is None: try: del os.environ[key] @@ -269,19 +343,19 @@ class CliRunner(object): termui.visible_prompt_func = old_visible_prompt_func termui.hidden_prompt_func = old_hidden_prompt_func termui._getchar = old__getchar_func - utils.should_strip_ansi = old_should_strip_ansi + utils.should_strip_ansi = old_should_strip_ansi # type: ignore formatting.FORCED_WIDTH = old_forced_width def invoke( self, - cli, - args=None, - input=None, - env=None, - catch_exceptions=True, - color=False, - **extra - ): + cli: "BaseCommand", + args: t.Optional[t.Union[str, t.Sequence[str]]] = None, + input: t.Optional[t.Union[str, bytes, t.IO[t.Any]]] = None, + env: t.Optional[t.Mapping[str, t.Optional[str]]] = None, + catch_exceptions: bool = True, + color: bool = False, + **extra: t.Any, + ) -> Result: """Invokes a command in an isolated environment. The arguments are forwarded directly to the command line script, the `extra` keyword arguments are passed to the :meth:`~clickpkg.Command.main` function of @@ -289,16 +363,6 @@ class CliRunner(object): This returns a :class:`Result` object. - .. versionadded:: 3.0 - The ``catch_exceptions`` parameter was added. - - .. versionchanged:: 3.0 - The result object now has an `exc_info` attribute with the - traceback if available. - - .. versionadded:: 4.0 - The ``color`` parameter was added. - :param cli: the command to invoke :param args: the arguments to invoke. It may be given as an iterable or a string. When given as string it will be interpreted @@ -311,13 +375,28 @@ class CliRunner(object): :param extra: the keyword arguments to pass to :meth:`main`. :param color: whether the output should contain color codes. The application can still override this explicitly. + + .. versionchanged:: 8.0 + The result object has the ``return_value`` attribute with + the value returned from the invoked command. + + .. versionchanged:: 4.0 + Added the ``color`` parameter. + + .. versionchanged:: 3.0 + Added the ``catch_exceptions`` parameter. + + .. versionchanged:: 3.0 + The result object has the ``exc_info`` attribute with the + traceback if available. """ exc_info = None with self.isolation(input=input, env=env, color=color) as outstreams: - exception = None + return_value = None + exception: t.Optional[BaseException] = None exit_code = 0 - if isinstance(args, string_types): + if isinstance(args, str): args = shlex.split(args) try: @@ -326,20 +405,23 @@ class CliRunner(object): prog_name = self.get_default_prog_name(cli) try: - cli.main(args=args or (), prog_name=prog_name, **extra) + return_value = cli.main(args=args or (), prog_name=prog_name, **extra) except SystemExit as e: exc_info = sys.exc_info() - exit_code = e.code - if exit_code is None: - exit_code = 0 + e_code = t.cast(t.Optional[t.Union[int, t.Any]], e.code) - if exit_code != 0: + if e_code is None: + e_code = 0 + + if e_code != 0: exception = e - if not isinstance(exit_code, int): - sys.stdout.write(str(exit_code)) + if not isinstance(e_code, int): + sys.stdout.write(str(e_code)) sys.stdout.write("\n") - exit_code = 1 + e_code = 1 + + exit_code = e_code except Exception as e: if not catch_exceptions: @@ -353,30 +435,45 @@ class CliRunner(object): if self.mix_stderr: stderr = None else: - stderr = outstreams[1].getvalue() + stderr = outstreams[1].getvalue() # type: ignore return Result( runner=self, stdout_bytes=stdout, stderr_bytes=stderr, + return_value=return_value, exit_code=exit_code, exception=exception, - exc_info=exc_info, + exc_info=exc_info, # type: ignore ) @contextlib.contextmanager - def isolated_filesystem(self): - """A context manager that creates a temporary folder and changes - the current working directory to it for isolated filesystem tests. + def isolated_filesystem( + self, temp_dir: t.Optional[t.Union[str, "os.PathLike[str]"]] = None + ) -> t.Iterator[str]: + """A context manager that creates a temporary directory and + changes the current working directory to it. This isolates tests + that affect the contents of the CWD to prevent them from + interfering with each other. + + :param temp_dir: Create the temporary directory under this + directory. If given, the created directory is not removed + when exiting. + + .. versionchanged:: 8.0 + Added the ``temp_dir`` parameter. """ cwd = os.getcwd() - t = tempfile.mkdtemp() - os.chdir(t) + dt = tempfile.mkdtemp(dir=temp_dir) + os.chdir(dt) + try: - yield t + yield dt finally: os.chdir(cwd) - try: - shutil.rmtree(t) - except (OSError, IOError): # noqa: B014 - pass + + if temp_dir is None: + try: + shutil.rmtree(dt) + except OSError: # noqa: B014 + pass diff --git a/third_party/python/click/click/types.py b/third_party/python/click/click/types.py index 505c39f85092..2b1d1797f2e1 100644 --- a/third_party/python/click/click/types.py +++ b/third_party/python/click/click/types.py @@ -1,37 +1,48 @@ import os import stat +import sys +import typing as t from datetime import datetime +from gettext import gettext as _ +from gettext import ngettext from ._compat import _get_argv_encoding -from ._compat import filename_to_ui -from ._compat import get_filesystem_encoding -from ._compat import get_streerror from ._compat import open_stream -from ._compat import PY2 -from ._compat import text_type from .exceptions import BadParameter +from .utils import format_filename from .utils import LazyFile from .utils import safecall +if t.TYPE_CHECKING: + import typing_extensions as te + from .core import Context + from .core import Parameter + from .shell_completion import CompletionItem -class ParamType(object): - """Helper for converting values through types. The following is - necessary for a valid type: - * it needs a name - * it needs to pass through None unchanged - * it needs to convert from a string - * it needs to convert its result type through unchanged - (eg: needs to be idempotent) - * it needs to be able to deal with param and context being `None`. - This can be the case when the object is used with prompt - inputs. +class ParamType: + """Represents the type of a parameter. Validates and converts values + from the command line or Python into the correct type. + + To implement a custom type, subclass and implement at least the + following: + + - The :attr:`name` class attribute must be set. + - Calling an instance of the type with ``None`` must return + ``None``. This is already implemented by default. + - :meth:`convert` must convert string values to the correct type. + - :meth:`convert` must accept values that are already the correct + type. + - It must be able to convert a value if the ``ctx`` and ``param`` + arguments are ``None``. This can occur when converting prompt + input. """ - is_composite = False + is_composite: t.ClassVar[bool] = False + arity: t.ClassVar[int] = 1 #: the descriptive name of this type - name = None + name: str #: if a list of this type is expected and the value is pulled from a #: string environment variable, this is what splits it up. `None` @@ -39,29 +50,73 @@ class ParamType(object): #: whitespace splits them up. The exception are paths and files which #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on #: Windows). - envvar_list_splitter = None + envvar_list_splitter: t.ClassVar[t.Optional[str]] = None - def __call__(self, value, param=None, ctx=None): + def to_info_dict(self) -> t.Dict[str, t.Any]: + """Gather information that could be useful for a tool generating + user-facing documentation. + + Use :meth:`click.Context.to_info_dict` to traverse the entire + CLI structure. + + .. versionadded:: 8.0 + """ + # The class name without the "ParamType" suffix. + param_type = type(self).__name__.partition("ParamType")[0] + param_type = param_type.partition("ParameterType")[0] + + # Custom subclasses might not remember to set a name. + if hasattr(self, "name"): + name = self.name + else: + name = param_type + + return {"param_type": param_type, "name": name} + + def __call__( + self, + value: t.Any, + param: t.Optional["Parameter"] = None, + ctx: t.Optional["Context"] = None, + ) -> t.Any: if value is not None: return self.convert(value, param, ctx) - def get_metavar(self, param): + def get_metavar(self, param: "Parameter") -> t.Optional[str]: """Returns the metavar default for this param if it provides one.""" - def get_missing_message(self, param): + def get_missing_message(self, param: "Parameter") -> t.Optional[str]: """Optionally might return extra information about a missing parameter. .. versionadded:: 2.0 """ - def convert(self, value, param, ctx): - """Converts the value. This is not invoked for values that are - `None` (the missing value). + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: + """Convert the value to the correct type. This is not called if + the value is ``None`` (the missing value). + + This must accept string values from the command line, as well as + values that are already the correct type. It may also convert + other compatible types. + + The ``param`` and ``ctx`` arguments may be ``None`` in certain + situations, such as when converting prompt input. + + If the value cannot be converted, call :meth:`fail` with a + descriptive message. + + :param value: The value to convert. + :param param: The parameter that is using this type to convert + its value. May be ``None``. + :param ctx: The current context that arrived at this value. May + be ``None``. """ return value - def split_envvar_value(self, rv): + def split_envvar_value(self, rv: str) -> t.Sequence[str]: """Given a value from an environment variable this splits it up into small chunks depending on the defined envvar list splitter. @@ -71,55 +126,89 @@ class ParamType(object): """ return (rv or "").split(self.envvar_list_splitter) - def fail(self, message, param=None, ctx=None): + def fail( + self, + message: str, + param: t.Optional["Parameter"] = None, + ctx: t.Optional["Context"] = None, + ) -> "t.NoReturn": """Helper method to fail with an invalid value message.""" raise BadParameter(message, ctx=ctx, param=param) + def shell_complete( + self, ctx: "Context", param: "Parameter", incomplete: str + ) -> t.List["CompletionItem"]: + """Return a list of + :class:`~click.shell_completion.CompletionItem` objects for the + incomplete value. Most types do not provide completions, but + some do, and this allows custom types to provide custom + completions as well. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + return [] + class CompositeParamType(ParamType): is_composite = True @property - def arity(self): + def arity(self) -> int: # type: ignore raise NotImplementedError() class FuncParamType(ParamType): - def __init__(self, func): - self.name = func.__name__ + def __init__(self, func: t.Callable[[t.Any], t.Any]) -> None: + self.name: str = func.__name__ self.func = func - def convert(self, value, param, ctx): + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict["func"] = self.func + return info_dict + + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: try: return self.func(value) except ValueError: try: - value = text_type(value) + value = str(value) except UnicodeError: - value = str(value).decode("utf-8", "replace") + value = value.decode("utf-8", "replace") + self.fail(value, param, ctx) class UnprocessedParamType(ParamType): name = "text" - def convert(self, value, param, ctx): + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: return value - def __repr__(self): + def __repr__(self) -> str: return "UNPROCESSED" class StringParamType(ParamType): name = "text" - def convert(self, value, param, ctx): + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: if isinstance(value, bytes): enc = _get_argv_encoding() try: value = value.decode(enc) except UnicodeError: - fs_enc = get_filesystem_encoding() + fs_enc = sys.getfilesystemencoding() if fs_enc != enc: try: value = value.decode(fs_enc) @@ -128,9 +217,9 @@ class StringParamType(ParamType): else: value = value.decode("utf-8", "replace") return value - return value + return str(value) - def __repr__(self): + def __repr__(self) -> str: return "STRING" @@ -153,17 +242,32 @@ class Choice(ParamType): name = "choice" - def __init__(self, choices, case_sensitive=True): + def __init__(self, choices: t.Sequence[str], case_sensitive: bool = True) -> None: self.choices = choices self.case_sensitive = case_sensitive - def get_metavar(self, param): - return "[{}]".format("|".join(self.choices)) + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict["choices"] = self.choices + info_dict["case_sensitive"] = self.case_sensitive + return info_dict - def get_missing_message(self, param): - return "Choose from:\n\t{}.".format(",\n\t".join(self.choices)) + def get_metavar(self, param: "Parameter") -> str: + choices_str = "|".join(self.choices) - def convert(self, value, param, ctx): + # Use curly braces to indicate a required argument. + if param.required and param.param_type_name == "argument": + return f"{{{choices_str}}}" + + # Use square braces to indicate an option or optional argument. + return f"[{choices_str}]" + + def get_missing_message(self, param: "Parameter") -> str: + return _("Choose from:\n\t{choices}").format(choices=",\n\t".join(self.choices)) + + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: # Match through normalization and case sensitivity # first do token_normalize_func, then lowercase # preserve original `value` to produce an accurate message in @@ -179,30 +283,51 @@ class Choice(ParamType): } if not self.case_sensitive: - if PY2: - lower = str.lower - else: - lower = str.casefold - - normed_value = lower(normed_value) + normed_value = normed_value.casefold() normed_choices = { - lower(normed_choice): original + normed_choice.casefold(): original for normed_choice, original in normed_choices.items() } if normed_value in normed_choices: return normed_choices[normed_value] + choices_str = ", ".join(map(repr, self.choices)) self.fail( - "invalid choice: {}. (choose from {})".format( - value, ", ".join(self.choices) - ), + ngettext( + "{value!r} is not {choice}.", + "{value!r} is not one of {choices}.", + len(self.choices), + ).format(value=value, choice=choices_str, choices=choices_str), param, ctx, ) - def __repr__(self): - return "Choice('{}')".format(list(self.choices)) + def __repr__(self) -> str: + return f"Choice({list(self.choices)})" + + def shell_complete( + self, ctx: "Context", param: "Parameter", incomplete: str + ) -> t.List["CompletionItem"]: + """Complete choices that start with the incomplete value. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + str_choices = map(str, self.choices) + + if self.case_sensitive: + matched = (c for c in str_choices if c.startswith(incomplete)) + else: + incomplete = incomplete.lower() + matched = (c for c in str_choices if c.lower().startswith(incomplete)) + + return [CompletionItem(c) for c in matched] class DateTime(ParamType): @@ -228,212 +353,289 @@ class DateTime(ParamType): name = "datetime" - def __init__(self, formats=None): - self.formats = formats or ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"] + def __init__(self, formats: t.Optional[t.Sequence[str]] = None): + self.formats: t.Sequence[str] = formats or [ + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + ] - def get_metavar(self, param): - return "[{}]".format("|".join(self.formats)) + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict["formats"] = self.formats + return info_dict - def _try_to_convert_date(self, value, format): + def get_metavar(self, param: "Parameter") -> str: + return f"[{'|'.join(self.formats)}]" + + def _try_to_convert_date(self, value: t.Any, format: str) -> t.Optional[datetime]: try: return datetime.strptime(value, format) except ValueError: return None - def convert(self, value, param, ctx): - # Exact match - for format in self.formats: - dtime = self._try_to_convert_date(value, format) - if dtime: - return dtime + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: + if isinstance(value, datetime): + return value + for format in self.formats: + converted = self._try_to_convert_date(value, format) + + if converted is not None: + return converted + + formats_str = ", ".join(map(repr, self.formats)) self.fail( - "invalid datetime format: {}. (choose from {})".format( - value, ", ".join(self.formats) - ) + ngettext( + "{value!r} does not match the format {format}.", + "{value!r} does not match the formats {formats}.", + len(self.formats), + ).format(value=value, format=formats_str, formats=formats_str), + param, + ctx, ) - def __repr__(self): + def __repr__(self) -> str: return "DateTime" -class IntParamType(ParamType): - name = "integer" +class _NumberParamTypeBase(ParamType): + _number_class: t.ClassVar[t.Type[t.Any]] - def convert(self, value, param, ctx): + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: try: - return int(value) + return self._number_class(value) except ValueError: - self.fail("{} is not a valid integer".format(value), param, ctx) + self.fail( + _("{value!r} is not a valid {number_type}.").format( + value=value, number_type=self.name + ), + param, + ctx, + ) - def __repr__(self): + +class _NumberRangeBase(_NumberParamTypeBase): + def __init__( + self, + min: t.Optional[float] = None, + max: t.Optional[float] = None, + min_open: bool = False, + max_open: bool = False, + clamp: bool = False, + ) -> None: + self.min = min + self.max = max + self.min_open = min_open + self.max_open = max_open + self.clamp = clamp + + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict.update( + min=self.min, + max=self.max, + min_open=self.min_open, + max_open=self.max_open, + clamp=self.clamp, + ) + return info_dict + + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: + import operator + + rv = super().convert(value, param, ctx) + lt_min: bool = self.min is not None and ( + operator.le if self.min_open else operator.lt + )(rv, self.min) + gt_max: bool = self.max is not None and ( + operator.ge if self.max_open else operator.gt + )(rv, self.max) + + if self.clamp: + if lt_min: + return self._clamp(self.min, 1, self.min_open) # type: ignore + + if gt_max: + return self._clamp(self.max, -1, self.max_open) # type: ignore + + if lt_min or gt_max: + self.fail( + _("{value} is not in the range {range}.").format( + value=rv, range=self._describe_range() + ), + param, + ctx, + ) + + return rv + + def _clamp(self, bound: float, dir: "te.Literal[1, -1]", open: bool) -> float: + """Find the valid value to clamp to bound in the given + direction. + + :param bound: The boundary value. + :param dir: 1 or -1 indicating the direction to move. + :param open: If true, the range does not include the bound. + """ + raise NotImplementedError + + def _describe_range(self) -> str: + """Describe the range for use in help text.""" + if self.min is None: + op = "<" if self.max_open else "<=" + return f"x{op}{self.max}" + + if self.max is None: + op = ">" if self.min_open else ">=" + return f"x{op}{self.min}" + + lop = "<" if self.min_open else "<=" + rop = "<" if self.max_open else "<=" + return f"{self.min}{lop}x{rop}{self.max}" + + def __repr__(self) -> str: + clamp = " clamped" if self.clamp else "" + return f"<{type(self).__name__} {self._describe_range()}{clamp}>" + + +class IntParamType(_NumberParamTypeBase): + name = "integer" + _number_class = int + + def __repr__(self) -> str: return "INT" -class IntRange(IntParamType): - """A parameter that works similar to :data:`click.INT` but restricts - the value to fit into a range. The default behavior is to fail if the - value falls outside the range, but it can also be silently clamped - between the two edges. +class IntRange(_NumberRangeBase, IntParamType): + """Restrict an :data:`click.INT` value to a range of accepted + values. See :ref:`ranges`. - See :ref:`ranges` for an example. + If ``min`` or ``max`` are not passed, any value is accepted in that + direction. If ``min_open`` or ``max_open`` are enabled, the + corresponding boundary is not included in the range. + + If ``clamp`` is enabled, a value outside the range is clamped to the + boundary instead of failing. + + .. versionchanged:: 8.0 + Added the ``min_open`` and ``max_open`` parameters. """ name = "integer range" - def __init__(self, min=None, max=None, clamp=False): - self.min = min - self.max = max - self.clamp = clamp + def _clamp( # type: ignore + self, bound: int, dir: "te.Literal[1, -1]", open: bool + ) -> int: + if not open: + return bound - def convert(self, value, param, ctx): - rv = IntParamType.convert(self, value, param, ctx) - if self.clamp: - if self.min is not None and rv < self.min: - return self.min - if self.max is not None and rv > self.max: - return self.max - if ( - self.min is not None - and rv < self.min - or self.max is not None - and rv > self.max - ): - if self.min is None: - self.fail( - "{} is bigger than the maximum valid value {}.".format( - rv, self.max - ), - param, - ctx, - ) - elif self.max is None: - self.fail( - "{} is smaller than the minimum valid value {}.".format( - rv, self.min - ), - param, - ctx, - ) - else: - self.fail( - "{} is not in the valid range of {} to {}.".format( - rv, self.min, self.max - ), - param, - ctx, - ) - return rv - - def __repr__(self): - return "IntRange({}, {})".format(self.min, self.max) + return bound + dir -class FloatParamType(ParamType): +class FloatParamType(_NumberParamTypeBase): name = "float" + _number_class = float - def convert(self, value, param, ctx): - try: - return float(value) - except ValueError: - self.fail( - "{} is not a valid floating point value".format(value), param, ctx - ) - - def __repr__(self): + def __repr__(self) -> str: return "FLOAT" -class FloatRange(FloatParamType): - """A parameter that works similar to :data:`click.FLOAT` but restricts - the value to fit into a range. The default behavior is to fail if the - value falls outside the range, but it can also be silently clamped - between the two edges. +class FloatRange(_NumberRangeBase, FloatParamType): + """Restrict a :data:`click.FLOAT` value to a range of accepted + values. See :ref:`ranges`. - See :ref:`ranges` for an example. + If ``min`` or ``max`` are not passed, any value is accepted in that + direction. If ``min_open`` or ``max_open`` are enabled, the + corresponding boundary is not included in the range. + + If ``clamp`` is enabled, a value outside the range is clamped to the + boundary instead of failing. This is not supported if either + boundary is marked ``open``. + + .. versionchanged:: 8.0 + Added the ``min_open`` and ``max_open`` parameters. """ name = "float range" - def __init__(self, min=None, max=None, clamp=False): - self.min = min - self.max = max - self.clamp = clamp + def __init__( + self, + min: t.Optional[float] = None, + max: t.Optional[float] = None, + min_open: bool = False, + max_open: bool = False, + clamp: bool = False, + ) -> None: + super().__init__( + min=min, max=max, min_open=min_open, max_open=max_open, clamp=clamp + ) - def convert(self, value, param, ctx): - rv = FloatParamType.convert(self, value, param, ctx) - if self.clamp: - if self.min is not None and rv < self.min: - return self.min - if self.max is not None and rv > self.max: - return self.max - if ( - self.min is not None - and rv < self.min - or self.max is not None - and rv > self.max - ): - if self.min is None: - self.fail( - "{} is bigger than the maximum valid value {}.".format( - rv, self.max - ), - param, - ctx, - ) - elif self.max is None: - self.fail( - "{} is smaller than the minimum valid value {}.".format( - rv, self.min - ), - param, - ctx, - ) - else: - self.fail( - "{} is not in the valid range of {} to {}.".format( - rv, self.min, self.max - ), - param, - ctx, - ) - return rv + if (min_open or max_open) and clamp: + raise TypeError("Clamping is not supported for open bounds.") - def __repr__(self): - return "FloatRange({}, {})".format(self.min, self.max) + def _clamp(self, bound: float, dir: "te.Literal[1, -1]", open: bool) -> float: + if not open: + return bound + + # Could use Python 3.9's math.nextafter here, but clamping an + # open float range doesn't seem to be particularly useful. It's + # left up to the user to write a callback to do it if needed. + raise RuntimeError("Clamping is not supported for open bounds.") class BoolParamType(ParamType): name = "boolean" - def convert(self, value, param, ctx): - if isinstance(value, bool): + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: + if value in {False, True}: return bool(value) - value = value.lower() - if value in ("true", "t", "1", "yes", "y"): - return True - elif value in ("false", "f", "0", "no", "n"): - return False - self.fail("{} is not a valid boolean".format(value), param, ctx) - def __repr__(self): + norm = value.strip().lower() + + if norm in {"1", "true", "t", "yes", "y", "on"}: + return True + + if norm in {"0", "false", "f", "no", "n", "off"}: + return False + + self.fail( + _("{value!r} is not a valid boolean.").format(value=value), param, ctx + ) + + def __repr__(self) -> str: return "BOOL" class UUIDParameterType(ParamType): name = "uuid" - def convert(self, value, param, ctx): + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: import uuid + if isinstance(value, uuid.UUID): + return value + + value = value.strip() + try: - if PY2 and isinstance(value, text_type): - value = value.encode("ascii") return uuid.UUID(value) except ValueError: - self.fail("{} is not a valid UUID value".format(value), param, ctx) + self.fail( + _("{value!r} is not a valid UUID.").format(value=value), param, ctx + ) - def __repr__(self): + def __repr__(self) -> str: return "UUID" @@ -465,44 +667,64 @@ class File(ParamType): """ name = "filename" - envvar_list_splitter = os.path.pathsep + envvar_list_splitter: t.ClassVar[str] = os.path.pathsep def __init__( - self, mode="r", encoding=None, errors="strict", lazy=None, atomic=False - ): + self, + mode: str = "r", + encoding: t.Optional[str] = None, + errors: t.Optional[str] = "strict", + lazy: t.Optional[bool] = None, + atomic: bool = False, + ) -> None: self.mode = mode self.encoding = encoding self.errors = errors self.lazy = lazy self.atomic = atomic - def resolve_lazy_flag(self, value): + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict.update(mode=self.mode, encoding=self.encoding) + return info_dict + + def resolve_lazy_flag(self, value: "t.Union[str, os.PathLike[str]]") -> bool: if self.lazy is not None: return self.lazy - if value == "-": + if os.fspath(value) == "-": return False elif "w" in self.mode: return True return False - def convert(self, value, param, ctx): - try: - if hasattr(value, "read") or hasattr(value, "write"): - return value + def convert( + self, + value: t.Union[str, "os.PathLike[str]", t.IO[t.Any]], + param: t.Optional["Parameter"], + ctx: t.Optional["Context"], + ) -> t.IO[t.Any]: + if _is_file_like(value): + return value + value = t.cast("t.Union[str, os.PathLike[str]]", value) + + try: lazy = self.resolve_lazy_flag(value) if lazy: - f = LazyFile( + lf = LazyFile( value, self.mode, self.encoding, self.errors, atomic=self.atomic ) + if ctx is not None: - ctx.call_on_close(f.close_intelligently) - return f + ctx.call_on_close(lf.close_intelligently) + + return t.cast(t.IO[t.Any], lf) f, should_close = open_stream( value, self.mode, self.encoding, self.errors, atomic=self.atomic ) + # If a context is provided, we automatically close the file # at the end of the context execution (or flush out). If a # context does not exist, it's the caller's responsibility to @@ -513,96 +735,138 @@ class File(ParamType): ctx.call_on_close(safecall(f.close)) else: ctx.call_on_close(safecall(f.flush)) + return f - except (IOError, OSError) as e: # noqa: B014 - self.fail( - "Could not open file: {}: {}".format( - filename_to_ui(value), get_streerror(e) - ), - param, - ctx, - ) + except OSError as e: # noqa: B014 + self.fail(f"'{format_filename(value)}': {e.strerror}", param, ctx) + + def shell_complete( + self, ctx: "Context", param: "Parameter", incomplete: str + ) -> t.List["CompletionItem"]: + """Return a special completion marker that tells the completion + system to use the shell to provide file path completions. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + return [CompletionItem(incomplete, type="file")] + + +def _is_file_like(value: t.Any) -> "te.TypeGuard[t.IO[t.Any]]": + return hasattr(value, "read") or hasattr(value, "write") class Path(ParamType): - """The path type is similar to the :class:`File` type but it performs - different checks. First of all, instead of returning an open file - handle it returns just the filename. Secondly, it can perform various - basic checks about what the file or directory should be. + """The ``Path`` type is similar to the :class:`File` type, but + returns the filename instead of an open file. Various checks can be + enabled to validate the type of file and permissions. + + :param exists: The file or directory needs to exist for the value to + be valid. If this is not set to ``True``, and the file does not + exist, then all further checks are silently skipped. + :param file_okay: Allow a file as a value. + :param dir_okay: Allow a directory as a value. + :param readable: if true, a readable check is performed. + :param writable: if true, a writable check is performed. + :param executable: if true, an executable check is performed. + :param resolve_path: Make the value absolute and resolve any + symlinks. A ``~`` is not expanded, as this is supposed to be + done by the shell only. + :param allow_dash: Allow a single dash as a value, which indicates + a standard stream (but does not open it). Use + :func:`~click.open_file` to handle opening this value. + :param path_type: Convert the incoming path value to this type. If + ``None``, keep Python's default, which is ``str``. Useful to + convert to :class:`pathlib.Path`. + + .. versionchanged:: 8.1 + Added the ``executable`` parameter. + + .. versionchanged:: 8.0 + Allow passing ``path_type=pathlib.Path``. .. versionchanged:: 6.0 - `allow_dash` was added. - - :param exists: if set to true, the file or directory needs to exist for - this value to be valid. If this is not required and a - file does indeed not exist, then all further checks are - silently skipped. - :param file_okay: controls if a file is a possible value. - :param dir_okay: controls if a directory is a possible value. - :param writable: if true, a writable check is performed. - :param readable: if true, a readable check is performed. - :param resolve_path: if this is true, then the path is fully resolved - before the value is passed onwards. This means - that it's absolute and symlinks are resolved. It - will not expand a tilde-prefix, as this is - supposed to be done by the shell only. - :param allow_dash: If this is set to `True`, a single dash to indicate - standard streams is permitted. - :param path_type: optionally a string type that should be used to - represent the path. The default is `None` which - means the return value will be either bytes or - unicode depending on what makes most sense given the - input data Click deals with. + Added the ``allow_dash`` parameter. """ - envvar_list_splitter = os.path.pathsep + envvar_list_splitter: t.ClassVar[str] = os.path.pathsep def __init__( self, - exists=False, - file_okay=True, - dir_okay=True, - writable=False, - readable=True, - resolve_path=False, - allow_dash=False, - path_type=None, + exists: bool = False, + file_okay: bool = True, + dir_okay: bool = True, + writable: bool = False, + readable: bool = True, + resolve_path: bool = False, + allow_dash: bool = False, + path_type: t.Optional[t.Type[t.Any]] = None, + executable: bool = False, ): self.exists = exists self.file_okay = file_okay self.dir_okay = dir_okay - self.writable = writable self.readable = readable + self.writable = writable + self.executable = executable self.resolve_path = resolve_path self.allow_dash = allow_dash self.type = path_type if self.file_okay and not self.dir_okay: - self.name = "file" - self.path_type = "File" + self.name: str = _("file") elif self.dir_okay and not self.file_okay: - self.name = "directory" - self.path_type = "Directory" + self.name = _("directory") else: - self.name = "path" - self.path_type = "Path" + self.name = _("path") - def coerce_path_result(self, rv): - if self.type is not None and not isinstance(rv, self.type): - if self.type is text_type: - rv = rv.decode(get_filesystem_encoding()) + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict.update( + exists=self.exists, + file_okay=self.file_okay, + dir_okay=self.dir_okay, + writable=self.writable, + readable=self.readable, + allow_dash=self.allow_dash, + ) + return info_dict + + def coerce_path_result( + self, value: "t.Union[str, os.PathLike[str]]" + ) -> "t.Union[str, bytes, os.PathLike[str]]": + if self.type is not None and not isinstance(value, self.type): + if self.type is str: + return os.fsdecode(value) + elif self.type is bytes: + return os.fsencode(value) else: - rv = rv.encode(get_filesystem_encoding()) - return rv + return t.cast("os.PathLike[str]", self.type(value)) - def convert(self, value, param, ctx): + return value + + def convert( + self, + value: "t.Union[str, os.PathLike[str]]", + param: t.Optional["Parameter"], + ctx: t.Optional["Context"], + ) -> "t.Union[str, bytes, os.PathLike[str]]": rv = value is_dash = self.file_okay and self.allow_dash and rv in (b"-", "-") if not is_dash: if self.resolve_path: - rv = os.path.realpath(rv) + # os.path.realpath doesn't resolve symlinks on Windows + # until Python 3.8. Use pathlib for now. + import pathlib + + rv = os.fsdecode(pathlib.Path(rv).resolve()) try: st = os.stat(rv) @@ -610,8 +874,8 @@ class Path(ParamType): if not self.exists: return self.coerce_path_result(rv) self.fail( - "{} '{}' does not exist.".format( - self.path_type, filename_to_ui(value) + _("{name} {filename!r} does not exist.").format( + name=self.name.title(), filename=format_filename(value) ), param, ctx, @@ -619,30 +883,43 @@ class Path(ParamType): if not self.file_okay and stat.S_ISREG(st.st_mode): self.fail( - "{} '{}' is a file.".format(self.path_type, filename_to_ui(value)), + _("{name} {filename!r} is a file.").format( + name=self.name.title(), filename=format_filename(value) + ), param, ctx, ) if not self.dir_okay and stat.S_ISDIR(st.st_mode): self.fail( - "{} '{}' is a directory.".format( - self.path_type, filename_to_ui(value) + _("{name} '{filename}' is a directory.").format( + name=self.name.title(), filename=format_filename(value) ), param, ctx, ) - if self.writable and not os.access(value, os.W_OK): + + if self.readable and not os.access(rv, os.R_OK): self.fail( - "{} '{}' is not writable.".format( - self.path_type, filename_to_ui(value) + _("{name} {filename!r} is not readable.").format( + name=self.name.title(), filename=format_filename(value) ), param, ctx, ) - if self.readable and not os.access(value, os.R_OK): + + if self.writable and not os.access(rv, os.W_OK): self.fail( - "{} '{}' is not readable.".format( - self.path_type, filename_to_ui(value) + _("{name} {filename!r} is not writable.").format( + name=self.name.title(), filename=format_filename(value) + ), + param, + ctx, + ) + + if self.executable and not os.access(value, os.X_OK): + self.fail( + _("{name} {filename!r} is not executable.").format( + name=self.name.title(), filename=format_filename(value) ), param, ctx, @@ -650,6 +927,24 @@ class Path(ParamType): return self.coerce_path_result(rv) + def shell_complete( + self, ctx: "Context", param: "Parameter", incomplete: str + ) -> t.List["CompletionItem"]: + """Return a special completion marker that tells the completion + system to use the shell to provide path completions for only + directories or any paths. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + type = "dir" if self.dir_okay and not self.file_okay else "file" + return [CompletionItem(incomplete, type=type)] + class Tuple(CompositeParamType): """The default behavior of Click is to apply a type on a value directly. @@ -665,75 +960,107 @@ class Tuple(CompositeParamType): :param types: a list of types that should be used for the tuple items. """ - def __init__(self, types): - self.types = [convert_type(ty) for ty in types] + def __init__(self, types: t.Sequence[t.Union[t.Type[t.Any], ParamType]]) -> None: + self.types: t.Sequence[ParamType] = [convert_type(ty) for ty in types] + + def to_info_dict(self) -> t.Dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict["types"] = [t.to_info_dict() for t in self.types] + return info_dict @property - def name(self): - return "<{}>".format(" ".join(ty.name for ty in self.types)) + def name(self) -> str: # type: ignore + return f"<{' '.join(ty.name for ty in self.types)}>" @property - def arity(self): + def arity(self) -> int: # type: ignore return len(self.types) - def convert(self, value, param, ctx): - if len(value) != len(self.types): - raise TypeError( - "It would appear that nargs is set to conflict with the" - " composite type arity." + def convert( + self, value: t.Any, param: t.Optional["Parameter"], ctx: t.Optional["Context"] + ) -> t.Any: + len_type = len(self.types) + len_value = len(value) + + if len_value != len_type: + self.fail( + ngettext( + "{len_type} values are required, but {len_value} was given.", + "{len_type} values are required, but {len_value} were given.", + len_value, + ).format(len_type=len_type, len_value=len_value), + param=param, + ctx=ctx, ) + return tuple(ty(x, param, ctx) for ty, x in zip(self.types, value)) -def convert_type(ty, default=None): - """Converts a callable or python type into the most appropriate - param type. +def convert_type(ty: t.Optional[t.Any], default: t.Optional[t.Any] = None) -> ParamType: + """Find the most appropriate :class:`ParamType` for the given Python + type. If the type isn't provided, it can be inferred from a default + value. """ guessed_type = False + if ty is None and default is not None: - if isinstance(default, tuple): - ty = tuple(map(type, default)) + if isinstance(default, (tuple, list)): + # If the default is empty, ty will remain None and will + # return STRING. + if default: + item = default[0] + + # A tuple of tuples needs to detect the inner types. + # Can't call convert recursively because that would + # incorrectly unwind the tuple to a single type. + if isinstance(item, (tuple, list)): + ty = tuple(map(type, item)) + else: + ty = type(item) else: ty = type(default) + guessed_type = True if isinstance(ty, tuple): return Tuple(ty) + if isinstance(ty, ParamType): return ty - if ty is text_type or ty is str or ty is None: + + if ty is str or ty is None: return STRING + if ty is int: return INT - # Booleans are only okay if not guessed. This is done because for - # flags the default value is actually a bit of a lie in that it - # indicates which of the flags is the one we want. See get_default() - # for more information. - if ty is bool and not guessed_type: - return BOOL + if ty is float: return FLOAT + + if ty is bool: + return BOOL + if guessed_type: return STRING - # Catch a common mistake if __debug__: try: if issubclass(ty, ParamType): raise AssertionError( - "Attempted to use an uninstantiated parameter type ({}).".format(ty) + f"Attempted to use an uninstantiated parameter type ({ty})." ) except TypeError: + # ty is an instance (correct), so issubclass fails. pass + return FuncParamType(ty) #: A dummy parameter type that just does nothing. From a user's -#: perspective this appears to just be the same as `STRING` but internally -#: no string conversion takes place. This is necessary to achieve the -#: same bytes/unicode behavior on Python 2/3 in situations where you want -#: to not convert argument types. This is usually useful when working -#: with file paths as they can appear in bytes and unicode. +#: perspective this appears to just be the same as `STRING` but +#: internally no string conversion takes place if the input was bytes. +#: This is usually useful when working with file paths as they can +#: appear in bytes and unicode. #: #: For path related uses the :class:`Path` type is a better choice but #: there are situations where an unprocessed type is useful which is why diff --git a/third_party/python/click/click/utils.py b/third_party/python/click/click/utils.py index 79265e732d49..d536434f0bd0 100644 --- a/third_party/python/click/click/utils.py +++ b/third_party/python/click/click/utils.py @@ -1,86 +1,109 @@ import os +import re import sys +import typing as t +from functools import update_wrapper +from types import ModuleType +from types import TracebackType from ._compat import _default_text_stderr from ._compat import _default_text_stdout +from ._compat import _find_binary_writer from ._compat import auto_wrap_for_ansi from ._compat import binary_streams -from ._compat import filename_to_ui -from ._compat import get_filesystem_encoding -from ._compat import get_streerror -from ._compat import is_bytes from ._compat import open_stream -from ._compat import PY2 from ._compat import should_strip_ansi -from ._compat import string_types from ._compat import strip_ansi from ._compat import text_streams -from ._compat import text_type from ._compat import WIN from .globals import resolve_color_default -if not PY2: - from ._compat import _find_binary_writer -elif WIN: - from ._winconsole import _get_windows_argv - from ._winconsole import _hash_py_argv - from ._winconsole import _initial_argv_hash +if t.TYPE_CHECKING: + import typing_extensions as te -echo_native_types = string_types + (bytes, bytearray) + P = te.ParamSpec("P") + +R = t.TypeVar("R") -def _posixify(name): +def _posixify(name: str) -> str: return "-".join(name.split()).lower() -def safecall(func): +def safecall(func: "t.Callable[P, R]") -> "t.Callable[P, t.Optional[R]]": """Wraps a function so that it swallows exceptions.""" - def wrapper(*args, **kwargs): + def wrapper(*args: "P.args", **kwargs: "P.kwargs") -> t.Optional[R]: try: return func(*args, **kwargs) except Exception: pass + return None - return wrapper + return update_wrapper(wrapper, func) -def make_str(value): +def make_str(value: t.Any) -> str: """Converts a value into a valid string.""" if isinstance(value, bytes): try: - return value.decode(get_filesystem_encoding()) + return value.decode(sys.getfilesystemencoding()) except UnicodeError: return value.decode("utf-8", "replace") - return text_type(value) + return str(value) -def make_default_short_help(help, max_length=45): - """Return a condensed version of help string.""" +def make_default_short_help(help: str, max_length: int = 45) -> str: + """Returns a condensed version of help string.""" + # Consider only the first paragraph. + paragraph_end = help.find("\n\n") + + if paragraph_end != -1: + help = help[:paragraph_end] + + # Collapse newlines, tabs, and spaces. words = help.split() + + if not words: + return "" + + # The first paragraph started with a "no rewrap" marker, ignore it. + if words[0] == "\b": + words = words[1:] + total_length = 0 - result = [] - done = False + last_index = len(words) - 1 - for word in words: - if word[-1:] == ".": - done = True - new_length = 1 + len(word) if result else len(word) - if total_length + new_length > max_length: - result.append("...") - done = True - else: - if result: - result.append(" ") - result.append(word) - if done: + for i, word in enumerate(words): + total_length += len(word) + (i > 0) + + if total_length > max_length: # too long, truncate break - total_length += new_length - return "".join(result) + if word[-1] == ".": # sentence end, truncate without "..." + return " ".join(words[: i + 1]) + + if total_length == max_length and i != last_index: + break # not at sentence end, truncate with "..." + else: + return " ".join(words) # no truncation needed + + # Account for the length of the suffix. + total_length += len("...") + + # remove words until the length is short enough + while i > 0: + total_length -= len(words[i]) + (i > 0) + + if total_length <= max_length: + break + + i -= 1 + + return " ".join(words[:i]) + "..." -class LazyFile(object): +class LazyFile: """A lazy file works like a regular file but it does not fully open the file but it does perform some basic checks early to see if the filename parameter does make sense. This is useful for safely opening @@ -88,15 +111,22 @@ class LazyFile(object): """ def __init__( - self, filename, mode="r", encoding=None, errors="strict", atomic=False + self, + filename: t.Union[str, "os.PathLike[str]"], + mode: str = "r", + encoding: t.Optional[str] = None, + errors: t.Optional[str] = "strict", + atomic: bool = False, ): - self.name = filename + self.name: str = os.fspath(filename) self.mode = mode self.encoding = encoding self.errors = errors self.atomic = atomic + self._f: t.Optional[t.IO[t.Any]] + self.should_close: bool - if filename == "-": + if self.name == "-": self._f, self.should_close = open_stream(filename, mode, encoding, errors) else: if "r" in mode: @@ -107,15 +137,15 @@ class LazyFile(object): self._f = None self.should_close = True - def __getattr__(self, name): + def __getattr__(self, name: str) -> t.Any: return getattr(self.open(), name) - def __repr__(self): + def __repr__(self) -> str: if self._f is not None: return repr(self._f) - return "".format(self.name, self.mode) + return f"" - def open(self): + def open(self) -> t.IO[t.Any]: """Opens the file if it's not yet open. This call might fail with a :exc:`FileError`. Not handling this error will produce an error that Click shows. @@ -126,102 +156,110 @@ class LazyFile(object): rv, self.should_close = open_stream( self.name, self.mode, self.encoding, self.errors, atomic=self.atomic ) - except (IOError, OSError) as e: # noqa: E402 + except OSError as e: # noqa: E402 from .exceptions import FileError - raise FileError(self.name, hint=get_streerror(e)) + raise FileError(self.name, hint=e.strerror) from e self._f = rv return rv - def close(self): + def close(self) -> None: """Closes the underlying file, no matter what.""" if self._f is not None: self._f.close() - def close_intelligently(self): + def close_intelligently(self) -> None: """This function only closes the file if it was opened by the lazy file wrapper. For instance this will never close stdin. """ if self.should_close: self.close() - def __enter__(self): + def __enter__(self) -> "LazyFile": return self - def __exit__(self, exc_type, exc_value, tb): + def __exit__( + self, + exc_type: t.Optional[t.Type[BaseException]], + exc_value: t.Optional[BaseException], + tb: t.Optional[TracebackType], + ) -> None: self.close_intelligently() - def __iter__(self): + def __iter__(self) -> t.Iterator[t.AnyStr]: self.open() - return iter(self._f) + return iter(self._f) # type: ignore -class KeepOpenFile(object): - def __init__(self, file): - self._file = file +class KeepOpenFile: + def __init__(self, file: t.IO[t.Any]) -> None: + self._file: t.IO[t.Any] = file - def __getattr__(self, name): + def __getattr__(self, name: str) -> t.Any: return getattr(self._file, name) - def __enter__(self): + def __enter__(self) -> "KeepOpenFile": return self - def __exit__(self, exc_type, exc_value, tb): + def __exit__( + self, + exc_type: t.Optional[t.Type[BaseException]], + exc_value: t.Optional[BaseException], + tb: t.Optional[TracebackType], + ) -> None: pass - def __repr__(self): + def __repr__(self) -> str: return repr(self._file) - def __iter__(self): + def __iter__(self) -> t.Iterator[t.AnyStr]: return iter(self._file) -def echo(message=None, file=None, nl=True, err=False, color=None): - """Prints a message plus a newline to the given file or stdout. On - first sight, this looks like the print function, but it has improved - support for handling Unicode and binary data that does not fail no - matter how badly configured the system is. +def echo( + message: t.Optional[t.Any] = None, + file: t.Optional[t.IO[t.Any]] = None, + nl: bool = True, + err: bool = False, + color: t.Optional[bool] = None, +) -> None: + """Print a message and newline to stdout or a file. This should be + used instead of :func:`print` because it provides better support + for different data, files, and environments. - Primarily it means that you can print binary data as well as Unicode - data on both 2.x and 3.x to the given file in the most appropriate way - possible. This is a very carefree function in that it will try its - best to not fail. As of Click 6.0 this includes support for unicode - output on the Windows console. + Compared to :func:`print`, this does the following: - In addition to that, if `colorama`_ is installed, the echo function will - also support clever handling of ANSI codes. Essentially it will then - do the following: + - Ensures that the output encoding is not misconfigured on Linux. + - Supports Unicode in the Windows console. + - Supports writing to binary outputs, and supports writing bytes + to text outputs. + - Supports colors and styles on Windows. + - Removes ANSI color and style codes if the output does not look + like an interactive terminal. + - Always flushes the output. - - add transparent handling of ANSI color codes on Windows. - - hide ANSI codes automatically if the destination file is not a - terminal. - - .. _colorama: https://pypi.org/project/colorama/ + :param message: The string or bytes to output. Other objects are + converted to strings. + :param file: The file to write to. Defaults to ``stdout``. + :param err: Write to ``stderr`` instead of ``stdout``. + :param nl: Print a newline after the message. Enabled by default. + :param color: Force showing or hiding colors and other styles. By + default Click will remove color if the output does not look like + an interactive terminal. .. versionchanged:: 6.0 - As of Click 6.0 the echo function will properly support unicode - output on the windows console. Not that click does not modify - the interpreter in any way which means that `sys.stdout` or the - print statement or function will still not provide unicode support. - - .. versionchanged:: 2.0 - Starting with version 2.0 of Click, the echo function will work - with colorama if it's installed. - - .. versionadded:: 3.0 - The `err` parameter was added. + Support Unicode output on the Windows console. Click does not + modify ``sys.stdout``, so ``sys.stdout.write()`` and ``print()`` + will still not support Unicode. .. versionchanged:: 4.0 - Added the `color` flag. + Added the ``color`` parameter. - :param message: the message to print - :param file: the file to write to (defaults to ``stdout``) - :param err: if set to true the file defaults to ``stderr`` instead of - ``stdout``. This is faster and easier than calling - :func:`get_text_stderr` yourself. - :param nl: if set to `True` (the default) a newline is printed afterwards. - :param color: controls if the terminal supports ANSI colors or not. The - default is autodetection. + .. versionadded:: 3.0 + Added the ``err`` parameter. + + .. versionchanged:: 2.0 + Support colors on Windows if colorama is installed. """ if file is None: if err: @@ -229,71 +267,79 @@ def echo(message=None, file=None, nl=True, err=False, color=None): else: file = _default_text_stdout() + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if file is None: + return + # Convert non bytes/text into the native string type. - if message is not None and not isinstance(message, echo_native_types): - message = text_type(message) + if message is not None and not isinstance(message, (str, bytes, bytearray)): + out: t.Optional[t.Union[str, bytes]] = str(message) + else: + out = message if nl: - message = message or u"" - if isinstance(message, text_type): - message += u"\n" + out = out or "" + if isinstance(out, str): + out += "\n" else: - message += b"\n" + out += b"\n" - # If there is a message, and we're in Python 3, and the value looks - # like bytes, we manually need to find the binary stream and write the - # message in there. This is done separately so that most stream - # types will work as you would expect. Eg: you can write to StringIO - # for other cases. - if message and not PY2 and is_bytes(message): + if not out: + file.flush() + return + + # If there is a message and the value looks like bytes, we manually + # need to find the binary stream and write the message in there. + # This is done separately so that most stream types will work as you + # would expect. Eg: you can write to StringIO for other cases. + if isinstance(out, (bytes, bytearray)): binary_file = _find_binary_writer(file) + if binary_file is not None: file.flush() - binary_file.write(message) + binary_file.write(out) binary_file.flush() return - # ANSI-style support. If there is no message or we are dealing with - # bytes nothing is happening. If we are connected to a file we want - # to strip colors. If we are on windows we either wrap the stream - # to strip the color or we use the colorama support to translate the - # ansi codes to API calls. - if message and not is_bytes(message): + # ANSI style code support. For no message or bytes, nothing happens. + # When outputting to a file instead of a terminal, strip codes. + else: color = resolve_color_default(color) + if should_strip_ansi(file, color): - message = strip_ansi(message) + out = strip_ansi(out) elif WIN: if auto_wrap_for_ansi is not None: - file = auto_wrap_for_ansi(file) + file = auto_wrap_for_ansi(file) # type: ignore elif not color: - message = strip_ansi(message) + out = strip_ansi(out) - if message: - file.write(message) + file.write(out) # type: ignore file.flush() -def get_binary_stream(name): - """Returns a system stream for byte processing. This essentially - returns the stream from the sys module with the given name but it - solves some compatibility issues between different Python versions. - Primarily this function is necessary for getting binary streams on - Python 3. +def get_binary_stream(name: "te.Literal['stdin', 'stdout', 'stderr']") -> t.BinaryIO: + """Returns a system stream for byte processing. :param name: the name of the stream to open. Valid names are ``'stdin'``, ``'stdout'`` and ``'stderr'`` """ opener = binary_streams.get(name) if opener is None: - raise TypeError("Unknown standard stream '{}'".format(name)) + raise TypeError(f"Unknown standard stream '{name}'") return opener() -def get_text_stream(name, encoding=None, errors="strict"): +def get_text_stream( + name: "te.Literal['stdin', 'stdout', 'stderr']", + encoding: t.Optional[str] = None, + errors: t.Optional[str] = "strict", +) -> t.TextIO: """Returns a system stream for text processing. This usually returns a wrapped stream around a binary stream returned from - :func:`get_binary_stream` but it also can take shortcuts on Python 3 - for already correctly configured streams. + :func:`get_binary_stream` but it also can take shortcuts for already + correctly configured streams. :param name: the name of the stream to open. Valid names are ``'stdin'``, ``'stdout'`` and ``'stderr'`` @@ -302,70 +348,80 @@ def get_text_stream(name, encoding=None, errors="strict"): """ opener = text_streams.get(name) if opener is None: - raise TypeError("Unknown standard stream '{}'".format(name)) + raise TypeError(f"Unknown standard stream '{name}'") return opener(encoding, errors) def open_file( - filename, mode="r", encoding=None, errors="strict", lazy=False, atomic=False -): - """This is similar to how the :class:`File` works but for manual - usage. Files are opened non lazy by default. This can open regular - files as well as stdin/stdout if ``'-'`` is passed. + filename: str, + mode: str = "r", + encoding: t.Optional[str] = None, + errors: t.Optional[str] = "strict", + lazy: bool = False, + atomic: bool = False, +) -> t.IO[t.Any]: + """Open a file, with extra behavior to handle ``'-'`` to indicate + a standard stream, lazy open on write, and atomic write. Similar to + the behavior of the :class:`~click.File` param type. - If stdin/stdout is returned the stream is wrapped so that the context - manager will not close the stream accidentally. This makes it possible - to always use the function like this without having to worry to - accidentally close a standard stream:: + If ``'-'`` is given to open ``stdout`` or ``stdin``, the stream is + wrapped so that using it in a context manager will not close it. + This makes it possible to use the function without accidentally + closing a standard stream: + + .. code-block:: python with open_file(filename) as f: ... - .. versionadded:: 3.0 + :param filename: The name of the file to open, or ``'-'`` for + ``stdin``/``stdout``. + :param mode: The mode in which to open the file. + :param encoding: The encoding to decode or encode a file opened in + text mode. + :param errors: The error handling mode. + :param lazy: Wait to open the file until it is accessed. For read + mode, the file is temporarily opened to raise access errors + early, then closed until it is read again. + :param atomic: Write to a temporary file and replace the given file + on close. - :param filename: the name of the file to open (or ``'-'`` for stdin/stdout). - :param mode: the mode in which to open the file. - :param encoding: the encoding to use. - :param errors: the error handling for this file. - :param lazy: can be flipped to true to open the file lazily. - :param atomic: in atomic mode writes go into a temporary file and it's - moved on close. + .. versionadded:: 3.0 """ if lazy: - return LazyFile(filename, mode, encoding, errors, atomic=atomic) + return t.cast( + t.IO[t.Any], LazyFile(filename, mode, encoding, errors, atomic=atomic) + ) + f, should_close = open_stream(filename, mode, encoding, errors, atomic=atomic) + if not should_close: - f = KeepOpenFile(f) + f = t.cast(t.IO[t.Any], KeepOpenFile(f)) + return f -def get_os_args(): - """This returns the argument part of sys.argv in the most appropriate - form for processing. What this means is that this return value is in - a format that works for Click to process but does not necessarily - correspond well to what's actually standard for the interpreter. +def format_filename( + filename: "t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]]", + shorten: bool = False, +) -> str: + """Format a filename as a string for display. Ensures the filename can be + displayed by replacing any invalid bytes or surrogate escapes in the name + with the replacement character ``�``. - On most environments the return value is ``sys.argv[:1]`` unchanged. - However if you are on Windows and running Python 2 the return value - will actually be a list of unicode strings instead because the - default behavior on that platform otherwise will not be able to - carry all possible values that sys.argv can have. + Invalid bytes or surrogate escapes will raise an error when written to a + stream with ``errors="strict". This will typically happen with ``stdout`` + when the locale is something like ``en_GB.UTF-8``. - .. versionadded:: 6.0 - """ - # We can only extract the unicode argv if sys.argv has not been - # changed since the startup of the application. - if PY2 and WIN and _initial_argv_hash == _hash_py_argv(): - return _get_windows_argv() - return sys.argv[1:] + Many scenarios *are* safe to write surrogates though, due to PEP 538 and + PEP 540, including: - -def format_filename(filename, shorten=False): - """Formats a filename for user display. The main purpose of this - function is to ensure that the filename can be displayed at all. This - will decode the filename to unicode if necessary in a way that it will - not fail. Optionally, it can shorten the filename to not include the - full path to the filename. + - Writing to ``stderr``, which uses ``errors="backslashreplace"``. + - The system has ``LANG=C.UTF-8``, ``C``, or ``POSIX``. Python opens + stdout and stderr with ``errors="surrogateescape"``. + - None of ``LANG/LC_*`` are set. Python assumes ``LANG=C.UTF-8``. + - Python is started in UTF-8 mode with ``PYTHONUTF8=1`` or ``-X utf8``. + Python opens stdout and stderr with ``errors="surrogateescape"``. :param filename: formats a filename for UI display. This will also convert the filename into unicode without failing. @@ -374,10 +430,20 @@ def format_filename(filename, shorten=False): """ if shorten: filename = os.path.basename(filename) - return filename_to_ui(filename) + else: + filename = os.fspath(filename) + + if isinstance(filename, bytes): + filename = filename.decode(sys.getfilesystemencoding(), "replace") + else: + filename = filename.encode("utf-8", "surrogateescape").decode( + "utf-8", "replace" + ) + + return filename -def get_app_dir(app_name, roaming=True, force_posix=False): +def get_app_dir(app_name: str, roaming: bool = True, force_posix: bool = False) -> str: r"""Returns the config folder for the application. The default behavior is to return whatever is most appropriate for the operating system. @@ -392,13 +458,9 @@ def get_app_dir(app_name, roaming=True, force_posix=False): ``~/.config/foo-bar`` Unix (POSIX): ``~/.foo-bar`` - Win XP (roaming): - ``C:\Documents and Settings\\Local Settings\Application Data\Foo Bar`` - Win XP (not roaming): - ``C:\Documents and Settings\\Application Data\Foo Bar`` - Win 7 (roaming): + Windows (roaming): ``C:\Users\\AppData\Roaming\Foo Bar`` - Win 7 (not roaming): + Windows (not roaming): ``C:\Users\\AppData\Local\Foo Bar`` .. versionadded:: 2.0 @@ -406,7 +468,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): :param app_name: the application name. This should be properly capitalized and can contain whitespace. :param roaming: controls if the folder should be roaming or not on Windows. - Has no affect otherwise. + Has no effect otherwise. :param force_posix: if this is set to `True` then on any POSIX system the folder will be stored in the home folder with a leading dot instead of the XDG config home or darwin's @@ -419,7 +481,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): folder = os.path.expanduser("~") return os.path.join(folder, app_name) if force_posix: - return os.path.join(os.path.expanduser("~/.{}".format(_posixify(app_name)))) + return os.path.join(os.path.expanduser(f"~/.{_posixify(app_name)}")) if sys.platform == "darwin": return os.path.join( os.path.expanduser("~/Library/Application Support"), app_name @@ -430,7 +492,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): ) -class PacifyFlushWrapper(object): +class PacifyFlushWrapper: """This wrapper is used to catch and suppress BrokenPipeErrors resulting from ``.flush()`` being called on broken pipe during the shutdown/final-GC of the Python interpreter. Notably ``.flush()`` is always called on @@ -439,17 +501,124 @@ class PacifyFlushWrapper(object): pipe, all calls and attributes are proxied. """ - def __init__(self, wrapped): + def __init__(self, wrapped: t.IO[t.Any]) -> None: self.wrapped = wrapped - def flush(self): + def flush(self) -> None: try: self.wrapped.flush() - except IOError as e: + except OSError as e: import errno if e.errno != errno.EPIPE: raise - def __getattr__(self, attr): + def __getattr__(self, attr: str) -> t.Any: return getattr(self.wrapped, attr) + + +def _detect_program_name( + path: t.Optional[str] = None, _main: t.Optional[ModuleType] = None +) -> str: + """Determine the command used to run the program, for use in help + text. If a file or entry point was executed, the file name is + returned. If ``python -m`` was used to execute a module or package, + ``python -m name`` is returned. + + This doesn't try to be too precise, the goal is to give a concise + name for help text. Files are only shown as their name without the + path. ``python`` is only shown for modules, and the full path to + ``sys.executable`` is not shown. + + :param path: The Python file being executed. Python puts this in + ``sys.argv[0]``, which is used by default. + :param _main: The ``__main__`` module. This should only be passed + during internal testing. + + .. versionadded:: 8.0 + Based on command args detection in the Werkzeug reloader. + + :meta private: + """ + if _main is None: + _main = sys.modules["__main__"] + + if not path: + path = sys.argv[0] + + # The value of __package__ indicates how Python was called. It may + # not exist if a setuptools script is installed as an egg. It may be + # set incorrectly for entry points created with pip on Windows. + # It is set to "" inside a Shiv or PEX zipapp. + if getattr(_main, "__package__", None) in {None, ""} or ( + os.name == "nt" + and _main.__package__ == "" + and not os.path.exists(path) + and os.path.exists(f"{path}.exe") + ): + # Executed a file, like "python app.py". + return os.path.basename(path) + + # Executed a module, like "python -m example". + # Rewritten by Python from "-m script" to "/path/to/script.py". + # Need to look at main module to determine how it was executed. + py_module = t.cast(str, _main.__package__) + name = os.path.splitext(os.path.basename(path))[0] + + # A submodule like "example.cli". + if name != "__main__": + py_module = f"{py_module}.{name}" + + return f"python -m {py_module.lstrip('.')}" + + +def _expand_args( + args: t.Iterable[str], + *, + user: bool = True, + env: bool = True, + glob_recursive: bool = True, +) -> t.List[str]: + """Simulate Unix shell expansion with Python functions. + + See :func:`glob.glob`, :func:`os.path.expanduser`, and + :func:`os.path.expandvars`. + + This is intended for use on Windows, where the shell does not do any + expansion. It may not exactly match what a Unix shell would do. + + :param args: List of command line arguments to expand. + :param user: Expand user home directory. + :param env: Expand environment variables. + :param glob_recursive: ``**`` matches directories recursively. + + .. versionchanged:: 8.1 + Invalid glob patterns are treated as empty expansions rather + than raising an error. + + .. versionadded:: 8.0 + + :meta private: + """ + from glob import glob + + out = [] + + for arg in args: + if user: + arg = os.path.expanduser(arg) + + if env: + arg = os.path.expandvars(arg) + + try: + matches = glob(arg, recursive=glob_recursive) + except re.error: + matches = [] + + if not matches: + out.append(arg) + else: + out.extend(matches) + + return out diff --git a/third_party/python/packaging/packaging-21.3.dist-info/METADATA b/third_party/python/packaging/packaging-21.3.dist-info/METADATA deleted file mode 100644 index 358ace53624f..000000000000 --- a/third_party/python/packaging/packaging-21.3.dist-info/METADATA +++ /dev/null @@ -1,453 +0,0 @@ -Metadata-Version: 2.1 -Name: packaging -Version: 21.3 -Summary: Core utilities for Python packages -Home-page: https://github.com/pypa/packaging -Author: Donald Stufft and individual contributors -Author-email: donald@stufft.io -License: BSD-2-Clause or Apache-2.0 -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: Apache Software License -Classifier: License :: OSI Approved :: BSD License -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Requires-Python: >=3.6 -Description-Content-Type: text/x-rst -License-File: LICENSE -License-File: LICENSE.APACHE -License-File: LICENSE.BSD -Requires-Dist: pyparsing (!=3.0.5,>=2.0.2) - -packaging -========= - -.. start-intro - -Reusable core utilities for various Python Packaging -`interoperability specifications `_. - -This library provides utilities that implement the interoperability -specifications which have clearly one correct behaviour (eg: :pep:`440`) -or benefit greatly from having a single shared implementation (eg: :pep:`425`). - -.. end-intro - -The ``packaging`` project includes the following: version handling, specifiers, -markers, requirements, tags, utilities. - -Documentation -------------- - -The `documentation`_ provides information and the API for the following: - -- Version Handling -- Specifiers -- Markers -- Requirements -- Tags -- Utilities - -Installation ------------- - -Use ``pip`` to install these utilities:: - - pip install packaging - -Discussion ----------- - -If you run into bugs, you can file them in our `issue tracker`_. - -You can also join ``#pypa`` on Freenode to ask questions or get involved. - - -.. _`documentation`: https://packaging.pypa.io/ -.. _`issue tracker`: https://github.com/pypa/packaging/issues - - -Code of Conduct ---------------- - -Everyone interacting in the packaging project's codebases, issue trackers, chat -rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. - -.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md - -Contributing ------------- - -The ``CONTRIBUTING.rst`` file outlines how to contribute to this project as -well as how to report a potential security issue. The documentation for this -project also covers information about `project development`_ and `security`_. - -.. _`project development`: https://packaging.pypa.io/en/latest/development/ -.. _`security`: https://packaging.pypa.io/en/latest/security/ - -Project History ---------------- - -Please review the ``CHANGELOG.rst`` file or the `Changelog documentation`_ for -recent changes and project history. - -.. _`Changelog documentation`: https://packaging.pypa.io/en/latest/changelog/ - -Changelog ---------- - -21.3 - 2021-11-17 -~~~~~~~~~~~~~~~~~ - -* Add a ``pp3-none-any`` tag (`#311 `__) -* Replace the blank pyparsing 3 exclusion with a 3.0.5 exclusion (`#481 `__, `#486 `__) -* Fix a spelling mistake (`#479 `__) - -21.2 - 2021-10-29 -~~~~~~~~~~~~~~~~~ - -* Update documentation entry for 21.1. - -21.1 - 2021-10-29 -~~~~~~~~~~~~~~~~~ - -* Update pin to pyparsing to exclude 3.0.0. - -21.0 - 2021-07-03 -~~~~~~~~~~~~~~~~~ - -* PEP 656: musllinux support (`#411 `__) -* Drop support for Python 2.7, Python 3.4 and Python 3.5. -* Replace distutils usage with sysconfig (`#396 `__) -* Add support for zip files in ``parse_sdist_filename`` (`#429 `__) -* Use cached ``_hash`` attribute to short-circuit tag equality comparisons (`#417 `__) -* Specify the default value for the ``specifier`` argument to ``SpecifierSet`` (`#437 `__) -* Proper keyword-only "warn" argument in packaging.tags (`#403 `__) -* Correctly remove prerelease suffixes from ~= check (`#366 `__) -* Fix type hints for ``Version.post`` and ``Version.dev`` (`#393 `__) -* Use typing alias ``UnparsedVersion`` (`#398 `__) -* Improve type inference for ``packaging.specifiers.filter()`` (`#430 `__) -* Tighten the return type of ``canonicalize_version()`` (`#402 `__) - -20.9 - 2021-01-29 -~~~~~~~~~~~~~~~~~ - -* Run `isort `_ over the code base (`#377 `__) -* Add support for the ``macosx_10_*_universal2`` platform tags (`#379 `__) -* Introduce ``packaging.utils.parse_wheel_filename()`` and ``parse_sdist_filename()`` - (`#387 `__ and `#389 `__) - -20.8 - 2020-12-11 -~~~~~~~~~~~~~~~~~ - -* Revert back to setuptools for compatibility purposes for some Linux distros (`#363 `__) -* Do not insert an underscore in wheel tags when the interpreter version number - is more than 2 digits (`#372 `__) - -20.7 - 2020-11-28 -~~~~~~~~~~~~~~~~~ - -No unreleased changes. - -20.6 - 2020-11-28 -~~~~~~~~~~~~~~~~~ - -.. note:: This release was subsequently yanked, and these changes were included in 20.7. - -* Fix flit configuration, to include LICENSE files (`#357 `__) -* Make `intel` a recognized CPU architecture for the `universal` macOS platform tag (`#361 `__) -* Add some missing type hints to `packaging.requirements` (issue:`350`) - -20.5 - 2020-11-27 -~~~~~~~~~~~~~~~~~ - -* Officially support Python 3.9 (`#343 `__) -* Deprecate the ``LegacyVersion`` and ``LegacySpecifier`` classes (`#321 `__) -* Handle ``OSError`` on non-dynamic executables when attempting to resolve - the glibc version string. - -20.4 - 2020-05-19 -~~~~~~~~~~~~~~~~~ - -* Canonicalize version before comparing specifiers. (`#282 `__) -* Change type hint for ``canonicalize_name`` to return - ``packaging.utils.NormalizedName``. - This enables the use of static typing tools (like mypy) to detect mixing of - normalized and un-normalized names. - -20.3 - 2020-03-05 -~~~~~~~~~~~~~~~~~ - -* Fix changelog for 20.2. - -20.2 - 2020-03-05 -~~~~~~~~~~~~~~~~~ - -* Fix a bug that caused a 32-bit OS that runs on a 64-bit ARM CPU (e.g. ARM-v8, - aarch64), to report the wrong bitness. - -20.1 - 2020-01-24 -~~~~~~~~~~~~~~~~~~~ - -* Fix a bug caused by reuse of an exhausted iterator. (`#257 `__) - -20.0 - 2020-01-06 -~~~~~~~~~~~~~~~~~ - -* Add type hints (`#191 `__) - -* Add proper trove classifiers for PyPy support (`#198 `__) - -* Scale back depending on ``ctypes`` for manylinux support detection (`#171 `__) - -* Use ``sys.implementation.name`` where appropriate for ``packaging.tags`` (`#193 `__) - -* Expand upon the API provided by ``packaging.tags``: ``interpreter_name()``, ``mac_platforms()``, ``compatible_tags()``, ``cpython_tags()``, ``generic_tags()`` (`#187 `__) - -* Officially support Python 3.8 (`#232 `__) - -* Add ``major``, ``minor``, and ``micro`` aliases to ``packaging.version.Version`` (`#226 `__) - -* Properly mark ``packaging`` has being fully typed by adding a `py.typed` file (`#226 `__) - -19.2 - 2019-09-18 -~~~~~~~~~~~~~~~~~ - -* Remove dependency on ``attrs`` (`#178 `__, `#179 `__) - -* Use appropriate fallbacks for CPython ABI tag (`#181 `__, `#185 `__) - -* Add manylinux2014 support (`#186 `__) - -* Improve ABI detection (`#181 `__) - -* Properly handle debug wheels for Python 3.8 (`#172 `__) - -* Improve detection of debug builds on Windows (`#194 `__) - -19.1 - 2019-07-30 -~~~~~~~~~~~~~~~~~ - -* Add the ``packaging.tags`` module. (`#156 `__) - -* Correctly handle two-digit versions in ``python_version`` (`#119 `__) - - -19.0 - 2019-01-20 -~~~~~~~~~~~~~~~~~ - -* Fix string representation of PEP 508 direct URL requirements with markers. - -* Better handling of file URLs - - This allows for using ``file:///absolute/path``, which was previously - prevented due to the missing ``netloc``. - - This allows for all file URLs that ``urlunparse`` turns back into the - original URL to be valid. - - -18.0 - 2018-09-26 -~~~~~~~~~~~~~~~~~ - -* Improve error messages when invalid requirements are given. (`#129 `__) - - -17.1 - 2017-02-28 -~~~~~~~~~~~~~~~~~ - -* Fix ``utils.canonicalize_version`` when supplying non PEP 440 versions. - - -17.0 - 2017-02-28 -~~~~~~~~~~~~~~~~~ - -* Drop support for python 2.6, 3.2, and 3.3. - -* Define minimal pyparsing version to 2.0.2 (`#91 `__). - -* Add ``epoch``, ``release``, ``pre``, ``dev``, and ``post`` attributes to - ``Version`` and ``LegacyVersion`` (`#34 `__). - -* Add ``Version().is_devrelease`` and ``LegacyVersion().is_devrelease`` to - make it easy to determine if a release is a development release. - -* Add ``utils.canonicalize_version`` to canonicalize version strings or - ``Version`` instances (`#121 `__). - - -16.8 - 2016-10-29 -~~~~~~~~~~~~~~~~~ - -* Fix markers that utilize ``in`` so that they render correctly. - -* Fix an erroneous test on Python RC releases. - - -16.7 - 2016-04-23 -~~~~~~~~~~~~~~~~~ - -* Add support for the deprecated ``python_implementation`` marker which was - an undocumented setuptools marker in addition to the newer markers. - - -16.6 - 2016-03-29 -~~~~~~~~~~~~~~~~~ - -* Add support for the deprecated, PEP 345 environment markers in addition to - the newer markers. - - -16.5 - 2016-02-26 -~~~~~~~~~~~~~~~~~ - -* Fix a regression in parsing requirements with whitespaces between the comma - separators. - - -16.4 - 2016-02-22 -~~~~~~~~~~~~~~~~~ - -* Fix a regression in parsing requirements like ``foo (==4)``. - - -16.3 - 2016-02-21 -~~~~~~~~~~~~~~~~~ - -* Fix a bug where ``packaging.requirements:Requirement`` was overly strict when - matching legacy requirements. - - -16.2 - 2016-02-09 -~~~~~~~~~~~~~~~~~ - -* Add a function that implements the name canonicalization from PEP 503. - - -16.1 - 2016-02-07 -~~~~~~~~~~~~~~~~~ - -* Implement requirement specifiers from PEP 508. - - -16.0 - 2016-01-19 -~~~~~~~~~~~~~~~~~ - -* Relicense so that packaging is available under *either* the Apache License, - Version 2.0 or a 2 Clause BSD license. - -* Support installation of packaging when only distutils is available. - -* Fix ``==`` comparison when there is a prefix and a local version in play. - (`#41 `__). - -* Implement environment markers from PEP 508. - - -15.3 - 2015-08-01 -~~~~~~~~~~~~~~~~~ - -* Normalize post-release spellings for rev/r prefixes. `#35 `__ - - -15.2 - 2015-05-13 -~~~~~~~~~~~~~~~~~ - -* Fix an error where the arbitrary specifier (``===``) was not correctly - allowing pre-releases when it was being used. - -* Expose the specifier and version parts through properties on the - ``Specifier`` classes. - -* Allow iterating over the ``SpecifierSet`` to get access to all of the - ``Specifier`` instances. - -* Allow testing if a version is contained within a specifier via the ``in`` - operator. - - -15.1 - 2015-04-13 -~~~~~~~~~~~~~~~~~ - -* Fix a logic error that was causing inconsistent answers about whether or not - a pre-release was contained within a ``SpecifierSet`` or not. - - -15.0 - 2015-01-02 -~~~~~~~~~~~~~~~~~ - -* Add ``Version().is_postrelease`` and ``LegacyVersion().is_postrelease`` to - make it easy to determine if a release is a post release. - -* Add ``Version().base_version`` and ``LegacyVersion().base_version`` to make - it easy to get the public version without any pre or post release markers. - -* Support the update to PEP 440 which removed the implied ``!=V.*`` when using - either ``>V`` or ``V`` or ````) operator. - - -14.3 - 2014-11-19 -~~~~~~~~~~~~~~~~~ - -* **BACKWARDS INCOMPATIBLE** Refactor specifier support so that it can sanely - handle legacy specifiers as well as PEP 440 specifiers. - -* **BACKWARDS INCOMPATIBLE** Move the specifier support out of - ``packaging.version`` into ``packaging.specifiers``. - - -14.2 - 2014-09-10 -~~~~~~~~~~~~~~~~~ - -* Add prerelease support to ``Specifier``. -* Remove the ability to do ``item in Specifier()`` and replace it with - ``Specifier().contains(item)`` in order to allow flags that signal if a - prerelease should be accepted or not. -* Add a method ``Specifier().filter()`` which will take an iterable and returns - an iterable with items that do not match the specifier filtered out. - - -14.1 - 2014-09-08 -~~~~~~~~~~~~~~~~~ - -* Allow ``LegacyVersion`` and ``Version`` to be sorted together. -* Add ``packaging.version.parse()`` to enable easily parsing a version string - as either a ``Version`` or a ``LegacyVersion`` depending on it's PEP 440 - validity. - - -14.0 - 2014-09-05 -~~~~~~~~~~~~~~~~~ - -* Initial release. - - -.. _`master`: https://github.com/pypa/packaging/ - - diff --git a/third_party/python/packaging/packaging-21.3.dist-info/RECORD b/third_party/python/packaging/packaging-21.3.dist-info/RECORD deleted file mode 100644 index 870a8eb175f1..000000000000 --- a/third_party/python/packaging/packaging-21.3.dist-info/RECORD +++ /dev/null @@ -1,19 +0,0 @@ -packaging/__about__.py,sha256=ugASIO2w1oUyH8_COqQ2X_s0rDhjbhQC3yJocD03h2c,661 -packaging/__init__.py,sha256=b9Kk5MF7KxhhLgcDmiUWukN-LatWFxPdNug0joPhHSk,497 -packaging/_manylinux.py,sha256=XcbiXB-qcjv3bcohp6N98TMpOP4_j3m-iOA8ptK2GWY,11488 -packaging/_musllinux.py,sha256=_KGgY_qc7vhMGpoqss25n2hiLCNKRtvz9mCrS7gkqyc,4378 -packaging/_structures.py,sha256=q3eVNmbWJGG_S0Dit_S3Ao8qQqz_5PYTXFAKBZe5yr4,1431 -packaging/markers.py,sha256=Fygi3_eZnjQ-3VJizW5AhI5wvo0Hb6RMk4DidsKpOC0,8475 -packaging/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -packaging/requirements.py,sha256=rjaGRCMepZS1mlYMjJ5Qh6rfq3gtsCRQUQmftGZ_bu8,4664 -packaging/specifiers.py,sha256=LRQ0kFsHrl5qfcFNEEJrIFYsnIHQUJXY9fIsakTrrqE,30110 -packaging/tags.py,sha256=lmsnGNiJ8C4D_Pf9PbM0qgbZvD9kmB9lpZBQUZa3R_Y,15699 -packaging/utils.py,sha256=dJjeat3BS-TYn1RrUFVwufUMasbtzLfYRoy_HXENeFQ,4200 -packaging/version.py,sha256=_fLRNrFrxYcHVfyo8vk9j8s6JM8N_xsSxVFr6RJyco8,14665 -packaging-21.3.dist-info/LICENSE,sha256=ytHvW9NA1z4HS6YU0m996spceUDD2MNIUuZcSQlobEg,197 -packaging-21.3.dist-info/LICENSE.APACHE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174 -packaging-21.3.dist-info/LICENSE.BSD,sha256=tw5-m3QvHMb5SLNMFqo5_-zpQZY2S8iP8NIYDwAo-sU,1344 -packaging-21.3.dist-info/METADATA,sha256=KuKIy6qDLP3svIt6ejCbxBDhvq11ebkgUN55MeyKFyc,15147 -packaging-21.3.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92 -packaging-21.3.dist-info/top_level.txt,sha256=zFdHrhWnPslzsiP455HutQsqPB6v0KCtNUMtUtrefDw,10 -packaging-21.3.dist-info/RECORD,, diff --git a/third_party/python/packaging/packaging-21.3.dist-info/top_level.txt b/third_party/python/packaging/packaging-21.3.dist-info/top_level.txt deleted file mode 100644 index 748809f75c47..000000000000 --- a/third_party/python/packaging/packaging-21.3.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -packaging diff --git a/third_party/python/packaging/packaging-21.3.dist-info/LICENSE b/third_party/python/packaging/packaging-23.1.dist-info/LICENSE similarity index 100% rename from third_party/python/packaging/packaging-21.3.dist-info/LICENSE rename to third_party/python/packaging/packaging-23.1.dist-info/LICENSE diff --git a/third_party/python/packaging/packaging-21.3.dist-info/LICENSE.APACHE b/third_party/python/packaging/packaging-23.1.dist-info/LICENSE.APACHE similarity index 100% rename from third_party/python/packaging/packaging-21.3.dist-info/LICENSE.APACHE rename to third_party/python/packaging/packaging-23.1.dist-info/LICENSE.APACHE diff --git a/third_party/python/packaging/packaging-21.3.dist-info/LICENSE.BSD b/third_party/python/packaging/packaging-23.1.dist-info/LICENSE.BSD similarity index 100% rename from third_party/python/packaging/packaging-21.3.dist-info/LICENSE.BSD rename to third_party/python/packaging/packaging-23.1.dist-info/LICENSE.BSD diff --git a/third_party/python/packaging/packaging-23.1.dist-info/METADATA b/third_party/python/packaging/packaging-23.1.dist-info/METADATA new file mode 100644 index 000000000000..c43882a826ab --- /dev/null +++ b/third_party/python/packaging/packaging-23.1.dist-info/METADATA @@ -0,0 +1,99 @@ +Metadata-Version: 2.1 +Name: packaging +Version: 23.1 +Summary: Core utilities for Python packages +Author-email: Donald Stufft +Requires-Python: >=3.7 +Description-Content-Type: text/x-rst +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Typing :: Typed +Project-URL: Documentation, https://packaging.pypa.io/ +Project-URL: Source, https://github.com/pypa/packaging + +packaging +========= + +.. start-intro + +Reusable core utilities for various Python Packaging +`interoperability specifications `_. + +This library provides utilities that implement the interoperability +specifications which have clearly one correct behaviour (eg: :pep:`440`) +or benefit greatly from having a single shared implementation (eg: :pep:`425`). + +.. end-intro + +The ``packaging`` project includes the following: version handling, specifiers, +markers, requirements, tags, utilities. + +Documentation +------------- + +The `documentation`_ provides information and the API for the following: + +- Version Handling +- Specifiers +- Markers +- Requirements +- Tags +- Utilities + +Installation +------------ + +Use ``pip`` to install these utilities:: + + pip install packaging + +Discussion +---------- + +If you run into bugs, you can file them in our `issue tracker`_. + +You can also join ``#pypa`` on Freenode to ask questions or get involved. + + +.. _`documentation`: https://packaging.pypa.io/ +.. _`issue tracker`: https://github.com/pypa/packaging/issues + + +Code of Conduct +--------------- + +Everyone interacting in the packaging project's codebases, issue trackers, chat +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. + +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md + +Contributing +------------ + +The ``CONTRIBUTING.rst`` file outlines how to contribute to this project as +well as how to report a potential security issue. The documentation for this +project also covers information about `project development`_ and `security`_. + +.. _`project development`: https://packaging.pypa.io/en/latest/development/ +.. _`security`: https://packaging.pypa.io/en/latest/security/ + +Project History +--------------- + +Please review the ``CHANGELOG.rst`` file or the `Changelog documentation`_ for +recent changes and project history. + +.. _`Changelog documentation`: https://packaging.pypa.io/en/latest/changelog/ + diff --git a/third_party/python/packaging/packaging-23.1.dist-info/RECORD b/third_party/python/packaging/packaging-23.1.dist-info/RECORD new file mode 100644 index 000000000000..fbd3a0a4854b --- /dev/null +++ b/third_party/python/packaging/packaging-23.1.dist-info/RECORD @@ -0,0 +1,21 @@ +packaging/__init__.py,sha256=kYVZSmXT6CWInT4UJPDtrSQBAZu8fMuFBxpv5GsDTLk,501 +packaging/_elffile.py,sha256=hbmK8OD6Z7fY6hwinHEUcD1by7czkGiNYu7ShnFEk2k,3266 +packaging/_manylinux.py,sha256=ESGrDEVmBc8jYTtdZRAWiLk72lOzAKWeezFgoJ_MuBc,8926 +packaging/_musllinux.py,sha256=mvPk7FNjjILKRLIdMxR7IvJ1uggLgCszo-L9rjfpi0M,2524 +packaging/_parser.py,sha256=KJQkBh_Xbfb-qsB560YIEItrTpCZaOh4_YMfBtd5XIY,10194 +packaging/_structures.py,sha256=q3eVNmbWJGG_S0Dit_S3Ao8qQqz_5PYTXFAKBZe5yr4,1431 +packaging/_tokenizer.py,sha256=alCtbwXhOFAmFGZ6BQ-wCTSFoRAJ2z-ysIf7__MTJ_k,5292 +packaging/markers.py,sha256=eH-txS2zq1HdNpTd9LcZUcVIwewAiNU0grmq5wjKnOk,8208 +packaging/metadata.py,sha256=PjELMLxKG_iu3HWjKAOdKhuNrHfWgpdTF2Q4nObsZeM,16397 +packaging/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +packaging/requirements.py,sha256=hJzvtJyAvENc_VfwfhnOZV1851-VW8JCGh-R96NE4Pc,3287 +packaging/specifiers.py,sha256=ZOpqL_w_Kj6ZF_OWdliQUzhEyHlDbi6989kr-sF5GHs,39206 +packaging/tags.py,sha256=_1gLX8h1SgpjAdYCP9XqU37zRjXtU5ZliGy3IM-WcSM,18106 +packaging/utils.py,sha256=es0cCezKspzriQ-3V88h3yJzxz028euV2sUwM61kE-o,4355 +packaging/version.py,sha256=2NH3E57hzRhn0BV9boUBvgPsxlTqLJeI0EpYQoNvGi0,16326 +packaging-23.1.dist-info/LICENSE,sha256=ytHvW9NA1z4HS6YU0m996spceUDD2MNIUuZcSQlobEg,197 +packaging-23.1.dist-info/LICENSE.APACHE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174 +packaging-23.1.dist-info/LICENSE.BSD,sha256=tw5-m3QvHMb5SLNMFqo5_-zpQZY2S8iP8NIYDwAo-sU,1344 +packaging-23.1.dist-info/WHEEL,sha256=rSgq_JpHF9fHR1lx53qwg_1-2LypZE_qmcuXbVUq948,81 +packaging-23.1.dist-info/METADATA,sha256=JnduJDlxs2IVeB-nIqAC3-HyNcPhP_MADd9_k_MjmaI,3082 +packaging-23.1.dist-info/RECORD,, diff --git a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/WHEEL b/third_party/python/packaging/packaging-23.1.dist-info/WHEEL similarity index 53% rename from third_party/python/pyparsing/pyparsing-2.4.7.dist-info/WHEEL rename to third_party/python/packaging/packaging-23.1.dist-info/WHEEL index ef99c6cf3283..db4a255f3a27 100644 --- a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/WHEEL +++ b/third_party/python/packaging/packaging-23.1.dist-info/WHEEL @@ -1,6 +1,4 @@ Wheel-Version: 1.0 -Generator: bdist_wheel (0.34.2) +Generator: flit 3.8.0 Root-Is-Purelib: true -Tag: py2-none-any Tag: py3-none-any - diff --git a/third_party/python/packaging/packaging/__about__.py b/third_party/python/packaging/packaging/__about__.py deleted file mode 100644 index 3551bc2d2984..000000000000 --- a/third_party/python/packaging/packaging/__about__.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -__all__ = [ - "__title__", - "__summary__", - "__uri__", - "__version__", - "__author__", - "__email__", - "__license__", - "__copyright__", -] - -__title__ = "packaging" -__summary__ = "Core utilities for Python packages" -__uri__ = "https://github.com/pypa/packaging" - -__version__ = "21.3" - -__author__ = "Donald Stufft and individual contributors" -__email__ = "donald@stufft.io" - -__license__ = "BSD-2-Clause or Apache-2.0" -__copyright__ = "2014-2019 %s" % __author__ diff --git a/third_party/python/packaging/packaging/__init__.py b/third_party/python/packaging/packaging/__init__.py index 3c50c5dcfeed..13cadc7f04d4 100644 --- a/third_party/python/packaging/packaging/__init__.py +++ b/third_party/python/packaging/packaging/__init__.py @@ -2,24 +2,14 @@ # 2.0, and the BSD License. See the LICENSE file in the root of this repository # for complete details. -from .__about__ import ( - __author__, - __copyright__, - __email__, - __license__, - __summary__, - __title__, - __uri__, - __version__, -) +__title__ = "packaging" +__summary__ = "Core utilities for Python packages" +__uri__ = "https://github.com/pypa/packaging" -__all__ = [ - "__title__", - "__summary__", - "__uri__", - "__version__", - "__author__", - "__email__", - "__license__", - "__copyright__", -] +__version__ = "23.1" + +__author__ = "Donald Stufft and individual contributors" +__email__ = "donald@stufft.io" + +__license__ = "BSD-2-Clause or Apache-2.0" +__copyright__ = "2014-2019 %s" % __author__ diff --git a/third_party/python/packaging/packaging/_elffile.py b/third_party/python/packaging/packaging/_elffile.py new file mode 100644 index 000000000000..6fb19b30bb53 --- /dev/null +++ b/third_party/python/packaging/packaging/_elffile.py @@ -0,0 +1,108 @@ +""" +ELF file parser. + +This provides a class ``ELFFile`` that parses an ELF executable in a similar +interface to ``ZipFile``. Only the read interface is implemented. + +Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca +ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html +""" + +import enum +import os +import struct +from typing import IO, Optional, Tuple + + +class ELFInvalid(ValueError): + pass + + +class EIClass(enum.IntEnum): + C32 = 1 + C64 = 2 + + +class EIData(enum.IntEnum): + Lsb = 1 + Msb = 2 + + +class EMachine(enum.IntEnum): + I386 = 3 + S390 = 22 + Arm = 40 + X8664 = 62 + AArc64 = 183 + + +class ELFFile: + """ + Representation of an ELF executable. + """ + + def __init__(self, f: IO[bytes]) -> None: + self._f = f + + try: + ident = self._read("16B") + except struct.error: + raise ELFInvalid("unable to parse identification") + magic = bytes(ident[:4]) + if magic != b"\x7fELF": + raise ELFInvalid(f"invalid magic: {magic!r}") + + self.capacity = ident[4] # Format for program header (bitness). + self.encoding = ident[5] # Data structure encoding (endianness). + + try: + # e_fmt: Format for program header. + # p_fmt: Format for section header. + # p_idx: Indexes to find p_type, p_offset, and p_filesz. + e_fmt, self._p_fmt, self._p_idx = { + (1, 1): ("HHIIIIIHHH", ">IIIIIIII", (0, 1, 4)), # 32-bit MSB. + (2, 1): ("HHIQQQIHHH", ">IIQQQQQQ", (0, 2, 5)), # 64-bit MSB. + }[(self.capacity, self.encoding)] + except KeyError: + raise ELFInvalid( + f"unrecognized capacity ({self.capacity}) or " + f"encoding ({self.encoding})" + ) + + try: + ( + _, + self.machine, # Architecture type. + _, + _, + self._e_phoff, # Offset of program header. + _, + self.flags, # Processor-specific flags. + _, + self._e_phentsize, # Size of section. + self._e_phnum, # Number of sections. + ) = self._read(e_fmt) + except struct.error as e: + raise ELFInvalid("unable to parse machine and section information") from e + + def _read(self, fmt: str) -> Tuple[int, ...]: + return struct.unpack(fmt, self._f.read(struct.calcsize(fmt))) + + @property + def interpreter(self) -> Optional[str]: + """ + The path recorded in the ``PT_INTERP`` section header. + """ + for index in range(self._e_phnum): + self._f.seek(self._e_phoff + self._e_phentsize * index) + try: + data = self._read(self._p_fmt) + except struct.error: + continue + if data[self._p_idx[0]] != 3: # Not PT_INTERP. + continue + self._f.seek(data[self._p_idx[1]]) + return os.fsdecode(self._f.read(data[self._p_idx[2]])).strip("\0") + return None diff --git a/third_party/python/packaging/packaging/_manylinux.py b/third_party/python/packaging/packaging/_manylinux.py index 4c379aa6f69f..449c655be65a 100644 --- a/third_party/python/packaging/packaging/_manylinux.py +++ b/third_party/python/packaging/packaging/_manylinux.py @@ -1,121 +1,60 @@ import collections +import contextlib import functools import os import re -import struct import sys import warnings -from typing import IO, Dict, Iterator, NamedTuple, Optional, Tuple +from typing import Dict, Generator, Iterator, NamedTuple, Optional, Tuple + +from ._elffile import EIClass, EIData, ELFFile, EMachine + +EF_ARM_ABIMASK = 0xFF000000 +EF_ARM_ABI_VER5 = 0x05000000 +EF_ARM_ABI_FLOAT_HARD = 0x00000400 -# Python does not provide platform information at sufficient granularity to -# identify the architecture of the running executable in some cases, so we -# determine it dynamically by reading the information from the running -# process. This only applies on Linux, which uses the ELF format. -class _ELFFileHeader: - # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header - class _InvalidELFFileHeader(ValueError): - """ - An invalid ELF file header was found. - """ - - ELF_MAGIC_NUMBER = 0x7F454C46 - ELFCLASS32 = 1 - ELFCLASS64 = 2 - ELFDATA2LSB = 1 - ELFDATA2MSB = 2 - EM_386 = 3 - EM_S390 = 22 - EM_ARM = 40 - EM_X86_64 = 62 - EF_ARM_ABIMASK = 0xFF000000 - EF_ARM_ABI_VER5 = 0x05000000 - EF_ARM_ABI_FLOAT_HARD = 0x00000400 - - def __init__(self, file: IO[bytes]) -> None: - def unpack(fmt: str) -> int: - try: - data = file.read(struct.calcsize(fmt)) - result: Tuple[int, ...] = struct.unpack(fmt, data) - except struct.error: - raise _ELFFileHeader._InvalidELFFileHeader() - return result[0] - - self.e_ident_magic = unpack(">I") - if self.e_ident_magic != self.ELF_MAGIC_NUMBER: - raise _ELFFileHeader._InvalidELFFileHeader() - self.e_ident_class = unpack("B") - if self.e_ident_class not in {self.ELFCLASS32, self.ELFCLASS64}: - raise _ELFFileHeader._InvalidELFFileHeader() - self.e_ident_data = unpack("B") - if self.e_ident_data not in {self.ELFDATA2LSB, self.ELFDATA2MSB}: - raise _ELFFileHeader._InvalidELFFileHeader() - self.e_ident_version = unpack("B") - self.e_ident_osabi = unpack("B") - self.e_ident_abiversion = unpack("B") - self.e_ident_pad = file.read(7) - format_h = "H" - format_i = "I" - format_q = "Q" - format_p = format_i if self.e_ident_class == self.ELFCLASS32 else format_q - self.e_type = unpack(format_h) - self.e_machine = unpack(format_h) - self.e_version = unpack(format_i) - self.e_entry = unpack(format_p) - self.e_phoff = unpack(format_p) - self.e_shoff = unpack(format_p) - self.e_flags = unpack(format_i) - self.e_ehsize = unpack(format_h) - self.e_phentsize = unpack(format_h) - self.e_phnum = unpack(format_h) - self.e_shentsize = unpack(format_h) - self.e_shnum = unpack(format_h) - self.e_shstrndx = unpack(format_h) - - -def _get_elf_header() -> Optional[_ELFFileHeader]: +# `os.PathLike` not a generic type until Python 3.9, so sticking with `str` +# as the type for `path` until then. +@contextlib.contextmanager +def _parse_elf(path: str) -> Generator[Optional[ELFFile], None, None]: try: - with open(sys.executable, "rb") as f: - elf_header = _ELFFileHeader(f) - except (OSError, TypeError, _ELFFileHeader._InvalidELFFileHeader): - return None - return elf_header + with open(path, "rb") as f: + yield ELFFile(f) + except (OSError, TypeError, ValueError): + yield None -def _is_linux_armhf() -> bool: +def _is_linux_armhf(executable: str) -> bool: # hard-float ABI can be detected from the ELF header of the running # process # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf - elf_header = _get_elf_header() - if elf_header is None: - return False - result = elf_header.e_ident_class == elf_header.ELFCLASS32 - result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB - result &= elf_header.e_machine == elf_header.EM_ARM - result &= ( - elf_header.e_flags & elf_header.EF_ARM_ABIMASK - ) == elf_header.EF_ARM_ABI_VER5 - result &= ( - elf_header.e_flags & elf_header.EF_ARM_ABI_FLOAT_HARD - ) == elf_header.EF_ARM_ABI_FLOAT_HARD - return result + with _parse_elf(executable) as f: + return ( + f is not None + and f.capacity == EIClass.C32 + and f.encoding == EIData.Lsb + and f.machine == EMachine.Arm + and f.flags & EF_ARM_ABIMASK == EF_ARM_ABI_VER5 + and f.flags & EF_ARM_ABI_FLOAT_HARD == EF_ARM_ABI_FLOAT_HARD + ) -def _is_linux_i686() -> bool: - elf_header = _get_elf_header() - if elf_header is None: - return False - result = elf_header.e_ident_class == elf_header.ELFCLASS32 - result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB - result &= elf_header.e_machine == elf_header.EM_386 - return result +def _is_linux_i686(executable: str) -> bool: + with _parse_elf(executable) as f: + return ( + f is not None + and f.capacity == EIClass.C32 + and f.encoding == EIData.Lsb + and f.machine == EMachine.I386 + ) -def _have_compatible_abi(arch: str) -> bool: +def _have_compatible_abi(executable: str, arch: str) -> bool: if arch == "armv7l": - return _is_linux_armhf() + return _is_linux_armhf(executable) if arch == "i686": - return _is_linux_i686() + return _is_linux_i686(executable) return arch in {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x"} @@ -141,10 +80,10 @@ def _glibc_version_string_confstr() -> Optional[str]: # platform module. # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183 try: - # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17". - version_string = os.confstr("CS_GNU_LIBC_VERSION") + # Should be a string like "glibc 2.17". + version_string: str = getattr(os, "confstr")("CS_GNU_LIBC_VERSION") assert version_string is not None - _, version = version_string.split() + _, version = version_string.rsplit() except (AssertionError, AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None @@ -211,8 +150,8 @@ def _parse_glibc_version(version_str: str) -> Tuple[int, int]: m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str) if not m: warnings.warn( - "Expected glibc version with 2 components major.minor," - " got: %s" % version_str, + f"Expected glibc version with 2 components major.minor," + f" got: {version_str}", RuntimeWarning, ) return -1, -1 @@ -265,7 +204,7 @@ _LEGACY_MANYLINUX_MAP = { def platform_tags(linux: str, arch: str) -> Iterator[str]: - if not _have_compatible_abi(arch): + if not _have_compatible_abi(sys.executable, arch): return # Oldest glibc to be supported regardless of architecture is (2, 17). too_old_glibc2 = _GLibCVersion(2, 16) diff --git a/third_party/python/packaging/packaging/_musllinux.py b/third_party/python/packaging/packaging/_musllinux.py index 8ac3059ba3c2..706ba600a93c 100644 --- a/third_party/python/packaging/packaging/_musllinux.py +++ b/third_party/python/packaging/packaging/_musllinux.py @@ -4,68 +4,13 @@ This module implements logic to detect if the currently running Python is linked against musl, and what musl version is used. """ -import contextlib import functools -import operator -import os import re -import struct import subprocess import sys -from typing import IO, Iterator, NamedTuple, Optional, Tuple +from typing import Iterator, NamedTuple, Optional - -def _read_unpacked(f: IO[bytes], fmt: str) -> Tuple[int, ...]: - return struct.unpack(fmt, f.read(struct.calcsize(fmt))) - - -def _parse_ld_musl_from_elf(f: IO[bytes]) -> Optional[str]: - """Detect musl libc location by parsing the Python executable. - - Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca - ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html - """ - f.seek(0) - try: - ident = _read_unpacked(f, "16B") - except struct.error: - return None - if ident[:4] != tuple(b"\x7fELF"): # Invalid magic, not ELF. - return None - f.seek(struct.calcsize("HHI"), 1) # Skip file type, machine, and version. - - try: - # e_fmt: Format for program header. - # p_fmt: Format for section header. - # p_idx: Indexes to find p_type, p_offset, and p_filesz. - e_fmt, p_fmt, p_idx = { - 1: ("IIIIHHH", "IIIIIIII", (0, 1, 4)), # 32-bit. - 2: ("QQQIHHH", "IIQQQQQQ", (0, 2, 5)), # 64-bit. - }[ident[4]] - except KeyError: - return None - else: - p_get = operator.itemgetter(*p_idx) - - # Find the interpreter section and return its content. - try: - _, e_phoff, _, _, _, e_phentsize, e_phnum = _read_unpacked(f, e_fmt) - except struct.error: - return None - for i in range(e_phnum + 1): - f.seek(e_phoff + e_phentsize * i) - try: - p_type, p_offset, p_filesz = p_get(_read_unpacked(f, p_fmt)) - except struct.error: - return None - if p_type != 3: # Not PT_INTERP. - continue - f.seek(p_offset) - interpreter = os.fsdecode(f.read(p_filesz)).strip("\0") - if "musl" not in interpreter: - return None - return interpreter - return None +from ._elffile import ELFFile class _MuslVersion(NamedTuple): @@ -95,13 +40,12 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]: Version 1.2.2 Dynamic Program Loader """ - with contextlib.ExitStack() as stack: - try: - f = stack.enter_context(open(executable, "rb")) - except OSError: - return None - ld = _parse_ld_musl_from_elf(f) - if not ld: + try: + with open(executable, "rb") as f: + ld = ELFFile(f).interpreter + except (OSError, TypeError, ValueError): + return None + if ld is None or "musl" not in ld: return None proc = subprocess.run([ld], stderr=subprocess.PIPE, universal_newlines=True) return _parse_musl_version(proc.stderr) diff --git a/third_party/python/packaging/packaging/_parser.py b/third_party/python/packaging/packaging/_parser.py new file mode 100644 index 000000000000..5a18b758fe00 --- /dev/null +++ b/third_party/python/packaging/packaging/_parser.py @@ -0,0 +1,353 @@ +"""Handwritten parser of dependency specifiers. + +The docstring for each __parse_* function contains ENBF-inspired grammar representing +the implementation. +""" + +import ast +from typing import Any, List, NamedTuple, Optional, Tuple, Union + +from ._tokenizer import DEFAULT_RULES, Tokenizer + + +class Node: + def __init__(self, value: str) -> None: + self.value = value + + def __str__(self) -> str: + return self.value + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}('{self}')>" + + def serialize(self) -> str: + raise NotImplementedError + + +class Variable(Node): + def serialize(self) -> str: + return str(self) + + +class Value(Node): + def serialize(self) -> str: + return f'"{self}"' + + +class Op(Node): + def serialize(self) -> str: + return str(self) + + +MarkerVar = Union[Variable, Value] +MarkerItem = Tuple[MarkerVar, Op, MarkerVar] +# MarkerAtom = Union[MarkerItem, List["MarkerAtom"]] +# MarkerList = List[Union["MarkerList", MarkerAtom, str]] +# mypy does not support recursive type definition +# https://github.com/python/mypy/issues/731 +MarkerAtom = Any +MarkerList = List[Any] + + +class ParsedRequirement(NamedTuple): + name: str + url: str + extras: List[str] + specifier: str + marker: Optional[MarkerList] + + +# -------------------------------------------------------------------------------------- +# Recursive descent parser for dependency specifier +# -------------------------------------------------------------------------------------- +def parse_requirement(source: str) -> ParsedRequirement: + return _parse_requirement(Tokenizer(source, rules=DEFAULT_RULES)) + + +def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: + """ + requirement = WS? IDENTIFIER WS? extras WS? requirement_details + """ + tokenizer.consume("WS") + + name_token = tokenizer.expect( + "IDENTIFIER", expected="package name at the start of dependency specifier" + ) + name = name_token.text + tokenizer.consume("WS") + + extras = _parse_extras(tokenizer) + tokenizer.consume("WS") + + url, specifier, marker = _parse_requirement_details(tokenizer) + tokenizer.expect("END", expected="end of dependency specifier") + + return ParsedRequirement(name, url, extras, specifier, marker) + + +def _parse_requirement_details( + tokenizer: Tokenizer, +) -> Tuple[str, str, Optional[MarkerList]]: + """ + requirement_details = AT URL (WS requirement_marker?)? + | specifier WS? (requirement_marker)? + """ + + specifier = "" + url = "" + marker = None + + if tokenizer.check("AT"): + tokenizer.read() + tokenizer.consume("WS") + + url_start = tokenizer.position + url = tokenizer.expect("URL", expected="URL after @").text + if tokenizer.check("END", peek=True): + return (url, specifier, marker) + + tokenizer.expect("WS", expected="whitespace after URL") + + # The input might end after whitespace. + if tokenizer.check("END", peek=True): + return (url, specifier, marker) + + marker = _parse_requirement_marker( + tokenizer, span_start=url_start, after="URL and whitespace" + ) + else: + specifier_start = tokenizer.position + specifier = _parse_specifier(tokenizer) + tokenizer.consume("WS") + + if tokenizer.check("END", peek=True): + return (url, specifier, marker) + + marker = _parse_requirement_marker( + tokenizer, + span_start=specifier_start, + after=( + "version specifier" + if specifier + else "name and no valid version specifier" + ), + ) + + return (url, specifier, marker) + + +def _parse_requirement_marker( + tokenizer: Tokenizer, *, span_start: int, after: str +) -> MarkerList: + """ + requirement_marker = SEMICOLON marker WS? + """ + + if not tokenizer.check("SEMICOLON"): + tokenizer.raise_syntax_error( + f"Expected end or semicolon (after {after})", + span_start=span_start, + ) + tokenizer.read() + + marker = _parse_marker(tokenizer) + tokenizer.consume("WS") + + return marker + + +def _parse_extras(tokenizer: Tokenizer) -> List[str]: + """ + extras = (LEFT_BRACKET wsp* extras_list? wsp* RIGHT_BRACKET)? + """ + if not tokenizer.check("LEFT_BRACKET", peek=True): + return [] + + with tokenizer.enclosing_tokens( + "LEFT_BRACKET", + "RIGHT_BRACKET", + around="extras", + ): + tokenizer.consume("WS") + extras = _parse_extras_list(tokenizer) + tokenizer.consume("WS") + + return extras + + +def _parse_extras_list(tokenizer: Tokenizer) -> List[str]: + """ + extras_list = identifier (wsp* ',' wsp* identifier)* + """ + extras: List[str] = [] + + if not tokenizer.check("IDENTIFIER"): + return extras + + extras.append(tokenizer.read().text) + + while True: + tokenizer.consume("WS") + if tokenizer.check("IDENTIFIER", peek=True): + tokenizer.raise_syntax_error("Expected comma between extra names") + elif not tokenizer.check("COMMA"): + break + + tokenizer.read() + tokenizer.consume("WS") + + extra_token = tokenizer.expect("IDENTIFIER", expected="extra name after comma") + extras.append(extra_token.text) + + return extras + + +def _parse_specifier(tokenizer: Tokenizer) -> str: + """ + specifier = LEFT_PARENTHESIS WS? version_many WS? RIGHT_PARENTHESIS + | WS? version_many WS? + """ + with tokenizer.enclosing_tokens( + "LEFT_PARENTHESIS", + "RIGHT_PARENTHESIS", + around="version specifier", + ): + tokenizer.consume("WS") + parsed_specifiers = _parse_version_many(tokenizer) + tokenizer.consume("WS") + + return parsed_specifiers + + +def _parse_version_many(tokenizer: Tokenizer) -> str: + """ + version_many = (SPECIFIER (WS? COMMA WS? SPECIFIER)*)? + """ + parsed_specifiers = "" + while tokenizer.check("SPECIFIER"): + span_start = tokenizer.position + parsed_specifiers += tokenizer.read().text + if tokenizer.check("VERSION_PREFIX_TRAIL", peek=True): + tokenizer.raise_syntax_error( + ".* suffix can only be used with `==` or `!=` operators", + span_start=span_start, + span_end=tokenizer.position + 1, + ) + if tokenizer.check("VERSION_LOCAL_LABEL_TRAIL", peek=True): + tokenizer.raise_syntax_error( + "Local version label can only be used with `==` or `!=` operators", + span_start=span_start, + span_end=tokenizer.position, + ) + tokenizer.consume("WS") + if not tokenizer.check("COMMA"): + break + parsed_specifiers += tokenizer.read().text + tokenizer.consume("WS") + + return parsed_specifiers + + +# -------------------------------------------------------------------------------------- +# Recursive descent parser for marker expression +# -------------------------------------------------------------------------------------- +def parse_marker(source: str) -> MarkerList: + return _parse_marker(Tokenizer(source, rules=DEFAULT_RULES)) + + +def _parse_marker(tokenizer: Tokenizer) -> MarkerList: + """ + marker = marker_atom (BOOLOP marker_atom)+ + """ + expression = [_parse_marker_atom(tokenizer)] + while tokenizer.check("BOOLOP"): + token = tokenizer.read() + expr_right = _parse_marker_atom(tokenizer) + expression.extend((token.text, expr_right)) + return expression + + +def _parse_marker_atom(tokenizer: Tokenizer) -> MarkerAtom: + """ + marker_atom = WS? LEFT_PARENTHESIS WS? marker WS? RIGHT_PARENTHESIS WS? + | WS? marker_item WS? + """ + + tokenizer.consume("WS") + if tokenizer.check("LEFT_PARENTHESIS", peek=True): + with tokenizer.enclosing_tokens( + "LEFT_PARENTHESIS", + "RIGHT_PARENTHESIS", + around="marker expression", + ): + tokenizer.consume("WS") + marker: MarkerAtom = _parse_marker(tokenizer) + tokenizer.consume("WS") + else: + marker = _parse_marker_item(tokenizer) + tokenizer.consume("WS") + return marker + + +def _parse_marker_item(tokenizer: Tokenizer) -> MarkerItem: + """ + marker_item = WS? marker_var WS? marker_op WS? marker_var WS? + """ + tokenizer.consume("WS") + marker_var_left = _parse_marker_var(tokenizer) + tokenizer.consume("WS") + marker_op = _parse_marker_op(tokenizer) + tokenizer.consume("WS") + marker_var_right = _parse_marker_var(tokenizer) + tokenizer.consume("WS") + return (marker_var_left, marker_op, marker_var_right) + + +def _parse_marker_var(tokenizer: Tokenizer) -> MarkerVar: + """ + marker_var = VARIABLE | QUOTED_STRING + """ + if tokenizer.check("VARIABLE"): + return process_env_var(tokenizer.read().text.replace(".", "_")) + elif tokenizer.check("QUOTED_STRING"): + return process_python_str(tokenizer.read().text) + else: + tokenizer.raise_syntax_error( + message="Expected a marker variable or quoted string" + ) + + +def process_env_var(env_var: str) -> Variable: + if ( + env_var == "platform_python_implementation" + or env_var == "python_implementation" + ): + return Variable("platform_python_implementation") + else: + return Variable(env_var) + + +def process_python_str(python_str: str) -> Value: + value = ast.literal_eval(python_str) + return Value(str(value)) + + +def _parse_marker_op(tokenizer: Tokenizer) -> Op: + """ + marker_op = IN | NOT IN | OP + """ + if tokenizer.check("IN"): + tokenizer.read() + return Op("in") + elif tokenizer.check("NOT"): + tokenizer.read() + tokenizer.expect("WS", expected="whitespace after 'not'") + tokenizer.expect("IN", expected="'in' after 'not'") + return Op("not in") + elif tokenizer.check("OP"): + return Op(tokenizer.read().text) + else: + return tokenizer.raise_syntax_error( + "Expected marker operator, one of " + "<=, <, !=, ==, >=, >, ~=, ===, in, not in" + ) diff --git a/third_party/python/packaging/packaging/_tokenizer.py b/third_party/python/packaging/packaging/_tokenizer.py new file mode 100644 index 000000000000..dd0d648d49a7 --- /dev/null +++ b/third_party/python/packaging/packaging/_tokenizer.py @@ -0,0 +1,192 @@ +import contextlib +import re +from dataclasses import dataclass +from typing import Dict, Iterator, NoReturn, Optional, Tuple, Union + +from .specifiers import Specifier + + +@dataclass +class Token: + name: str + text: str + position: int + + +class ParserSyntaxError(Exception): + """The provided source text could not be parsed correctly.""" + + def __init__( + self, + message: str, + *, + source: str, + span: Tuple[int, int], + ) -> None: + self.span = span + self.message = message + self.source = source + + super().__init__() + + def __str__(self) -> str: + marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^" + return "\n ".join([self.message, self.source, marker]) + + +DEFAULT_RULES: "Dict[str, Union[str, re.Pattern[str]]]" = { + "LEFT_PARENTHESIS": r"\(", + "RIGHT_PARENTHESIS": r"\)", + "LEFT_BRACKET": r"\[", + "RIGHT_BRACKET": r"\]", + "SEMICOLON": r";", + "COMMA": r",", + "QUOTED_STRING": re.compile( + r""" + ( + ('[^']*') + | + ("[^"]*") + ) + """, + re.VERBOSE, + ), + "OP": r"(===|==|~=|!=|<=|>=|<|>)", + "BOOLOP": r"\b(or|and)\b", + "IN": r"\bin\b", + "NOT": r"\bnot\b", + "VARIABLE": re.compile( + r""" + \b( + python_version + |python_full_version + |os[._]name + |sys[._]platform + |platform_(release|system) + |platform[._](version|machine|python_implementation) + |python_implementation + |implementation_(name|version) + |extra + )\b + """, + re.VERBOSE, + ), + "SPECIFIER": re.compile( + Specifier._operator_regex_str + Specifier._version_regex_str, + re.VERBOSE | re.IGNORECASE, + ), + "AT": r"\@", + "URL": r"[^ \t]+", + "IDENTIFIER": r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b", + "VERSION_PREFIX_TRAIL": r"\.\*", + "VERSION_LOCAL_LABEL_TRAIL": r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*", + "WS": r"[ \t]+", + "END": r"$", +} + + +class Tokenizer: + """Context-sensitive token parsing. + + Provides methods to examine the input stream to check whether the next token + matches. + """ + + def __init__( + self, + source: str, + *, + rules: "Dict[str, Union[str, re.Pattern[str]]]", + ) -> None: + self.source = source + self.rules: Dict[str, re.Pattern[str]] = { + name: re.compile(pattern) for name, pattern in rules.items() + } + self.next_token: Optional[Token] = None + self.position = 0 + + def consume(self, name: str) -> None: + """Move beyond provided token name, if at current position.""" + if self.check(name): + self.read() + + def check(self, name: str, *, peek: bool = False) -> bool: + """Check whether the next token has the provided name. + + By default, if the check succeeds, the token *must* be read before + another check. If `peek` is set to `True`, the token is not loaded and + would need to be checked again. + """ + assert ( + self.next_token is None + ), f"Cannot check for {name!r}, already have {self.next_token!r}" + assert name in self.rules, f"Unknown token name: {name!r}" + + expression = self.rules[name] + + match = expression.match(self.source, self.position) + if match is None: + return False + if not peek: + self.next_token = Token(name, match[0], self.position) + return True + + def expect(self, name: str, *, expected: str) -> Token: + """Expect a certain token name next, failing with a syntax error otherwise. + + The token is *not* read. + """ + if not self.check(name): + raise self.raise_syntax_error(f"Expected {expected}") + return self.read() + + def read(self) -> Token: + """Consume the next token and return it.""" + token = self.next_token + assert token is not None + + self.position += len(token.text) + self.next_token = None + + return token + + def raise_syntax_error( + self, + message: str, + *, + span_start: Optional[int] = None, + span_end: Optional[int] = None, + ) -> NoReturn: + """Raise ParserSyntaxError at the given position.""" + span = ( + self.position if span_start is None else span_start, + self.position if span_end is None else span_end, + ) + raise ParserSyntaxError( + message, + source=self.source, + span=span, + ) + + @contextlib.contextmanager + def enclosing_tokens( + self, open_token: str, close_token: str, *, around: str + ) -> Iterator[None]: + if self.check(open_token): + open_position = self.position + self.read() + else: + open_position = None + + yield + + if open_position is None: + return + + if not self.check(close_token): + self.raise_syntax_error( + f"Expected matching {close_token} for {open_token}, after {around}", + span_start=open_position, + ) + + self.read() diff --git a/third_party/python/packaging/packaging/markers.py b/third_party/python/packaging/packaging/markers.py index cb640e8f9b80..8b98fca7233b 100644 --- a/third_party/python/packaging/packaging/markers.py +++ b/third_party/python/packaging/packaging/markers.py @@ -8,19 +8,17 @@ import platform import sys from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from pyparsing import ( # noqa: N817 - Forward, - Group, - Literal as L, - ParseException, - ParseResults, - QuotedString, - ZeroOrMore, - stringEnd, - stringStart, +from ._parser import ( + MarkerAtom, + MarkerList, + Op, + Value, + Variable, + parse_marker as _parse_marker, ) - +from ._tokenizer import ParserSyntaxError from .specifiers import InvalidSpecifier, Specifier +from .utils import canonicalize_name __all__ = [ "InvalidMarker", @@ -52,101 +50,24 @@ class UndefinedEnvironmentName(ValueError): """ -class Node: - def __init__(self, value: Any) -> None: - self.value = value - - def __str__(self) -> str: - return str(self.value) - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" - - def serialize(self) -> str: - raise NotImplementedError - - -class Variable(Node): - def serialize(self) -> str: - return str(self) - - -class Value(Node): - def serialize(self) -> str: - return f'"{self}"' - - -class Op(Node): - def serialize(self) -> str: - return str(self) - - -VARIABLE = ( - L("implementation_version") - | L("platform_python_implementation") - | L("implementation_name") - | L("python_full_version") - | L("platform_release") - | L("platform_version") - | L("platform_machine") - | L("platform_system") - | L("python_version") - | L("sys_platform") - | L("os_name") - | L("os.name") # PEP-345 - | L("sys.platform") # PEP-345 - | L("platform.version") # PEP-345 - | L("platform.machine") # PEP-345 - | L("platform.python_implementation") # PEP-345 - | L("python_implementation") # undocumented setuptools legacy - | L("extra") # PEP-508 -) -ALIASES = { - "os.name": "os_name", - "sys.platform": "sys_platform", - "platform.version": "platform_version", - "platform.machine": "platform_machine", - "platform.python_implementation": "platform_python_implementation", - "python_implementation": "platform_python_implementation", -} -VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) - -VERSION_CMP = ( - L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") -) - -MARKER_OP = VERSION_CMP | L("not in") | L("in") -MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) - -MARKER_VALUE = QuotedString("'") | QuotedString('"') -MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) - -BOOLOP = L("and") | L("or") - -MARKER_VAR = VARIABLE | MARKER_VALUE - -MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) -MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) - -LPAREN = L("(").suppress() -RPAREN = L(")").suppress() - -MARKER_EXPR = Forward() -MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) -MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) - -MARKER = stringStart + MARKER_EXPR + stringEnd - - -def _coerce_parse_result(results: Union[ParseResults, List[Any]]) -> List[Any]: - if isinstance(results, ParseResults): - return [_coerce_parse_result(i) for i in results] - else: - return results +def _normalize_extra_values(results: Any) -> Any: + """ + Normalize extra values. + """ + if isinstance(results[0], tuple): + lhs, op, rhs = results[0] + if isinstance(lhs, Variable) and lhs.value == "extra": + normalized_extra = canonicalize_name(rhs.value) + rhs = Value(normalized_extra) + elif isinstance(rhs, Variable) and rhs.value == "extra": + normalized_extra = canonicalize_name(lhs.value) + lhs = Value(normalized_extra) + results[0] = lhs, op, rhs + return results def _format_marker( - marker: Union[List[str], Tuple[Node, ...], str], first: Optional[bool] = True + marker: Union[List[str], MarkerAtom, str], first: Optional[bool] = True ) -> str: assert isinstance(marker, (list, tuple, str)) @@ -192,7 +113,7 @@ def _eval_op(lhs: str, op: Op, rhs: str) -> bool: except InvalidSpecifier: pass else: - return spec.contains(lhs) + return spec.contains(lhs, prereleases=True) oper: Optional[Operator] = _operators.get(op.serialize()) if oper is None: @@ -201,25 +122,19 @@ def _eval_op(lhs: str, op: Op, rhs: str) -> bool: return oper(lhs, rhs) -class Undefined: - pass +def _normalize(*values: str, key: str) -> Tuple[str, ...]: + # PEP 685 – Comparison of extra names for optional distribution dependencies + # https://peps.python.org/pep-0685/ + # > When comparing extra names, tools MUST normalize the names being + # > compared using the semantics outlined in PEP 503 for names + if key == "extra": + return tuple(canonicalize_name(v) for v in values) + + # other environment markers don't have such standards + return values -_undefined = Undefined() - - -def _get_env(environment: Dict[str, str], name: str) -> str: - value: Union[str, Undefined] = environment.get(name, _undefined) - - if isinstance(value, Undefined): - raise UndefinedEnvironmentName( - f"{name!r} does not exist in evaluation environment." - ) - - return value - - -def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: +def _evaluate_markers(markers: MarkerList, environment: Dict[str, str]) -> bool: groups: List[List[bool]] = [[]] for marker in markers: @@ -231,12 +146,15 @@ def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: lhs, op, rhs = marker if isinstance(lhs, Variable): - lhs_value = _get_env(environment, lhs.value) + environment_key = lhs.value + lhs_value = environment[environment_key] rhs_value = rhs.value else: lhs_value = lhs.value - rhs_value = _get_env(environment, rhs.value) + environment_key = rhs.value + rhs_value = environment[environment_key] + lhs_value, rhs_value = _normalize(lhs_value, rhs_value, key=environment_key) groups[-1].append(_eval_op(lhs_value, op, rhs_value)) else: assert marker in ["and", "or"] @@ -274,13 +192,29 @@ def default_environment() -> Dict[str, str]: class Marker: def __init__(self, marker: str) -> None: + # Note: We create a Marker object without calling this constructor in + # packaging.requirements.Requirement. If any additional logic is + # added here, make sure to mirror/adapt Requirement. try: - self._markers = _coerce_parse_result(MARKER.parseString(marker)) - except ParseException as e: - raise InvalidMarker( - f"Invalid marker: {marker!r}, parse error at " - f"{marker[e.loc : e.loc + 8]!r}" - ) + self._markers = _normalize_extra_values(_parse_marker(marker)) + # The attribute `_markers` can be described in terms of a recursive type: + # MarkerList = List[Union[Tuple[Node, ...], str, MarkerList]] + # + # For example, the following expression: + # python_version > "3.6" or (python_version == "3.6" and os_name == "unix") + # + # is parsed into: + # [ + # (, ')>, ), + # 'and', + # [ + # (, , ), + # 'or', + # (, , ) + # ] + # ] + except ParserSyntaxError as e: + raise InvalidMarker(str(e)) from e def __str__(self) -> str: return _format_marker(self._markers) @@ -288,6 +222,15 @@ class Marker: def __repr__(self) -> str: return f"" + def __hash__(self) -> int: + return hash((self.__class__.__name__, str(self))) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Marker): + return NotImplemented + + return str(self) == str(other) + def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool: """Evaluate a marker. @@ -298,7 +241,12 @@ class Marker: The environment is determined from the current Python process. """ current_environment = default_environment() + current_environment["extra"] = "" if environment is not None: current_environment.update(environment) + # The API used to allow setting extra to None. We need to handle this + # case for backwards compatibility. + if current_environment["extra"] is None: + current_environment["extra"] = "" return _evaluate_markers(self._markers, current_environment) diff --git a/third_party/python/packaging/packaging/metadata.py b/third_party/python/packaging/packaging/metadata.py new file mode 100644 index 000000000000..e76a60c395eb --- /dev/null +++ b/third_party/python/packaging/packaging/metadata.py @@ -0,0 +1,408 @@ +import email.feedparser +import email.header +import email.message +import email.parser +import email.policy +import sys +import typing +from typing import Dict, List, Optional, Tuple, Union, cast + +if sys.version_info >= (3, 8): # pragma: no cover + from typing import TypedDict +else: # pragma: no cover + if typing.TYPE_CHECKING: + from typing_extensions import TypedDict + else: + try: + from typing_extensions import TypedDict + except ImportError: + + class TypedDict: + def __init_subclass__(*_args, **_kwargs): + pass + + +# The RawMetadata class attempts to make as few assumptions about the underlying +# serialization formats as possible. The idea is that as long as a serialization +# formats offer some very basic primitives in *some* way then we can support +# serializing to and from that format. +class RawMetadata(TypedDict, total=False): + """A dictionary of raw core metadata. + + Each field in core metadata maps to a key of this dictionary (when data is + provided). The key is lower-case and underscores are used instead of dashes + compared to the equivalent core metadata field. Any core metadata field that + can be specified multiple times or can hold multiple values in a single + field have a key with a plural name. + + Core metadata fields that can be specified multiple times are stored as a + list or dict depending on which is appropriate for the field. Any fields + which hold multiple values in a single field are stored as a list. + + """ + + # Metadata 1.0 - PEP 241 + metadata_version: str + name: str + version: str + platforms: List[str] + summary: str + description: str + keywords: List[str] + home_page: str + author: str + author_email: str + license: str + + # Metadata 1.1 - PEP 314 + supported_platforms: List[str] + download_url: str + classifiers: List[str] + requires: List[str] + provides: List[str] + obsoletes: List[str] + + # Metadata 1.2 - PEP 345 + maintainer: str + maintainer_email: str + requires_dist: List[str] + provides_dist: List[str] + obsoletes_dist: List[str] + requires_python: str + requires_external: List[str] + project_urls: Dict[str, str] + + # Metadata 2.0 + # PEP 426 attempted to completely revamp the metadata format + # but got stuck without ever being able to build consensus on + # it and ultimately ended up withdrawn. + # + # However, a number of tools had started emiting METADATA with + # `2.0` Metadata-Version, so for historical reasons, this version + # was skipped. + + # Metadata 2.1 - PEP 566 + description_content_type: str + provides_extra: List[str] + + # Metadata 2.2 - PEP 643 + dynamic: List[str] + + # Metadata 2.3 - PEP 685 + # No new fields were added in PEP 685, just some edge case were + # tightened up to provide better interoptability. + + +_STRING_FIELDS = { + "author", + "author_email", + "description", + "description_content_type", + "download_url", + "home_page", + "license", + "maintainer", + "maintainer_email", + "metadata_version", + "name", + "requires_python", + "summary", + "version", +} + +_LIST_STRING_FIELDS = { + "classifiers", + "dynamic", + "obsoletes", + "obsoletes_dist", + "platforms", + "provides", + "provides_dist", + "provides_extra", + "requires", + "requires_dist", + "requires_external", + "supported_platforms", +} + + +def _parse_keywords(data: str) -> List[str]: + """Split a string of comma-separate keyboards into a list of keywords.""" + return [k.strip() for k in data.split(",")] + + +def _parse_project_urls(data: List[str]) -> Dict[str, str]: + """Parse a list of label/URL string pairings separated by a comma.""" + urls = {} + for pair in data: + # Our logic is slightly tricky here as we want to try and do + # *something* reasonable with malformed data. + # + # The main thing that we have to worry about, is data that does + # not have a ',' at all to split the label from the Value. There + # isn't a singular right answer here, and we will fail validation + # later on (if the caller is validating) so it doesn't *really* + # matter, but since the missing value has to be an empty str + # and our return value is dict[str, str], if we let the key + # be the missing value, then they'd have multiple '' values that + # overwrite each other in a accumulating dict. + # + # The other potentional issue is that it's possible to have the + # same label multiple times in the metadata, with no solid "right" + # answer with what to do in that case. As such, we'll do the only + # thing we can, which is treat the field as unparseable and add it + # to our list of unparsed fields. + parts = [p.strip() for p in pair.split(",", 1)] + parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items + + # TODO: The spec doesn't say anything about if the keys should be + # considered case sensitive or not... logically they should + # be case-preserving and case-insensitive, but doing that + # would open up more cases where we might have duplicate + # entries. + label, url = parts + if label in urls: + # The label already exists in our set of urls, so this field + # is unparseable, and we can just add the whole thing to our + # unparseable data and stop processing it. + raise KeyError("duplicate labels in project urls") + urls[label] = url + + return urls + + +def _get_payload(msg: email.message.Message, source: Union[bytes, str]) -> str: + """Get the body of the message.""" + # If our source is a str, then our caller has managed encodings for us, + # and we don't need to deal with it. + if isinstance(source, str): + payload: str = msg.get_payload() + return payload + # If our source is a bytes, then we're managing the encoding and we need + # to deal with it. + else: + bpayload: bytes = msg.get_payload(decode=True) + try: + return bpayload.decode("utf8", "strict") + except UnicodeDecodeError: + raise ValueError("payload in an invalid encoding") + + +# The various parse_FORMAT functions here are intended to be as lenient as +# possible in their parsing, while still returning a correctly typed +# RawMetadata. +# +# To aid in this, we also generally want to do as little touching of the +# data as possible, except where there are possibly some historic holdovers +# that make valid data awkward to work with. +# +# While this is a lower level, intermediate format than our ``Metadata`` +# class, some light touch ups can make a massive difference in usability. + +# Map METADATA fields to RawMetadata. +_EMAIL_TO_RAW_MAPPING = { + "author": "author", + "author-email": "author_email", + "classifier": "classifiers", + "description": "description", + "description-content-type": "description_content_type", + "download-url": "download_url", + "dynamic": "dynamic", + "home-page": "home_page", + "keywords": "keywords", + "license": "license", + "maintainer": "maintainer", + "maintainer-email": "maintainer_email", + "metadata-version": "metadata_version", + "name": "name", + "obsoletes": "obsoletes", + "obsoletes-dist": "obsoletes_dist", + "platform": "platforms", + "project-url": "project_urls", + "provides": "provides", + "provides-dist": "provides_dist", + "provides-extra": "provides_extra", + "requires": "requires", + "requires-dist": "requires_dist", + "requires-external": "requires_external", + "requires-python": "requires_python", + "summary": "summary", + "supported-platform": "supported_platforms", + "version": "version", +} + + +def parse_email(data: Union[bytes, str]) -> Tuple[RawMetadata, Dict[str, List[str]]]: + """Parse a distribution's metadata. + + This function returns a two-item tuple of dicts. The first dict is of + recognized fields from the core metadata specification. Fields that can be + parsed and translated into Python's built-in types are converted + appropriately. All other fields are left as-is. Fields that are allowed to + appear multiple times are stored as lists. + + The second dict contains all other fields from the metadata. This includes + any unrecognized fields. It also includes any fields which are expected to + be parsed into a built-in type but were not formatted appropriately. Finally, + any fields that are expected to appear only once but are repeated are + included in this dict. + + """ + raw: Dict[str, Union[str, List[str], Dict[str, str]]] = {} + unparsed: Dict[str, List[str]] = {} + + if isinstance(data, str): + parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) + else: + parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) + + # We have to wrap parsed.keys() in a set, because in the case of multiple + # values for a key (a list), the key will appear multiple times in the + # list of keys, but we're avoiding that by using get_all(). + for name in frozenset(parsed.keys()): + # Header names in RFC are case insensitive, so we'll normalize to all + # lower case to make comparisons easier. + name = name.lower() + + # We use get_all() here, even for fields that aren't multiple use, + # because otherwise someone could have e.g. two Name fields, and we + # would just silently ignore it rather than doing something about it. + headers = parsed.get_all(name) + + # The way the email module works when parsing bytes is that it + # unconditionally decodes the bytes as ascii using the surrogateescape + # handler. When you pull that data back out (such as with get_all() ), + # it looks to see if the str has any surrogate escapes, and if it does + # it wraps it in a Header object instead of returning the string. + # + # As such, we'll look for those Header objects, and fix up the encoding. + value = [] + # Flag if we have run into any issues processing the headers, thus + # signalling that the data belongs in 'unparsed'. + valid_encoding = True + for h in headers: + # It's unclear if this can return more types than just a Header or + # a str, so we'll just assert here to make sure. + assert isinstance(h, (email.header.Header, str)) + + # If it's a header object, we need to do our little dance to get + # the real data out of it. In cases where there is invalid data + # we're going to end up with mojibake, but there's no obvious, good + # way around that without reimplementing parts of the Header object + # ourselves. + # + # That should be fine since, if mojibacked happens, this key is + # going into the unparsed dict anyways. + if isinstance(h, email.header.Header): + # The Header object stores it's data as chunks, and each chunk + # can be independently encoded, so we'll need to check each + # of them. + chunks: List[Tuple[bytes, Optional[str]]] = [] + for bin, encoding in email.header.decode_header(h): + try: + bin.decode("utf8", "strict") + except UnicodeDecodeError: + # Enable mojibake. + encoding = "latin1" + valid_encoding = False + else: + encoding = "utf8" + chunks.append((bin, encoding)) + + # Turn our chunks back into a Header object, then let that + # Header object do the right thing to turn them into a + # string for us. + value.append(str(email.header.make_header(chunks))) + # This is already a string, so just add it. + else: + value.append(h) + + # We've processed all of our values to get them into a list of str, + # but we may have mojibake data, in which case this is an unparsed + # field. + if not valid_encoding: + unparsed[name] = value + continue + + raw_name = _EMAIL_TO_RAW_MAPPING.get(name) + if raw_name is None: + # This is a bit of a weird situation, we've encountered a key that + # we don't know what it means, so we don't know whether it's meant + # to be a list or not. + # + # Since we can't really tell one way or another, we'll just leave it + # as a list, even though it may be a single item list, because that's + # what makes the most sense for email headers. + unparsed[name] = value + continue + + # If this is one of our string fields, then we'll check to see if our + # value is a list of a single item. If it is then we'll assume that + # it was emitted as a single string, and unwrap the str from inside + # the list. + # + # If it's any other kind of data, then we haven't the faintest clue + # what we should parse it as, and we have to just add it to our list + # of unparsed stuff. + if raw_name in _STRING_FIELDS and len(value) == 1: + raw[raw_name] = value[0] + # If this is one of our list of string fields, then we can just assign + # the value, since email *only* has strings, and our get_all() call + # above ensures that this is a list. + elif raw_name in _LIST_STRING_FIELDS: + raw[raw_name] = value + # Special Case: Keywords + # The keywords field is implemented in the metadata spec as a str, + # but it conceptually is a list of strings, and is serialized using + # ", ".join(keywords), so we'll do some light data massaging to turn + # this into what it logically is. + elif raw_name == "keywords" and len(value) == 1: + raw[raw_name] = _parse_keywords(value[0]) + # Special Case: Project-URL + # The project urls is implemented in the metadata spec as a list of + # specially-formatted strings that represent a key and a value, which + # is fundamentally a mapping, however the email format doesn't support + # mappings in a sane way, so it was crammed into a list of strings + # instead. + # + # We will do a little light data massaging to turn this into a map as + # it logically should be. + elif raw_name == "project_urls": + try: + raw[raw_name] = _parse_project_urls(value) + except KeyError: + unparsed[name] = value + # Nothing that we've done has managed to parse this, so it'll just + # throw it in our unparseable data and move on. + else: + unparsed[name] = value + + # We need to support getting the Description from the message payload in + # addition to getting it from the the headers. This does mean, though, there + # is the possibility of it being set both ways, in which case we put both + # in 'unparsed' since we don't know which is right. + try: + payload = _get_payload(parsed, data) + except ValueError: + unparsed.setdefault("description", []).append( + parsed.get_payload(decode=isinstance(data, bytes)) + ) + else: + if payload: + # Check to see if we've already got a description, if so then both + # it, and this body move to unparseable. + if "description" in raw: + description_header = cast(str, raw.pop("description")) + unparsed.setdefault("description", []).extend( + [description_header, payload] + ) + elif "description" in unparsed: + unparsed["description"].append(payload) + else: + raw["description"] = payload + + # We need to cast our `raw` to a metadata, because a TypedDict only support + # literal key names, but we're computing our key names on purpose, but the + # way this function is implemented, our `TypedDict` can only have valid key + # names. + return cast(RawMetadata, raw), unparsed diff --git a/third_party/python/packaging/packaging/requirements.py b/third_party/python/packaging/packaging/requirements.py index 53f9a3aa42b8..f34bfa85c802 100644 --- a/third_party/python/packaging/packaging/requirements.py +++ b/third_party/python/packaging/packaging/requirements.py @@ -2,26 +2,13 @@ # 2.0, and the BSD License. See the LICENSE file in the root of this repository # for complete details. -import re -import string import urllib.parse -from typing import List, Optional as TOptional, Set +from typing import Any, List, Optional, Set -from pyparsing import ( # noqa - Combine, - Literal as L, - Optional, - ParseException, - Regex, - Word, - ZeroOrMore, - originalTextFor, - stringEnd, - stringStart, -) - -from .markers import MARKER_EXPR, Marker -from .specifiers import LegacySpecifier, Specifier, SpecifierSet +from ._parser import parse_requirement as _parse_requirement +from ._tokenizer import ParserSyntaxError +from .markers import Marker, _normalize_extra_values +from .specifiers import SpecifierSet class InvalidRequirement(ValueError): @@ -30,60 +17,6 @@ class InvalidRequirement(ValueError): """ -ALPHANUM = Word(string.ascii_letters + string.digits) - -LBRACKET = L("[").suppress() -RBRACKET = L("]").suppress() -LPAREN = L("(").suppress() -RPAREN = L(")").suppress() -COMMA = L(",").suppress() -SEMICOLON = L(";").suppress() -AT = L("@").suppress() - -PUNCTUATION = Word("-_.") -IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM) -IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) - -NAME = IDENTIFIER("name") -EXTRA = IDENTIFIER - -URI = Regex(r"[^ ]+")("url") -URL = AT + URI - -EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) -EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") - -VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) -VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) - -VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY -VERSION_MANY = Combine( - VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False -)("_raw_spec") -_VERSION_SPEC = Optional((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY) -_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "") - -VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") -VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) - -MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") -MARKER_EXPR.setParseAction( - lambda s, l, t: Marker(s[t._original_start : t._original_end]) -) -MARKER_SEPARATOR = SEMICOLON -MARKER = MARKER_SEPARATOR + MARKER_EXPR - -VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) -URL_AND_MARKER = URL + Optional(MARKER) - -NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) - -REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd -# pyparsing isn't thread safe during initialization, so we do it eagerly, see -# issue #104 -REQUIREMENT.parseString("x[]") - - class Requirement: """Parse a requirement. @@ -99,28 +32,29 @@ class Requirement: def __init__(self, requirement_string: str) -> None: try: - req = REQUIREMENT.parseString(requirement_string) - except ParseException as e: - raise InvalidRequirement( - f'Parse error at "{ requirement_string[e.loc : e.loc + 8]!r}": {e.msg}' - ) + parsed = _parse_requirement(requirement_string) + except ParserSyntaxError as e: + raise InvalidRequirement(str(e)) from e - self.name: str = req.name - if req.url: - parsed_url = urllib.parse.urlparse(req.url) + self.name: str = parsed.name + if parsed.url: + parsed_url = urllib.parse.urlparse(parsed.url) if parsed_url.scheme == "file": - if urllib.parse.urlunparse(parsed_url) != req.url: + if urllib.parse.urlunparse(parsed_url) != parsed.url: raise InvalidRequirement("Invalid URL given") elif not (parsed_url.scheme and parsed_url.netloc) or ( not parsed_url.scheme and not parsed_url.netloc ): - raise InvalidRequirement(f"Invalid URL: {req.url}") - self.url: TOptional[str] = req.url + raise InvalidRequirement(f"Invalid URL: {parsed.url}") + self.url: Optional[str] = parsed.url else: self.url = None - self.extras: Set[str] = set(req.extras.asList() if req.extras else []) - self.specifier: SpecifierSet = SpecifierSet(req.specifier) - self.marker: TOptional[Marker] = req.marker if req.marker else None + self.extras: Set[str] = set(parsed.extras if parsed.extras else []) + self.specifier: SpecifierSet = SpecifierSet(parsed.specifier) + self.marker: Optional[Marker] = None + if parsed.marker is not None: + self.marker = Marker.__new__(Marker) + self.marker._markers = _normalize_extra_values(parsed.marker) def __str__(self) -> str: parts: List[str] = [self.name] @@ -144,3 +78,18 @@ class Requirement: def __repr__(self) -> str: return f"" + + def __hash__(self) -> int: + return hash((self.__class__.__name__, str(self))) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Requirement): + return NotImplemented + + return ( + self.name == other.name + and self.extras == other.extras + and self.specifier == other.specifier + and self.url == other.url + and self.marker == other.marker + ) diff --git a/third_party/python/packaging/packaging/specifiers.py b/third_party/python/packaging/packaging/specifiers.py index 0e218a6f9f75..ba8fe37b7f7f 100644 --- a/third_party/python/packaging/packaging/specifiers.py +++ b/third_party/python/packaging/packaging/specifiers.py @@ -1,20 +1,22 @@ # This file is dual licensed under the terms of the Apache License, Version # 2.0, and the BSD License. See the LICENSE file in the root of this repository # for complete details. +""" +.. testsetup:: + + from packaging.specifiers import Specifier, SpecifierSet, InvalidSpecifier + from packaging.version import Version +""" import abc -import functools import itertools import re -import warnings from typing import ( Callable, - Dict, Iterable, Iterator, List, Optional, - Pattern, Set, Tuple, TypeVar, @@ -22,17 +24,28 @@ from typing import ( ) from .utils import canonicalize_version -from .version import LegacyVersion, Version, parse +from .version import Version -ParsedVersion = Union[Version, LegacyVersion] -UnparsedVersion = Union[Version, LegacyVersion, str] -VersionTypeVar = TypeVar("VersionTypeVar", bound=UnparsedVersion) -CallableOperator = Callable[[ParsedVersion, str], bool] +UnparsedVersion = Union[Version, str] +UnparsedVersionVar = TypeVar("UnparsedVersionVar", bound=UnparsedVersion) +CallableOperator = Callable[[Version, str], bool] + + +def _coerce_version(version: UnparsedVersion) -> Version: + if not isinstance(version, Version): + version = Version(version) + return version class InvalidSpecifier(ValueError): """ - An invalid specifier was found, users should refer to PEP 440. + Raised when attempting to create a :class:`Specifier` with a specifier + string that is invalid. + + >>> Specifier("lolwat") + Traceback (most recent call last): + ... + packaging.specifiers.InvalidSpecifier: Invalid specifier: 'lolwat' """ @@ -40,35 +53,39 @@ class BaseSpecifier(metaclass=abc.ABCMeta): @abc.abstractmethod def __str__(self) -> str: """ - Returns the str representation of this Specifier like object. This + Returns the str representation of this Specifier-like object. This should be representative of the Specifier itself. """ @abc.abstractmethod def __hash__(self) -> int: """ - Returns a hash value for this Specifier like object. + Returns a hash value for this Specifier-like object. """ @abc.abstractmethod def __eq__(self, other: object) -> bool: """ - Returns a boolean representing whether or not the two Specifier like + Returns a boolean representing whether or not the two Specifier-like objects are equal. + + :param other: The other object to check against. """ - @abc.abstractproperty + @property + @abc.abstractmethod def prereleases(self) -> Optional[bool]: - """ - Returns whether or not pre-releases as a whole are allowed by this - specifier. + """Whether or not pre-releases as a whole are allowed. + + This can be set to either ``True`` or ``False`` to explicitly enable or disable + prereleases or it can be set to ``None`` (the default) to use default semantics. """ @prereleases.setter def prereleases(self, value: bool) -> None: - """ - Sets whether or not pre-releases as a whole are allowed by this - specifier. + """Setter for :attr:`prereleases`. + + :param value: The value to set. """ @abc.abstractmethod @@ -79,227 +96,28 @@ class BaseSpecifier(metaclass=abc.ABCMeta): @abc.abstractmethod def filter( - self, iterable: Iterable[VersionTypeVar], prereleases: Optional[bool] = None - ) -> Iterable[VersionTypeVar]: + self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None + ) -> Iterator[UnparsedVersionVar]: """ Takes an iterable of items and filters them so that only items which are contained within this specifier are allowed in it. """ -class _IndividualSpecifier(BaseSpecifier): +class Specifier(BaseSpecifier): + """This class abstracts handling of version specifiers. - _operators: Dict[str, str] = {} - _regex: Pattern[str] + .. tip:: - def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None: - match = self._regex.search(spec) - if not match: - raise InvalidSpecifier(f"Invalid specifier: '{spec}'") + It is generally not required to instantiate this manually. You should instead + prefer to work with :class:`SpecifierSet` instead, which can parse + comma-separated version specifiers (which is what package metadata contains). + """ - self._spec: Tuple[str, str] = ( - match.group("operator").strip(), - match.group("version").strip(), - ) - - # Store whether or not this Specifier should accept prereleases - self._prereleases = prereleases - - def __repr__(self) -> str: - pre = ( - f", prereleases={self.prereleases!r}" - if self._prereleases is not None - else "" - ) - - return f"<{self.__class__.__name__}({str(self)!r}{pre})>" - - def __str__(self) -> str: - return "{}{}".format(*self._spec) - - @property - def _canonical_spec(self) -> Tuple[str, str]: - return self._spec[0], canonicalize_version(self._spec[1]) - - def __hash__(self) -> int: - return hash(self._canonical_spec) - - def __eq__(self, other: object) -> bool: - if isinstance(other, str): - try: - other = self.__class__(str(other)) - except InvalidSpecifier: - return NotImplemented - elif not isinstance(other, self.__class__): - return NotImplemented - - return self._canonical_spec == other._canonical_spec - - def _get_operator(self, op: str) -> CallableOperator: - operator_callable: CallableOperator = getattr( - self, f"_compare_{self._operators[op]}" - ) - return operator_callable - - def _coerce_version(self, version: UnparsedVersion) -> ParsedVersion: - if not isinstance(version, (LegacyVersion, Version)): - version = parse(version) - return version - - @property - def operator(self) -> str: - return self._spec[0] - - @property - def version(self) -> str: - return self._spec[1] - - @property - def prereleases(self) -> Optional[bool]: - return self._prereleases - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - - def __contains__(self, item: str) -> bool: - return self.contains(item) - - def contains( - self, item: UnparsedVersion, prereleases: Optional[bool] = None - ) -> bool: - - # Determine if prereleases are to be allowed or not. - if prereleases is None: - prereleases = self.prereleases - - # Normalize item to a Version or LegacyVersion, this allows us to have - # a shortcut for ``"2.0" in Specifier(">=2") - normalized_item = self._coerce_version(item) - - # Determine if we should be supporting prereleases in this specifier - # or not, if we do not support prereleases than we can short circuit - # logic if this version is a prereleases. - if normalized_item.is_prerelease and not prereleases: - return False - - # Actually do the comparison to determine if this item is contained - # within this Specifier or not. - operator_callable: CallableOperator = self._get_operator(self.operator) - return operator_callable(normalized_item, self.version) - - def filter( - self, iterable: Iterable[VersionTypeVar], prereleases: Optional[bool] = None - ) -> Iterable[VersionTypeVar]: - - yielded = False - found_prereleases = [] - - kw = {"prereleases": prereleases if prereleases is not None else True} - - # Attempt to iterate over all the values in the iterable and if any of - # them match, yield them. - for version in iterable: - parsed_version = self._coerce_version(version) - - if self.contains(parsed_version, **kw): - # If our version is a prerelease, and we were not set to allow - # prereleases, then we'll store it for later in case nothing - # else matches this specifier. - if parsed_version.is_prerelease and not ( - prereleases or self.prereleases - ): - found_prereleases.append(version) - # Either this is not a prerelease, or we should have been - # accepting prereleases from the beginning. - else: - yielded = True - yield version - - # Now that we've iterated over everything, determine if we've yielded - # any values, and if we have not and we have any prereleases stored up - # then we will go ahead and yield the prereleases. - if not yielded and found_prereleases: - for version in found_prereleases: - yield version - - -class LegacySpecifier(_IndividualSpecifier): - - _regex_str = r""" - (?P(==|!=|<=|>=|<|>)) - \s* - (?P - [^,;\s)]* # Since this is a "legacy" specifier, and the version - # string can be just about anything, we match everything - # except for whitespace, a semi-colon for marker support, - # a closing paren since versions can be enclosed in - # them, and a comma since it's a version separator. - ) - """ - - _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) - - _operators = { - "==": "equal", - "!=": "not_equal", - "<=": "less_than_equal", - ">=": "greater_than_equal", - "<": "less_than", - ">": "greater_than", - } - - def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None: - super().__init__(spec, prereleases) - - warnings.warn( - "Creating a LegacyVersion has been deprecated and will be " - "removed in the next major release", - DeprecationWarning, - ) - - def _coerce_version(self, version: UnparsedVersion) -> LegacyVersion: - if not isinstance(version, LegacyVersion): - version = LegacyVersion(str(version)) - return version - - def _compare_equal(self, prospective: LegacyVersion, spec: str) -> bool: - return prospective == self._coerce_version(spec) - - def _compare_not_equal(self, prospective: LegacyVersion, spec: str) -> bool: - return prospective != self._coerce_version(spec) - - def _compare_less_than_equal(self, prospective: LegacyVersion, spec: str) -> bool: - return prospective <= self._coerce_version(spec) - - def _compare_greater_than_equal( - self, prospective: LegacyVersion, spec: str - ) -> bool: - return prospective >= self._coerce_version(spec) - - def _compare_less_than(self, prospective: LegacyVersion, spec: str) -> bool: - return prospective < self._coerce_version(spec) - - def _compare_greater_than(self, prospective: LegacyVersion, spec: str) -> bool: - return prospective > self._coerce_version(spec) - - -def _require_version_compare( - fn: Callable[["Specifier", ParsedVersion, str], bool] -) -> Callable[["Specifier", ParsedVersion, str], bool]: - @functools.wraps(fn) - def wrapped(self: "Specifier", prospective: ParsedVersion, spec: str) -> bool: - if not isinstance(prospective, Version): - return False - return fn(self, prospective, spec) - - return wrapped - - -class Specifier(_IndividualSpecifier): - - _regex_str = r""" + _operator_regex_str = r""" (?P(~=|==|!=|<=|>=|<|>|===)) + """ + _version_regex_str = r""" (?P (?: # The identity operators allow for an escape hatch that will @@ -309,8 +127,10 @@ class Specifier(_IndividualSpecifier): # but included entirely as an escape hatch. (?<====) # Only match for the identity operator \s* - [^\s]* # We just match everything, except for whitespace - # since we are only testing for strict identity. + [^\s;)]* # The arbitrary version can be just about anything, + # we match everything except for whitespace, a + # semi-colon for marker support, and a closing paren + # since versions can be enclosed in them. ) | (?: @@ -323,23 +143,23 @@ class Specifier(_IndividualSpecifier): v? (?:[0-9]+!)? # epoch [0-9]+(?:\.[0-9]+)* # release - (?: # pre release - [-_\.]? - (a|b|c|rc|alpha|beta|pre|preview) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - # You cannot use a wild card and a dev or local version - # together so group them with a | and make them optional. + # You cannot use a wild card and a pre-release, post-release, a dev or + # local version together so group them with a | and make them optional. (?: + \.\* # Wild card syntax of .* + | + (?: # pre release + [-_\.]? + (alpha|beta|preview|pre|a|b|c|rc) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local - | - \.\* # Wild card syntax of .* )? ) | @@ -354,7 +174,7 @@ class Specifier(_IndividualSpecifier): [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) (?: # pre release [-_\.]? - (a|b|c|rc|alpha|beta|pre|preview) + (alpha|beta|preview|pre|a|b|c|rc) [-_\.]? [0-9]* )? @@ -379,7 +199,7 @@ class Specifier(_IndividualSpecifier): [0-9]+(?:\.[0-9]+)* # release (?: # pre release [-_\.]? - (a|b|c|rc|alpha|beta|pre|preview) + (alpha|beta|preview|pre|a|b|c|rc) [-_\.]? [0-9]* )? @@ -391,7 +211,10 @@ class Specifier(_IndividualSpecifier): ) """ - _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile( + r"^\s*" + _operator_regex_str + _version_regex_str + r"\s*$", + re.VERBOSE | re.IGNORECASE, + ) _operators = { "~=": "compatible", @@ -404,8 +227,153 @@ class Specifier(_IndividualSpecifier): "===": "arbitrary", } - @_require_version_compare - def _compare_compatible(self, prospective: ParsedVersion, spec: str) -> bool: + def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None: + """Initialize a Specifier instance. + + :param spec: + The string representation of a specifier which will be parsed and + normalized before use. + :param prereleases: + This tells the specifier if it should accept prerelease versions if + applicable or not. The default of ``None`` will autodetect it from the + given specifiers. + :raises InvalidSpecifier: + If the given specifier is invalid (i.e. bad syntax). + """ + match = self._regex.search(spec) + if not match: + raise InvalidSpecifier(f"Invalid specifier: '{spec}'") + + self._spec: Tuple[str, str] = ( + match.group("operator").strip(), + match.group("version").strip(), + ) + + # Store whether or not this Specifier should accept prereleases + self._prereleases = prereleases + + # https://github.com/python/mypy/pull/13475#pullrequestreview-1079784515 + @property # type: ignore[override] + def prereleases(self) -> bool: + # If there is an explicit prereleases set for this, then we'll just + # blindly use that. + if self._prereleases is not None: + return self._prereleases + + # Look at all of our specifiers and determine if they are inclusive + # operators, and if they are if they are including an explicit + # prerelease. + operator, version = self._spec + if operator in ["==", ">=", "<=", "~=", "==="]: + # The == specifier can include a trailing .*, if it does we + # want to remove before parsing. + if operator == "==" and version.endswith(".*"): + version = version[:-2] + + # Parse the version, and if it is a pre-release than this + # specifier allows pre-releases. + if Version(version).is_prerelease: + return True + + return False + + @prereleases.setter + def prereleases(self, value: bool) -> None: + self._prereleases = value + + @property + def operator(self) -> str: + """The operator of this specifier. + + >>> Specifier("==1.2.3").operator + '==' + """ + return self._spec[0] + + @property + def version(self) -> str: + """The version of this specifier. + + >>> Specifier("==1.2.3").version + '1.2.3' + """ + return self._spec[1] + + def __repr__(self) -> str: + """A representation of the Specifier that shows all internal state. + + >>> Specifier('>=1.0.0') + =1.0.0')> + >>> Specifier('>=1.0.0', prereleases=False) + =1.0.0', prereleases=False)> + >>> Specifier('>=1.0.0', prereleases=True) + =1.0.0', prereleases=True)> + """ + pre = ( + f", prereleases={self.prereleases!r}" + if self._prereleases is not None + else "" + ) + + return f"<{self.__class__.__name__}({str(self)!r}{pre})>" + + def __str__(self) -> str: + """A string representation of the Specifier that can be round-tripped. + + >>> str(Specifier('>=1.0.0')) + '>=1.0.0' + >>> str(Specifier('>=1.0.0', prereleases=False)) + '>=1.0.0' + """ + return "{}{}".format(*self._spec) + + @property + def _canonical_spec(self) -> Tuple[str, str]: + canonical_version = canonicalize_version( + self._spec[1], + strip_trailing_zero=(self._spec[0] != "~="), + ) + return self._spec[0], canonical_version + + def __hash__(self) -> int: + return hash(self._canonical_spec) + + def __eq__(self, other: object) -> bool: + """Whether or not the two Specifier-like objects are equal. + + :param other: The other object to check against. + + The value of :attr:`prereleases` is ignored. + + >>> Specifier("==1.2.3") == Specifier("== 1.2.3.0") + True + >>> (Specifier("==1.2.3", prereleases=False) == + ... Specifier("==1.2.3", prereleases=True)) + True + >>> Specifier("==1.2.3") == "==1.2.3" + True + >>> Specifier("==1.2.3") == Specifier("==1.2.4") + False + >>> Specifier("==1.2.3") == Specifier("~=1.2.3") + False + """ + if isinstance(other, str): + try: + other = self.__class__(str(other)) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._canonical_spec == other._canonical_spec + + def _get_operator(self, op: str) -> CallableOperator: + operator_callable: CallableOperator = getattr( + self, f"_compare_{self._operators[op]}" + ) + return operator_callable + + def _compare_compatible(self, prospective: Version, spec: str) -> bool: # Compatible releases have an equivalent combination of >= and ==. That # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to @@ -426,34 +394,35 @@ class Specifier(_IndividualSpecifier): prospective, prefix ) - @_require_version_compare - def _compare_equal(self, prospective: ParsedVersion, spec: str) -> bool: + def _compare_equal(self, prospective: Version, spec: str) -> bool: # We need special logic to handle prefix matching if spec.endswith(".*"): # In the case of prefix matching we want to ignore local segment. - prospective = Version(prospective.public) + normalized_prospective = canonicalize_version( + prospective.public, strip_trailing_zero=False + ) + # Get the normalized version string ignoring the trailing .* + normalized_spec = canonicalize_version(spec[:-2], strip_trailing_zero=False) # Split the spec out by dots, and pretend that there is an implicit # dot in between a release segment and a pre-release segment. - split_spec = _version_split(spec[:-2]) # Remove the trailing .* + split_spec = _version_split(normalized_spec) # Split the prospective version out by dots, and pretend that there # is an implicit dot in between a release segment and a pre-release # segment. - split_prospective = _version_split(str(prospective)) + split_prospective = _version_split(normalized_prospective) + + # 0-pad the prospective version before shortening it to get the correct + # shortened version. + padded_prospective, _ = _pad_version(split_prospective, split_spec) # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the # prospective version or not. - shortened_prospective = split_prospective[: len(split_spec)] + shortened_prospective = padded_prospective[: len(split_spec)] - # Pad out our two sides with zeros so that they both equal the same - # length. - padded_spec, padded_prospective = _pad_version( - split_spec, shortened_prospective - ) - - return padded_prospective == padded_spec + return shortened_prospective == split_spec else: # Convert our spec string into a Version spec_version = Version(spec) @@ -466,30 +435,24 @@ class Specifier(_IndividualSpecifier): return prospective == spec_version - @_require_version_compare - def _compare_not_equal(self, prospective: ParsedVersion, spec: str) -> bool: + def _compare_not_equal(self, prospective: Version, spec: str) -> bool: return not self._compare_equal(prospective, spec) - @_require_version_compare - def _compare_less_than_equal(self, prospective: ParsedVersion, spec: str) -> bool: + def _compare_less_than_equal(self, prospective: Version, spec: str) -> bool: # NB: Local version identifiers are NOT permitted in the version # specifier, so local version labels can be universally removed from # the prospective version. return Version(prospective.public) <= Version(spec) - @_require_version_compare - def _compare_greater_than_equal( - self, prospective: ParsedVersion, spec: str - ) -> bool: + def _compare_greater_than_equal(self, prospective: Version, spec: str) -> bool: # NB: Local version identifiers are NOT permitted in the version # specifier, so local version labels can be universally removed from # the prospective version. return Version(prospective.public) >= Version(spec) - @_require_version_compare - def _compare_less_than(self, prospective: ParsedVersion, spec_str: str) -> bool: + def _compare_less_than(self, prospective: Version, spec_str: str) -> bool: # Convert our spec to a Version instance, since we'll want to work with # it as a version. @@ -514,8 +477,7 @@ class Specifier(_IndividualSpecifier): # version in the spec. return True - @_require_version_compare - def _compare_greater_than(self, prospective: ParsedVersion, spec_str: str) -> bool: + def _compare_greater_than(self, prospective: Version, spec_str: str) -> bool: # Convert our spec to a Version instance, since we'll want to work with # it as a version. @@ -549,34 +511,133 @@ class Specifier(_IndividualSpecifier): def _compare_arbitrary(self, prospective: Version, spec: str) -> bool: return str(prospective).lower() == str(spec).lower() - @property - def prereleases(self) -> bool: + def __contains__(self, item: Union[str, Version]) -> bool: + """Return whether or not the item is contained in this specifier. - # If there is an explicit prereleases set for this, then we'll just - # blindly use that. - if self._prereleases is not None: - return self._prereleases + :param item: The item to check for. - # Look at all of our specifiers and determine if they are inclusive - # operators, and if they are if they are including an explicit - # prerelease. - operator, version = self._spec - if operator in ["==", ">=", "<=", "~=", "==="]: - # The == specifier can include a trailing .*, if it does we - # want to remove before parsing. - if operator == "==" and version.endswith(".*"): - version = version[:-2] + This is used for the ``in`` operator and behaves the same as + :meth:`contains` with no ``prereleases`` argument passed. - # Parse the version, and if it is a pre-release than this - # specifier allows pre-releases. - if parse(version).is_prerelease: - return True + >>> "1.2.3" in Specifier(">=1.2.3") + True + >>> Version("1.2.3") in Specifier(">=1.2.3") + True + >>> "1.0.0" in Specifier(">=1.2.3") + False + >>> "1.3.0a1" in Specifier(">=1.2.3") + False + >>> "1.3.0a1" in Specifier(">=1.2.3", prereleases=True) + True + """ + return self.contains(item) - return False + def contains( + self, item: UnparsedVersion, prereleases: Optional[bool] = None + ) -> bool: + """Return whether or not the item is contained in this specifier. - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value + :param item: + The item to check for, which can be a version string or a + :class:`Version` instance. + :param prereleases: + Whether or not to match prereleases with this Specifier. If set to + ``None`` (the default), it uses :attr:`prereleases` to determine + whether or not prereleases are allowed. + + >>> Specifier(">=1.2.3").contains("1.2.3") + True + >>> Specifier(">=1.2.3").contains(Version("1.2.3")) + True + >>> Specifier(">=1.2.3").contains("1.0.0") + False + >>> Specifier(">=1.2.3").contains("1.3.0a1") + False + >>> Specifier(">=1.2.3", prereleases=True).contains("1.3.0a1") + True + >>> Specifier(">=1.2.3").contains("1.3.0a1", prereleases=True) + True + """ + + # Determine if prereleases are to be allowed or not. + if prereleases is None: + prereleases = self.prereleases + + # Normalize item to a Version, this allows us to have a shortcut for + # "2.0" in Specifier(">=2") + normalized_item = _coerce_version(item) + + # Determine if we should be supporting prereleases in this specifier + # or not, if we do not support prereleases than we can short circuit + # logic if this version is a prereleases. + if normalized_item.is_prerelease and not prereleases: + return False + + # Actually do the comparison to determine if this item is contained + # within this Specifier or not. + operator_callable: CallableOperator = self._get_operator(self.operator) + return operator_callable(normalized_item, self.version) + + def filter( + self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None + ) -> Iterator[UnparsedVersionVar]: + """Filter items in the given iterable, that match the specifier. + + :param iterable: + An iterable that can contain version strings and :class:`Version` instances. + The items in the iterable will be filtered according to the specifier. + :param prereleases: + Whether or not to allow prereleases in the returned iterator. If set to + ``None`` (the default), it will be intelligently decide whether to allow + prereleases or not (based on the :attr:`prereleases` attribute, and + whether the only versions matching are prereleases). + + This method is smarter than just ``filter(Specifier().contains, [...])`` + because it implements the rule from :pep:`440` that a prerelease item + SHOULD be accepted if no other versions match the given specifier. + + >>> list(Specifier(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) + ['1.3'] + >>> list(Specifier(">=1.2.3").filter(["1.2", "1.2.3", "1.3", Version("1.4")])) + ['1.2.3', '1.3', ] + >>> list(Specifier(">=1.2.3").filter(["1.2", "1.5a1"])) + ['1.5a1'] + >>> list(Specifier(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) + ['1.3', '1.5a1'] + >>> list(Specifier(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) + ['1.3', '1.5a1'] + """ + + yielded = False + found_prereleases = [] + + kw = {"prereleases": prereleases if prereleases is not None else True} + + # Attempt to iterate over all the values in the iterable and if any of + # them match, yield them. + for version in iterable: + parsed_version = _coerce_version(version) + + if self.contains(parsed_version, **kw): + # If our version is a prerelease, and we were not set to allow + # prereleases, then we'll store it for later in case nothing + # else matches this specifier. + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): + found_prereleases.append(version) + # Either this is not a prerelease, or we should have been + # accepting prereleases from the beginning. + else: + yielded = True + yield version + + # Now that we've iterated over everything, determine if we've yielded + # any values, and if we have not and we have any prereleases stored up + # then we will go ahead and yield the prereleases. + if not yielded and found_prereleases: + for version in found_prereleases: + yield version _prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") @@ -618,22 +679,39 @@ def _pad_version(left: List[str], right: List[str]) -> Tuple[List[str], List[str class SpecifierSet(BaseSpecifier): + """This class abstracts handling of a set of version specifiers. + + It can be passed a single specifier (``>=3.0``), a comma-separated list of + specifiers (``>=3.0,!=3.1``), or no specifier at all. + """ + def __init__( self, specifiers: str = "", prereleases: Optional[bool] = None ) -> None: + """Initialize a SpecifierSet instance. - # Split on , to break each individual specifier into it's own item, and + :param specifiers: + The string representation of a specifier or a comma-separated list of + specifiers which will be parsed and normalized before use. + :param prereleases: + This tells the SpecifierSet if it should accept prerelease versions if + applicable or not. The default of ``None`` will autodetect it from the + given specifiers. + + :raises InvalidSpecifier: + If the given ``specifiers`` are not parseable than this exception will be + raised. + """ + + # Split on `,` to break each individual specifier into it's own item, and # strip each item to remove leading/trailing whitespace. split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] # Parsed each individual specifier, attempting first to make it a - # Specifier and falling back to a LegacySpecifier. - parsed: Set[_IndividualSpecifier] = set() + # Specifier. + parsed: Set[Specifier] = set() for specifier in split_specifiers: - try: - parsed.add(Specifier(specifier)) - except InvalidSpecifier: - parsed.add(LegacySpecifier(specifier)) + parsed.add(Specifier(specifier)) # Turn our parsed specifiers into a frozen set and save them for later. self._specs = frozenset(parsed) @@ -642,7 +720,40 @@ class SpecifierSet(BaseSpecifier): # we accept prereleases or not. self._prereleases = prereleases + @property + def prereleases(self) -> Optional[bool]: + # If we have been given an explicit prerelease modifier, then we'll + # pass that through here. + if self._prereleases is not None: + return self._prereleases + + # If we don't have any specifiers, and we don't have a forced value, + # then we'll just return None since we don't know if this should have + # pre-releases or not. + if not self._specs: + return None + + # Otherwise we'll see if any of the given specifiers accept + # prereleases, if any of them do we'll return True, otherwise False. + return any(s.prereleases for s in self._specs) + + @prereleases.setter + def prereleases(self, value: bool) -> None: + self._prereleases = value + def __repr__(self) -> str: + """A representation of the specifier set that shows all internal state. + + Note that the ordering of the individual specifiers within the set may not + match the input string. + + >>> SpecifierSet('>=1.0.0,!=2.0.0') + =1.0.0')> + >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=False) + =1.0.0', prereleases=False)> + >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=True) + =1.0.0', prereleases=True)> + """ pre = ( f", prereleases={self.prereleases!r}" if self._prereleases is not None @@ -652,12 +763,31 @@ class SpecifierSet(BaseSpecifier): return f"" def __str__(self) -> str: + """A string representation of the specifier set that can be round-tripped. + + Note that the ordering of the individual specifiers within the set may not + match the input string. + + >>> str(SpecifierSet(">=1.0.0,!=1.0.1")) + '!=1.0.1,>=1.0.0' + >>> str(SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False)) + '!=1.0.1,>=1.0.0' + """ return ",".join(sorted(str(s) for s in self._specs)) def __hash__(self) -> int: return hash(self._specs) def __and__(self, other: Union["SpecifierSet", str]) -> "SpecifierSet": + """Return a SpecifierSet which is a combination of the two sets. + + :param other: The other object to combine with. + + >>> SpecifierSet(">=1.0.0,!=1.0.1") & '<=2.0.0,!=2.0.1' + =1.0.0')> + >>> SpecifierSet(">=1.0.0,!=1.0.1") & SpecifierSet('<=2.0.0,!=2.0.1') + =1.0.0')> + """ if isinstance(other, str): other = SpecifierSet(other) elif not isinstance(other, SpecifierSet): @@ -681,7 +811,25 @@ class SpecifierSet(BaseSpecifier): return specifier def __eq__(self, other: object) -> bool: - if isinstance(other, (str, _IndividualSpecifier)): + """Whether or not the two SpecifierSet-like objects are equal. + + :param other: The other object to check against. + + The value of :attr:`prereleases` is ignored. + + >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.1") + True + >>> (SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False) == + ... SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True)) + True + >>> SpecifierSet(">=1.0.0,!=1.0.1") == ">=1.0.0,!=1.0.1" + True + >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0") + False + >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.2") + False + """ + if isinstance(other, (str, Specifier)): other = SpecifierSet(str(other)) elif not isinstance(other, SpecifierSet): return NotImplemented @@ -689,43 +837,72 @@ class SpecifierSet(BaseSpecifier): return self._specs == other._specs def __len__(self) -> int: + """Returns the number of specifiers in this specifier set.""" return len(self._specs) - def __iter__(self) -> Iterator[_IndividualSpecifier]: + def __iter__(self) -> Iterator[Specifier]: + """ + Returns an iterator over all the underlying :class:`Specifier` instances + in this specifier set. + + >>> sorted(SpecifierSet(">=1.0.0,!=1.0.1"), key=str) + [, =1.0.0')>] + """ return iter(self._specs) - @property - def prereleases(self) -> Optional[bool]: - - # If we have been given an explicit prerelease modifier, then we'll - # pass that through here. - if self._prereleases is not None: - return self._prereleases - - # If we don't have any specifiers, and we don't have a forced value, - # then we'll just return None since we don't know if this should have - # pre-releases or not. - if not self._specs: - return None - - # Otherwise we'll see if any of the given specifiers accept - # prereleases, if any of them do we'll return True, otherwise False. - return any(s.prereleases for s in self._specs) - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - def __contains__(self, item: UnparsedVersion) -> bool: + """Return whether or not the item is contained in this specifier. + + :param item: The item to check for. + + This is used for the ``in`` operator and behaves the same as + :meth:`contains` with no ``prereleases`` argument passed. + + >>> "1.2.3" in SpecifierSet(">=1.0.0,!=1.0.1") + True + >>> Version("1.2.3") in SpecifierSet(">=1.0.0,!=1.0.1") + True + >>> "1.0.1" in SpecifierSet(">=1.0.0,!=1.0.1") + False + >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1") + False + >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True) + True + """ return self.contains(item) def contains( - self, item: UnparsedVersion, prereleases: Optional[bool] = None + self, + item: UnparsedVersion, + prereleases: Optional[bool] = None, + installed: Optional[bool] = None, ) -> bool: + """Return whether or not the item is contained in this SpecifierSet. - # Ensure that our item is a Version or LegacyVersion instance. - if not isinstance(item, (LegacyVersion, Version)): - item = parse(item) + :param item: + The item to check for, which can be a version string or a + :class:`Version` instance. + :param prereleases: + Whether or not to match prereleases with this SpecifierSet. If set to + ``None`` (the default), it uses :attr:`prereleases` to determine + whether or not prereleases are allowed. + + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.2.3") + True + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains(Version("1.2.3")) + True + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.0.1") + False + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1") + False + >>> SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True).contains("1.3.0a1") + True + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1", prereleases=True) + True + """ + # Ensure that our item is a Version instance. + if not isinstance(item, Version): + item = Version(item) # Determine if we're forcing a prerelease or not, if we're not forcing # one for this particular filter call, then we'll use whatever the @@ -742,6 +919,9 @@ class SpecifierSet(BaseSpecifier): if not prereleases and item.is_prerelease: return False + if installed and item.is_prerelease: + item = Version(item.base_version) + # We simply dispatch to the underlying specs here to make sure that the # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers @@ -749,9 +929,46 @@ class SpecifierSet(BaseSpecifier): return all(s.contains(item, prereleases=prereleases) for s in self._specs) def filter( - self, iterable: Iterable[VersionTypeVar], prereleases: Optional[bool] = None - ) -> Iterable[VersionTypeVar]: + self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None + ) -> Iterator[UnparsedVersionVar]: + """Filter items in the given iterable, that match the specifiers in this set. + :param iterable: + An iterable that can contain version strings and :class:`Version` instances. + The items in the iterable will be filtered according to the specifier. + :param prereleases: + Whether or not to allow prereleases in the returned iterator. If set to + ``None`` (the default), it will be intelligently decide whether to allow + prereleases or not (based on the :attr:`prereleases` attribute, and + whether the only versions matching are prereleases). + + This method is smarter than just ``filter(SpecifierSet(...).contains, [...])`` + because it implements the rule from :pep:`440` that a prerelease item + SHOULD be accepted if no other versions match the given specifier. + + >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) + ['1.3'] + >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", Version("1.4")])) + ['1.3', ] + >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.5a1"])) + [] + >>> list(SpecifierSet(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) + ['1.3', '1.5a1'] + >>> list(SpecifierSet(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) + ['1.3', '1.5a1'] + + An "empty" SpecifierSet will filter items based on the presence of prerelease + versions in the set. + + >>> list(SpecifierSet("").filter(["1.3", "1.5a1"])) + ['1.3'] + >>> list(SpecifierSet("").filter(["1.5a1"])) + ['1.5a1'] + >>> list(SpecifierSet("", prereleases=True).filter(["1.3", "1.5a1"])) + ['1.3', '1.5a1'] + >>> list(SpecifierSet("").filter(["1.3", "1.5a1"], prereleases=True)) + ['1.3', '1.5a1'] + """ # Determine if we're forcing a prerelease or not, if we're not forcing # one for this particular filter call, then we'll use whatever the # SpecifierSet thinks for whether or not we should support prereleases. @@ -764,27 +981,16 @@ class SpecifierSet(BaseSpecifier): if self._specs: for spec in self._specs: iterable = spec.filter(iterable, prereleases=bool(prereleases)) - return iterable + return iter(iterable) # If we do not have any specifiers, then we need to have a rough filter # which will filter out any pre-releases, unless there are no final - # releases, and which will filter out LegacyVersion in general. + # releases. else: - filtered: List[VersionTypeVar] = [] - found_prereleases: List[VersionTypeVar] = [] - - item: UnparsedVersion - parsed_version: Union[Version, LegacyVersion] + filtered: List[UnparsedVersionVar] = [] + found_prereleases: List[UnparsedVersionVar] = [] for item in iterable: - # Ensure that we some kind of Version class for this item. - if not isinstance(item, (LegacyVersion, Version)): - parsed_version = parse(item) - else: - parsed_version = item - - # Filter out any item which is parsed as a LegacyVersion - if isinstance(parsed_version, LegacyVersion): - continue + parsed_version = _coerce_version(item) # Store any item which is a pre-release for later unless we've # already found a final version or we are accepting prereleases @@ -797,6 +1003,6 @@ class SpecifierSet(BaseSpecifier): # If we've found no items except for pre-releases, then we'll go # ahead and use the pre-releases if not filtered and found_prereleases and prereleases is None: - return found_prereleases + return iter(found_prereleases) - return filtered + return iter(filtered) diff --git a/third_party/python/packaging/packaging/tags.py b/third_party/python/packaging/packaging/tags.py index 9a3d25a71c75..76d243414d00 100644 --- a/third_party/python/packaging/packaging/tags.py +++ b/third_party/python/packaging/packaging/tags.py @@ -4,6 +4,7 @@ import logging import platform +import subprocess import sys import sysconfig from importlib.machinery import EXTENSION_SUFFIXES @@ -36,7 +37,7 @@ INTERPRETER_SHORT_NAMES: Dict[str, str] = { } -_32_BIT_INTERPRETER = sys.maxsize <= 2 ** 32 +_32_BIT_INTERPRETER = sys.maxsize <= 2**32 class Tag: @@ -110,7 +111,7 @@ def parse_tag(tag: str) -> FrozenSet[Tag]: def _get_config_var(name: str, warn: bool = False) -> Union[int, str, None]: - value = sysconfig.get_config_var(name) + value: Union[int, str, None] = sysconfig.get_config_var(name) if value is None and warn: logger.debug( "Config variable '%s' is unset, Python ABI tag may be incorrect", name @@ -119,7 +120,7 @@ def _get_config_var(name: str, warn: bool = False) -> Union[int, str, None]: def _normalize_string(string: str) -> str: - return string.replace(".", "_").replace("-", "_") + return string.replace(".", "_").replace("-", "_").replace(" ", "_") def _abi3_applies(python_version: PythonVersion) -> bool: @@ -224,10 +225,45 @@ def cpython_tags( yield Tag(interpreter, "abi3", platform_) -def _generic_abi() -> Iterator[str]: - abi = sysconfig.get_config_var("SOABI") - if abi: - yield _normalize_string(abi) +def _generic_abi() -> List[str]: + """ + Return the ABI tag based on EXT_SUFFIX. + """ + # The following are examples of `EXT_SUFFIX`. + # We want to keep the parts which are related to the ABI and remove the + # parts which are related to the platform: + # - linux: '.cpython-310-x86_64-linux-gnu.so' => cp310 + # - mac: '.cpython-310-darwin.so' => cp310 + # - win: '.cp310-win_amd64.pyd' => cp310 + # - win: '.pyd' => cp37 (uses _cpython_abis()) + # - pypy: '.pypy38-pp73-x86_64-linux-gnu.so' => pypy38_pp73 + # - graalpy: '.graalpy-38-native-x86_64-darwin.dylib' + # => graalpy_38_native + + ext_suffix = _get_config_var("EXT_SUFFIX", warn=True) + if not isinstance(ext_suffix, str) or ext_suffix[0] != ".": + raise SystemError("invalid sysconfig.get_config_var('EXT_SUFFIX')") + parts = ext_suffix.split(".") + if len(parts) < 3: + # CPython3.7 and earlier uses ".pyd" on Windows. + return _cpython_abis(sys.version_info[:2]) + soabi = parts[1] + if soabi.startswith("cpython"): + # non-windows + abi = "cp" + soabi.split("-")[1] + elif soabi.startswith("cp"): + # windows + abi = soabi.split("-")[0] + elif soabi.startswith("pypy"): + abi = "-".join(soabi.split("-")[:2]) + elif soabi.startswith("graalpy"): + abi = "-".join(soabi.split("-")[:3]) + elif soabi: + # pyston, ironpython, others? + abi = soabi + else: + return [] + return [_normalize_string(abi)] def generic_tags( @@ -251,8 +287,9 @@ def generic_tags( interpreter = "".join([interp_name, interp_version]) if abis is None: abis = _generic_abi() + else: + abis = list(abis) platforms = list(platforms or platform_tags()) - abis = list(abis) if "none" not in abis: abis.append("none") for abi in abis: @@ -356,6 +393,22 @@ def mac_platforms( version_str, _, cpu_arch = platform.mac_ver() if version is None: version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) + if version == (10, 16): + # When built against an older macOS SDK, Python will report macOS 10.16 + # instead of the real version. + version_str = subprocess.run( + [ + sys.executable, + "-sS", + "-c", + "import platform; print(platform.mac_ver()[0])", + ], + check=True, + env={"SYSTEM_VERSION_COMPAT": "0"}, + stdout=subprocess.PIPE, + universal_newlines=True, + ).stdout + version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) else: version = version if arch is None: @@ -446,6 +499,9 @@ def platform_tags() -> Iterator[str]: def interpreter_name() -> str: """ Returns the name of the running interpreter. + + Some implementations have a reserved, two-letter abbreviation which will + be returned when appropriate. """ name = sys.implementation.name return INTERPRETER_SHORT_NAMES.get(name) or name @@ -482,6 +538,9 @@ def sys_tags(*, warn: bool = False) -> Iterator[Tag]: yield from generic_tags() if interp_name == "pp": - yield from compatible_tags(interpreter="pp3") + interp = "pp3" + elif interp_name == "cp": + interp = "cp" + interpreter_version(warn=warn) else: - yield from compatible_tags() + interp = None + yield from compatible_tags(interpreter=interp) diff --git a/third_party/python/packaging/packaging/utils.py b/third_party/python/packaging/packaging/utils.py index bab11b80c60f..33c613b749a4 100644 --- a/third_party/python/packaging/packaging/utils.py +++ b/third_party/python/packaging/packaging/utils.py @@ -35,7 +35,9 @@ def canonicalize_name(name: str) -> NormalizedName: return cast(NormalizedName, value) -def canonicalize_version(version: Union[Version, str]) -> str: +def canonicalize_version( + version: Union[Version, str], *, strip_trailing_zero: bool = True +) -> str: """ This is very similar to Version.__str__, but has one subtle difference with the way it handles the release segment. @@ -56,8 +58,11 @@ def canonicalize_version(version: Union[Version, str]) -> str: parts.append(f"{parsed.epoch}!") # Release segment - # NB: This strips trailing '.0's to normalize - parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in parsed.release))) + release_segment = ".".join(str(x) for x in parsed.release) + if strip_trailing_zero: + # NB: This strips trailing '.0's to normalize + release_segment = re.sub(r"(\.0)+$", "", release_segment) + parts.append(release_segment) # Pre-release if parsed.pre is not None: diff --git a/third_party/python/packaging/packaging/version.py b/third_party/python/packaging/packaging/version.py index de9a09a4ed3b..b30e8cbf84f2 100644 --- a/third_party/python/packaging/packaging/version.py +++ b/third_party/python/packaging/packaging/version.py @@ -1,16 +1,20 @@ # This file is dual licensed under the terms of the Apache License, Version # 2.0, and the BSD License. See the LICENSE file in the root of this repository # for complete details. +""" +.. testsetup:: + + from packaging.version import parse, Version +""" import collections import itertools import re -import warnings -from typing import Callable, Iterator, List, Optional, SupportsInt, Tuple, Union +from typing import Any, Callable, Optional, SupportsInt, Tuple, Union from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType -__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] +__all__ = ["VERSION_PATTERN", "parse", "Version", "InvalidVersion"] InfiniteTypes = Union[InfinityType, NegativeInfinityType] PrePostDevType = Union[InfiniteTypes, Tuple[str, int]] @@ -29,36 +33,37 @@ LocalType = Union[ CmpKey = Tuple[ int, Tuple[int, ...], PrePostDevType, PrePostDevType, PrePostDevType, LocalType ] -LegacyCmpKey = Tuple[int, Tuple[str, ...]] -VersionComparisonMethod = Callable[ - [Union[CmpKey, LegacyCmpKey], Union[CmpKey, LegacyCmpKey]], bool -] +VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] _Version = collections.namedtuple( "_Version", ["epoch", "release", "dev", "pre", "post", "local"] ) -def parse(version: str) -> Union["LegacyVersion", "Version"]: +def parse(version: str) -> "Version": + """Parse the given version string. + + >>> parse('1.0.dev1') + + + :param version: The version string to parse. + :raises InvalidVersion: When the version string is not a valid version. """ - Parse the given version string and return either a :class:`Version` object - or a :class:`LegacyVersion` object depending on if the given version is - a valid PEP 440 version or a legacy version. - """ - try: - return Version(version) - except InvalidVersion: - return LegacyVersion(version) + return Version(version) class InvalidVersion(ValueError): - """ - An invalid version was found, users should refer to PEP 440. + """Raised when a version string is not a valid version. + + >>> Version("invalid") + Traceback (most recent call last): + ... + packaging.version.InvalidVersion: Invalid version: 'invalid' """ class _BaseVersion: - _key: Union[CmpKey, LegacyCmpKey] + _key: Tuple[Any, ...] def __hash__(self) -> int: return hash(self._key) @@ -103,126 +108,9 @@ class _BaseVersion: return self._key != other._key -class LegacyVersion(_BaseVersion): - def __init__(self, version: str) -> None: - self._version = str(version) - self._key = _legacy_cmpkey(self._version) - - warnings.warn( - "Creating a LegacyVersion has been deprecated and will be " - "removed in the next major release", - DeprecationWarning, - ) - - def __str__(self) -> str: - return self._version - - def __repr__(self) -> str: - return f"" - - @property - def public(self) -> str: - return self._version - - @property - def base_version(self) -> str: - return self._version - - @property - def epoch(self) -> int: - return -1 - - @property - def release(self) -> None: - return None - - @property - def pre(self) -> None: - return None - - @property - def post(self) -> None: - return None - - @property - def dev(self) -> None: - return None - - @property - def local(self) -> None: - return None - - @property - def is_prerelease(self) -> bool: - return False - - @property - def is_postrelease(self) -> bool: - return False - - @property - def is_devrelease(self) -> bool: - return False - - -_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) - -_legacy_version_replacement_map = { - "pre": "c", - "preview": "c", - "-": "final-", - "rc": "c", - "dev": "@", -} - - -def _parse_version_parts(s: str) -> Iterator[str]: - for part in _legacy_version_component_re.split(s): - part = _legacy_version_replacement_map.get(part, part) - - if not part or part == ".": - continue - - if part[:1] in "0123456789": - # pad for numeric comparison - yield part.zfill(8) - else: - yield "*" + part - - # ensure that alpha/beta/candidate are before final - yield "*final" - - -def _legacy_cmpkey(version: str) -> LegacyCmpKey: - - # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch - # greater than or equal to 0. This will effectively put the LegacyVersion, - # which uses the defacto standard originally implemented by setuptools, - # as before all PEP 440 versions. - epoch = -1 - - # This scheme is taken from pkg_resources.parse_version setuptools prior to - # it's adoption of the packaging library. - parts: List[str] = [] - for part in _parse_version_parts(version.lower()): - if part.startswith("*"): - # remove "-" before a prerelease tag - if part < "*final": - while parts and parts[-1] == "*final-": - parts.pop() - - # remove trailing zeros from each series of numeric parts - while parts and parts[-1] == "00000000": - parts.pop() - - parts.append(part) - - return epoch, tuple(parts) - - # Deliberately not anchored to the start and end of the string, to make it # easier for 3rd party code to reuse -VERSION_PATTERN = r""" +_VERSION_PATTERN = r""" v? (?: (?:(?P[0-9]+)!)? # epoch @@ -253,12 +141,56 @@ VERSION_PATTERN = r""" (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version """ +VERSION_PATTERN = _VERSION_PATTERN +""" +A string containing the regular expression used to match a valid version. + +The pattern is not anchored at either end, and is intended for embedding in larger +expressions (for example, matching a version number as part of a file name). The +regular expression should be compiled with the ``re.VERBOSE`` and ``re.IGNORECASE`` +flags set. + +:meta hide-value: +""" + class Version(_BaseVersion): + """This class abstracts handling of a project's versions. + + A :class:`Version` instance is comparison aware and can be compared and + sorted using the standard Python interfaces. + + >>> v1 = Version("1.0a5") + >>> v2 = Version("1.0") + >>> v1 + + >>> v2 + + >>> v1 < v2 + True + >>> v1 == v2 + False + >>> v1 > v2 + False + >>> v1 >= v2 + False + >>> v1 <= v2 + True + """ _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) + _key: CmpKey def __init__(self, version: str) -> None: + """Initialize a Version object. + + :param version: + The string representation of a version which will be parsed and normalized + before use. + :raises InvalidVersion: + If the ``version`` does not conform to PEP 440 in any way then this + exception will be raised. + """ # Validate the version and parse it into pieces match = self._regex.search(version) @@ -288,9 +220,19 @@ class Version(_BaseVersion): ) def __repr__(self) -> str: + """A representation of the Version that shows all internal state. + + >>> Version('1.0.0') + + """ return f"" def __str__(self) -> str: + """A string representation of the version that can be rounded-tripped. + + >>> str(Version("1.0a5")) + '1.0a5' + """ parts = [] # Epoch @@ -320,29 +262,80 @@ class Version(_BaseVersion): @property def epoch(self) -> int: + """The epoch of the version. + + >>> Version("2.0.0").epoch + 0 + >>> Version("1!2.0.0").epoch + 1 + """ _epoch: int = self._version.epoch return _epoch @property def release(self) -> Tuple[int, ...]: + """The components of the "release" segment of the version. + + >>> Version("1.2.3").release + (1, 2, 3) + >>> Version("2.0.0").release + (2, 0, 0) + >>> Version("1!2.0.0.post0").release + (2, 0, 0) + + Includes trailing zeroes but not the epoch or any pre-release / development / + post-release suffixes. + """ _release: Tuple[int, ...] = self._version.release return _release @property def pre(self) -> Optional[Tuple[str, int]]: + """The pre-release segment of the version. + + >>> print(Version("1.2.3").pre) + None + >>> Version("1.2.3a1").pre + ('a', 1) + >>> Version("1.2.3b1").pre + ('b', 1) + >>> Version("1.2.3rc1").pre + ('rc', 1) + """ _pre: Optional[Tuple[str, int]] = self._version.pre return _pre @property def post(self) -> Optional[int]: + """The post-release number of the version. + + >>> print(Version("1.2.3").post) + None + >>> Version("1.2.3.post1").post + 1 + """ return self._version.post[1] if self._version.post else None @property def dev(self) -> Optional[int]: + """The development number of the version. + + >>> print(Version("1.2.3").dev) + None + >>> Version("1.2.3.dev1").dev + 1 + """ return self._version.dev[1] if self._version.dev else None @property def local(self) -> Optional[str]: + """The local version segment of the version. + + >>> print(Version("1.2.3").local) + None + >>> Version("1.2.3+abc").local + 'abc' + """ if self._version.local: return ".".join(str(x) for x in self._version.local) else: @@ -350,10 +343,31 @@ class Version(_BaseVersion): @property def public(self) -> str: + """The public portion of the version. + + >>> Version("1.2.3").public + '1.2.3' + >>> Version("1.2.3+abc").public + '1.2.3' + >>> Version("1.2.3+abc.dev1").public + '1.2.3' + """ return str(self).split("+", 1)[0] @property def base_version(self) -> str: + """The "base version" of the version. + + >>> Version("1.2.3").base_version + '1.2.3' + >>> Version("1.2.3+abc").base_version + '1.2.3' + >>> Version("1!1.2.3+abc.dev1").base_version + '1!1.2.3' + + The "base version" is the public version of the project without any pre or post + release markers. + """ parts = [] # Epoch @@ -367,26 +381,72 @@ class Version(_BaseVersion): @property def is_prerelease(self) -> bool: + """Whether this version is a pre-release. + + >>> Version("1.2.3").is_prerelease + False + >>> Version("1.2.3a1").is_prerelease + True + >>> Version("1.2.3b1").is_prerelease + True + >>> Version("1.2.3rc1").is_prerelease + True + >>> Version("1.2.3dev1").is_prerelease + True + """ return self.dev is not None or self.pre is not None @property def is_postrelease(self) -> bool: + """Whether this version is a post-release. + + >>> Version("1.2.3").is_postrelease + False + >>> Version("1.2.3.post1").is_postrelease + True + """ return self.post is not None @property def is_devrelease(self) -> bool: + """Whether this version is a development release. + + >>> Version("1.2.3").is_devrelease + False + >>> Version("1.2.3.dev1").is_devrelease + True + """ return self.dev is not None @property def major(self) -> int: + """The first item of :attr:`release` or ``0`` if unavailable. + + >>> Version("1.2.3").major + 1 + """ return self.release[0] if len(self.release) >= 1 else 0 @property def minor(self) -> int: + """The second item of :attr:`release` or ``0`` if unavailable. + + >>> Version("1.2.3").minor + 2 + >>> Version("1").minor + 0 + """ return self.release[1] if len(self.release) >= 2 else 0 @property def micro(self) -> int: + """The third item of :attr:`release` or ``0`` if unavailable. + + >>> Version("1.2.3").micro + 3 + >>> Version("1").micro + 0 + """ return self.release[2] if len(self.release) >= 3 else 0 diff --git a/third_party/python/poetry.lock b/third_party/python/poetry.lock index af895564754a..191979bf5f83 100644 --- a/third_party/python/poetry.lock +++ b/third_party/python/poetry.lock @@ -205,16 +205,20 @@ files = [ [[package]] name = "click" -version = "7.1.2" +version = "8.1.6" description = "Composable command line interface toolkit" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" files = [ - {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, - {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, ] +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + [[package]] name = "colorama" version = "0.4.5" @@ -745,19 +749,16 @@ files = [ [[package]] name = "packaging" -version = "21.3" +version = "23.1" description = "Core utilities for Python packages" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - [[package]] name = "pathspec" version = "0.9.0" @@ -863,18 +864,6 @@ files = [ {file = "pylru-1.0.9.tar.gz", hash = "sha256:71376192671f0ad1690b2a7427d39a29b1df994c8469a9b46b03ed7e28c0172c"}, ] -[[package]] -name = "pyparsing" -version = "2.4.7" -description = "Python parsing module" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, - {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, -] - [[package]] name = "pyrsistent" version = "0.16.0" @@ -1415,4 +1404,4 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black ( [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "195fb2c45ba5de3c5c98f5d2fa44b5ce64fd3dd45c324d528477637245413d0f" +content-hash = "8623b01fb798d70285240fd5709396279a2229b3816ccb9db99dcb4e35c8c885" diff --git a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/LICENSE b/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/LICENSE deleted file mode 100644 index 1bf98523e331..000000000000 --- a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/LICENSE +++ /dev/null @@ -1,18 +0,0 @@ -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/METADATA b/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/METADATA deleted file mode 100644 index 2206ad94ed99..000000000000 --- a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/METADATA +++ /dev/null @@ -1,104 +0,0 @@ -Metadata-Version: 2.1 -Name: pyparsing -Version: 2.4.7 -Summary: Python parsing module -Home-page: https://github.com/pyparsing/pyparsing/ -Author: Paul McGuire -Author-email: ptmcg@users.sourceforge.net -License: MIT License -Download-URL: https://pypi.org/project/pyparsing/ -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Information Technology -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Requires-Python: >=2.6, !=3.0.*, !=3.1.*, !=3.2.* - -PyParsing -- A Python Parsing Module -==================================== - -|Build Status| - -Introduction -============ - -The pyparsing module is an alternative approach to creating and -executing simple grammars, vs. the traditional lex/yacc approach, or the -use of regular expressions. The pyparsing module provides a library of -classes that client code uses to construct the grammar directly in -Python code. - -*[Since first writing this description of pyparsing in late 2003, this -technique for developing parsers has become more widespread, under the -name Parsing Expression Grammars - PEGs. See more information on PEGs at* -https://en.wikipedia.org/wiki/Parsing_expression_grammar *.]* - -Here is a program to parse ``"Hello, World!"`` (or any greeting of the form -``"salutation, addressee!"``): - -.. code:: python - - from pyparsing import Word, alphas - greet = Word(alphas) + "," + Word(alphas) + "!" - hello = "Hello, World!" - print(hello, "->", greet.parseString(hello)) - -The program outputs the following:: - - Hello, World! -> ['Hello', ',', 'World', '!'] - -The Python representation of the grammar is quite readable, owing to the -self-explanatory class names, and the use of '+', '|' and '^' operator -definitions. - -The parsed results returned from ``parseString()`` can be accessed as a -nested list, a dictionary, or an object with named attributes. - -The pyparsing module handles some of the problems that are typically -vexing when writing text parsers: - -- extra or missing whitespace (the above program will also handle ``"Hello,World!"``, ``"Hello , World !"``, etc.) -- quoted strings -- embedded comments - -The examples directory includes a simple SQL parser, simple CORBA IDL -parser, a config file parser, a chemical formula parser, and a four- -function algebraic notation parser, among many others. - -Documentation -============= - -There are many examples in the online docstrings of the classes -and methods in pyparsing. You can find them compiled into online docs -at https://pyparsing-docs.readthedocs.io/en/latest/. Additional -documentation resources and project info are listed in the online -GitHub wiki, at https://github.com/pyparsing/pyparsing/wiki. An -entire directory of examples is at -https://github.com/pyparsing/pyparsing/tree/master/examples. - -License -======= - -MIT License. See header of pyparsing.py - -History -======= - -See CHANGES file. - -.. |Build Status| image:: https://travis-ci.org/pyparsing/pyparsing.svg?branch=master - :target: https://travis-ci.org/pyparsing/pyparsing - - diff --git a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/RECORD b/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/RECORD deleted file mode 100644 index 39a2bc593719..000000000000 --- a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/RECORD +++ /dev/null @@ -1,6 +0,0 @@ -pyparsing.py,sha256=oxX_ZOz8t-eros-UWY7nJgcdUgD-rQ53Ck0qp7_v3Ig,273365 -pyparsing-2.4.7.dist-info/LICENSE,sha256=ENUSChaAWAT_2otojCIL-06POXQbVzIGBNRVowngGXI,1023 -pyparsing-2.4.7.dist-info/METADATA,sha256=Ry40soZZiZrAkSMQT_KU1_1REe6FKa5UWzbT6YA8Mxs,3636 -pyparsing-2.4.7.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 -pyparsing-2.4.7.dist-info/top_level.txt,sha256=eUOjGzJVhlQ3WS2rFAy2mN3LX_7FKTM5GSJ04jfnLmU,10 -pyparsing-2.4.7.dist-info/RECORD,, diff --git a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/top_level.txt b/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/top_level.txt deleted file mode 100644 index 210dfec50b01..000000000000 --- a/third_party/python/pyparsing/pyparsing-2.4.7.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -pyparsing diff --git a/third_party/python/pyparsing/pyparsing.py b/third_party/python/pyparsing/pyparsing.py deleted file mode 100644 index 581d5bbb8a95..000000000000 --- a/third_party/python/pyparsing/pyparsing.py +++ /dev/null @@ -1,7107 +0,0 @@ -# -*- coding: utf-8 -*- -# module pyparsing.py -# -# Copyright (c) 2003-2019 Paul T. McGuire -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__doc__ = \ -""" -pyparsing module - Classes and methods to define and execute parsing grammars -============================================================================= - -The pyparsing module is an alternative approach to creating and -executing simple grammars, vs. the traditional lex/yacc approach, or the -use of regular expressions. With pyparsing, you don't need to learn -a new syntax for defining grammars or matching expressions - the parsing -module provides a library of classes that you use to construct the -grammar directly in Python. - -Here is a program to parse "Hello, World!" (or any greeting of the form -``", !"``), built up using :class:`Word`, -:class:`Literal`, and :class:`And` elements -(the :class:`'+'` operators create :class:`And` expressions, -and the strings are auto-converted to :class:`Literal` expressions):: - - from pyparsing import Word, alphas - - # define grammar of a greeting - greet = Word(alphas) + "," + Word(alphas) + "!" - - hello = "Hello, World!" - print (hello, "->", greet.parseString(hello)) - -The program outputs the following:: - - Hello, World! -> ['Hello', ',', 'World', '!'] - -The Python representation of the grammar is quite readable, owing to the -self-explanatory class names, and the use of '+', '|' and '^' operators. - -The :class:`ParseResults` object returned from -:class:`ParserElement.parseString` can be -accessed as a nested list, a dictionary, or an object with named -attributes. - -The pyparsing module handles some of the problems that are typically -vexing when writing text parsers: - - - extra or missing whitespace (the above program will also handle - "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments - - -Getting Started - ------------------ -Visit the classes :class:`ParserElement` and :class:`ParseResults` to -see the base classes that most other pyparsing -classes inherit from. Use the docstrings for examples of how to: - - - construct literal match expressions from :class:`Literal` and - :class:`CaselessLiteral` classes - - construct character word-group expressions using the :class:`Word` - class - - see how to create repetitive expressions using :class:`ZeroOrMore` - and :class:`OneOrMore` classes - - use :class:`'+'`, :class:`'|'`, :class:`'^'`, - and :class:`'&'` operators to combine simple expressions into - more complex ones - - associate names with your parsed results using - :class:`ParserElement.setResultsName` - - access the parsed data, which is returned as a :class:`ParseResults` - object - - find some helpful expression short-cuts like :class:`delimitedList` - and :class:`oneOf` - - find more useful common expressions in the :class:`pyparsing_common` - namespace class -""" - -__version__ = "2.4.7" -__versionTime__ = "30 Mar 2020 00:43 UTC" -__author__ = "Paul McGuire " - -import string -from weakref import ref as wkref -import copy -import sys -import warnings -import re -import sre_constants -import collections -import pprint -import traceback -import types -from datetime import datetime -from operator import itemgetter -import itertools -from functools import wraps -from contextlib import contextmanager - -try: - # Python 3 - from itertools import filterfalse -except ImportError: - from itertools import ifilterfalse as filterfalse - -try: - from _thread import RLock -except ImportError: - from threading import RLock - -try: - # Python 3 - from collections.abc import Iterable - from collections.abc import MutableMapping, Mapping -except ImportError: - # Python 2.7 - from collections import Iterable - from collections import MutableMapping, Mapping - -try: - from collections import OrderedDict as _OrderedDict -except ImportError: - try: - from ordereddict import OrderedDict as _OrderedDict - except ImportError: - _OrderedDict = None - -try: - from types import SimpleNamespace -except ImportError: - class SimpleNamespace: pass - -# version compatibility configuration -__compat__ = SimpleNamespace() -__compat__.__doc__ = """ - A cross-version compatibility configuration for pyparsing features that will be - released in a future version. By setting values in this configuration to True, - those features can be enabled in prior versions for compatibility development - and testing. - - - collect_all_And_tokens - flag to enable fix for Issue #63 that fixes erroneous grouping - of results names when an And expression is nested within an Or or MatchFirst; set to - True to enable bugfix released in pyparsing 2.3.0, or False to preserve - pre-2.3.0 handling of named results -""" -__compat__.collect_all_And_tokens = True - -__diag__ = SimpleNamespace() -__diag__.__doc__ = """ -Diagnostic configuration (all default to False) - - warn_multiple_tokens_in_named_alternation - flag to enable warnings when a results - name is defined on a MatchFirst or Or expression with one or more And subexpressions - (only warns if __compat__.collect_all_And_tokens is False) - - warn_ungrouped_named_tokens_in_collection - flag to enable warnings when a results - name is defined on a containing expression with ungrouped subexpressions that also - have results names - - warn_name_set_on_empty_Forward - flag to enable warnings whan a Forward is defined - with a results name, but has no contents defined - - warn_on_multiple_string_args_to_oneof - flag to enable warnings whan oneOf is - incorrectly called with multiple str arguments - - enable_debug_on_named_expressions - flag to auto-enable debug on all subsequent - calls to ParserElement.setName() -""" -__diag__.warn_multiple_tokens_in_named_alternation = False -__diag__.warn_ungrouped_named_tokens_in_collection = False -__diag__.warn_name_set_on_empty_Forward = False -__diag__.warn_on_multiple_string_args_to_oneof = False -__diag__.enable_debug_on_named_expressions = False -__diag__._all_names = [nm for nm in vars(__diag__) if nm.startswith("enable_") or nm.startswith("warn_")] - -def _enable_all_warnings(): - __diag__.warn_multiple_tokens_in_named_alternation = True - __diag__.warn_ungrouped_named_tokens_in_collection = True - __diag__.warn_name_set_on_empty_Forward = True - __diag__.warn_on_multiple_string_args_to_oneof = True -__diag__.enable_all_warnings = _enable_all_warnings - - -__all__ = ['__version__', '__versionTime__', '__author__', '__compat__', '__diag__', - 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', - 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', - 'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', - 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', - 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', - 'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', - 'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', - 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', - 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', - 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', - 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', - 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', - 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', - 'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', - 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', - 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', - 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation', 'locatedExpr', 'withClass', - 'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', - 'conditionAsParseAction', 're', - ] - -system_version = tuple(sys.version_info)[:3] -PY_3 = system_version[0] == 3 -if PY_3: - _MAX_INT = sys.maxsize - basestring = str - unichr = chr - unicode = str - _ustr = str - - # build list of single arg builtins, that can be used as parse actions - singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max] - -else: - _MAX_INT = sys.maxint - range = xrange - - def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode - friendly. It first tries str(obj). If that fails with - a UnicodeEncodeError, then it tries unicode(obj). It then - < returns the unicode object | encodes it with the default - encoding | ... >. - """ - if isinstance(obj, unicode): - return obj - - try: - # If this works, then _ustr(obj) has the same behaviour as str(obj), so - # it won't break any existing code. - return str(obj) - - except UnicodeEncodeError: - # Else encode it - ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace') - xmlcharref = Regex(r'&#\d+;') - xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:]) - return xmlcharref.transformString(ret) - - # build list of single arg builtins, tolerant of Python version, that can be used as parse actions - singleArgBuiltins = [] - import __builtin__ - - for fname in "sum len sorted reversed list tuple set any all min max".split(): - try: - singleArgBuiltins.append(getattr(__builtin__, fname)) - except AttributeError: - continue - -_generatorType = type((y for y in range(1))) - -def _xml_escape(data): - """Escape &, <, >, ", ', etc. in a string of data.""" - - # ampersand must be replaced first - from_symbols = '&><"\'' - to_symbols = ('&' + s + ';' for s in "amp gt lt quot apos".split()) - for from_, to_ in zip(from_symbols, to_symbols): - data = data.replace(from_, to_) - return data - -alphas = string.ascii_uppercase + string.ascii_lowercase -nums = "0123456789" -hexnums = nums + "ABCDEFabcdef" -alphanums = alphas + nums -_bslash = chr(92) -printables = "".join(c for c in string.printable if c not in string.whitespace) - - -def conditionAsParseAction(fn, message=None, fatal=False): - msg = message if message is not None else "failed user-defined condition" - exc_type = ParseFatalException if fatal else ParseException - fn = _trim_arity(fn) - - @wraps(fn) - def pa(s, l, t): - if not bool(fn(s, l, t)): - raise exc_type(s, l, msg) - - return pa - -class ParseBaseException(Exception): - """base exception class for all parsing runtime exceptions""" - # Performance tuning: we construct a *lot* of these, so keep this - # constructor as small and fast as possible - def __init__(self, pstr, loc=0, msg=None, elem=None): - self.loc = loc - if msg is None: - self.msg = pstr - self.pstr = "" - else: - self.msg = msg - self.pstr = pstr - self.parserElement = elem - self.args = (pstr, loc, msg) - - @classmethod - def _from_exception(cls, pe): - """ - internal factory method to simplify creating one type of ParseException - from another - avoids having __init__ signature conflicts among subclasses - """ - return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) - - def __getattr__(self, aname): - """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - """ - if aname == "lineno": - return lineno(self.loc, self.pstr) - elif aname in ("col", "column"): - return col(self.loc, self.pstr) - elif aname == "line": - return line(self.loc, self.pstr) - else: - raise AttributeError(aname) - - def __str__(self): - if self.pstr: - if self.loc >= len(self.pstr): - foundstr = ', found end of text' - else: - foundstr = (', found %r' % self.pstr[self.loc:self.loc + 1]).replace(r'\\', '\\') - else: - foundstr = '' - return ("%s%s (at char %d), (line:%d, col:%d)" % - (self.msg, foundstr, self.loc, self.lineno, self.column)) - def __repr__(self): - return _ustr(self) - def markInputline(self, markerString=">!<"): - """Extracts the exception line from the input string, and marks - the location of the exception with a special symbol. - """ - line_str = self.line - line_column = self.column - 1 - if markerString: - line_str = "".join((line_str[:line_column], - markerString, line_str[line_column:])) - return line_str.strip() - def __dir__(self): - return "lineno col line".split() + dir(type(self)) - -class ParseException(ParseBaseException): - """ - Exception thrown when parse expressions don't match class; - supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - - Example:: - - try: - Word(nums).setName("integer").parseString("ABC") - except ParseException as pe: - print(pe) - print("column: {}".format(pe.col)) - - prints:: - - Expected integer (at char 0), (line:1, col:1) - column: 1 - - """ - - @staticmethod - def explain(exc, depth=16): - """ - Method to take an exception and translate the Python internal traceback into a list - of the pyparsing expressions that caused the exception to be raised. - - Parameters: - - - exc - exception raised during parsing (need not be a ParseException, in support - of Python exceptions that might be raised in a parse action) - - depth (default=16) - number of levels back in the stack trace to list expression - and function names; if None, the full stack trace names will be listed; if 0, only - the failing input line, marker, and exception string will be shown - - Returns a multi-line string listing the ParserElements and/or function names in the - exception's stack trace. - - Note: the diagnostic output will include string representations of the expressions - that failed to parse. These representations will be more helpful if you use `setName` to - give identifiable names to your expressions. Otherwise they will use the default string - forms, which may be cryptic to read. - - explain() is only supported under Python 3. - """ - import inspect - - if depth is None: - depth = sys.getrecursionlimit() - ret = [] - if isinstance(exc, ParseBaseException): - ret.append(exc.line) - ret.append(' ' * (exc.col - 1) + '^') - ret.append("{0}: {1}".format(type(exc).__name__, exc)) - - if depth > 0: - callers = inspect.getinnerframes(exc.__traceback__, context=depth) - seen = set() - for i, ff in enumerate(callers[-depth:]): - frm = ff[0] - - f_self = frm.f_locals.get('self', None) - if isinstance(f_self, ParserElement): - if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): - continue - if f_self in seen: - continue - seen.add(f_self) - - self_type = type(f_self) - ret.append("{0}.{1} - {2}".format(self_type.__module__, - self_type.__name__, - f_self)) - elif f_self is not None: - self_type = type(f_self) - ret.append("{0}.{1}".format(self_type.__module__, - self_type.__name__)) - else: - code = frm.f_code - if code.co_name in ('wrapper', ''): - continue - - ret.append("{0}".format(code.co_name)) - - depth -= 1 - if not depth: - break - - return '\n'.join(ret) - - -class ParseFatalException(ParseBaseException): - """user-throwable exception thrown when inconsistent parse content - is found; stops all parsing immediately""" - pass - -class ParseSyntaxException(ParseFatalException): - """just like :class:`ParseFatalException`, but thrown internally - when an :class:`ErrorStop` ('-' operator) indicates - that parsing is to stop immediately because an unbacktrackable - syntax error has been found. - """ - pass - -#~ class ReparseException(ParseBaseException): - #~ """Experimental class - parse actions can raise this exception to cause - #~ pyparsing to reparse the input string: - #~ - with a modified input string, and/or - #~ - with a modified start location - #~ Set the values of the ReparseException in the constructor, and raise the - #~ exception in a parse action to cause pyparsing to use the new string/location. - #~ Setting the values as None causes no change to be made. - #~ """ - #~ def __init_( self, newstring, restartLoc ): - #~ self.newParseText = newstring - #~ self.reparseLoc = restartLoc - -class RecursiveGrammarException(Exception): - """exception thrown by :class:`ParserElement.validate` if the - grammar could be improperly recursive - """ - def __init__(self, parseElementList): - self.parseElementTrace = parseElementList - - def __str__(self): - return "RecursiveGrammarException: %s" % self.parseElementTrace - -class _ParseResultsWithOffset(object): - def __init__(self, p1, p2): - self.tup = (p1, p2) - def __getitem__(self, i): - return self.tup[i] - def __repr__(self): - return repr(self.tup[0]) - def setOffset(self, i): - self.tup = (self.tup[0], i) - -class ParseResults(object): - """Structured parse results, to provide multiple means of access to - the parsed data: - - - as a list (``len(results)``) - - by list index (``results[0], results[1]``, etc.) - - by attribute (``results.`` - see :class:`ParserElement.setResultsName`) - - Example:: - - integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' - + integer.setResultsName("day")) - # equivalent form: - # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - # parseString returns a ParseResults object - result = date_str.parseString("1999/12/31") - - def test(s, fn=repr): - print("%s -> %s" % (s, fn(eval(s)))) - test("list(result)") - test("result[0]") - test("result['month']") - test("result.day") - test("'month' in result") - test("'minutes' in result") - test("result.dump()", str) - - prints:: - - list(result) -> ['1999', '/', '12', '/', '31'] - result[0] -> '1999' - result['month'] -> '12' - result.day -> '31' - 'month' in result -> True - 'minutes' in result -> False - result.dump() -> ['1999', '/', '12', '/', '31'] - - day: 31 - - month: 12 - - year: 1999 - """ - def __new__(cls, toklist=None, name=None, asList=True, modal=True): - if isinstance(toklist, cls): - return toklist - retobj = object.__new__(cls) - retobj.__doinit = True - return retobj - - # Performance tuning: we construct a *lot* of these, so keep this - # constructor as small and fast as possible - def __init__(self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance): - if self.__doinit: - self.__doinit = False - self.__name = None - self.__parent = None - self.__accumNames = {} - self.__asList = asList - self.__modal = modal - if toklist is None: - toklist = [] - if isinstance(toklist, list): - self.__toklist = toklist[:] - elif isinstance(toklist, _generatorType): - self.__toklist = list(toklist) - else: - self.__toklist = [toklist] - self.__tokdict = dict() - - if name is not None and name: - if not modal: - self.__accumNames[name] = 0 - if isinstance(name, int): - name = _ustr(name) # will always return a str, but use _ustr for consistency - self.__name = name - if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None, '', [])): - if isinstance(toklist, basestring): - toklist = [toklist] - if asList: - if isinstance(toklist, ParseResults): - self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) - else: - self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]), 0) - self[name].__name = name - else: - try: - self[name] = toklist[0] - except (KeyError, TypeError, IndexError): - self[name] = toklist - - def __getitem__(self, i): - if isinstance(i, (int, slice)): - return self.__toklist[i] - else: - if i not in self.__accumNames: - return self.__tokdict[i][-1][0] - else: - return ParseResults([v[0] for v in self.__tokdict[i]]) - - def __setitem__(self, k, v, isinstance=isinstance): - if isinstance(v, _ParseResultsWithOffset): - self.__tokdict[k] = self.__tokdict.get(k, list()) + [v] - sub = v[0] - elif isinstance(k, (int, slice)): - self.__toklist[k] = v - sub = v - else: - self.__tokdict[k] = self.__tokdict.get(k, list()) + [_ParseResultsWithOffset(v, 0)] - sub = v - if isinstance(sub, ParseResults): - sub.__parent = wkref(self) - - def __delitem__(self, i): - if isinstance(i, (int, slice)): - mylen = len(self.__toklist) - del self.__toklist[i] - - # convert int to slice - if isinstance(i, int): - if i < 0: - i += mylen - i = slice(i, i + 1) - # get removed indices - removed = list(range(*i.indices(mylen))) - removed.reverse() - # fixup indices in token dictionary - for name, occurrences in self.__tokdict.items(): - for j in removed: - for k, (value, position) in enumerate(occurrences): - occurrences[k] = _ParseResultsWithOffset(value, position - (position > j)) - else: - del self.__tokdict[i] - - def __contains__(self, k): - return k in self.__tokdict - - def __len__(self): - return len(self.__toklist) - - def __bool__(self): - return (not not self.__toklist) - __nonzero__ = __bool__ - - def __iter__(self): - return iter(self.__toklist) - - def __reversed__(self): - return iter(self.__toklist[::-1]) - - def _iterkeys(self): - if hasattr(self.__tokdict, "iterkeys"): - return self.__tokdict.iterkeys() - else: - return iter(self.__tokdict) - - def _itervalues(self): - return (self[k] for k in self._iterkeys()) - - def _iteritems(self): - return ((k, self[k]) for k in self._iterkeys()) - - if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys.""" - - values = _itervalues - """Returns an iterator of all named result values.""" - - items = _iteritems - """Returns an iterator of all named result key-value tuples.""" - - else: - iterkeys = _iterkeys - """Returns an iterator of all named result keys (Python 2.x only).""" - - itervalues = _itervalues - """Returns an iterator of all named result values (Python 2.x only).""" - - iteritems = _iteritems - """Returns an iterator of all named result key-value tuples (Python 2.x only).""" - - def keys(self): - """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x).""" - return list(self.iterkeys()) - - def values(self): - """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" - return list(self.itervalues()) - - def items(self): - """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" - return list(self.iteritems()) - - def haskeys(self): - """Since keys() returns an iterator, this method is helpful in bypassing - code that looks for the existence of any defined results names.""" - return bool(self.__tokdict) - - def pop(self, *args, **kwargs): - """ - Removes and returns item at specified index (default= ``last``). - Supports both ``list`` and ``dict`` semantics for ``pop()``. If - passed no argument or an integer argument, it will use ``list`` - semantics and pop tokens from the list of parsed tokens. If passed - a non-integer argument (most likely a string), it will use ``dict`` - semantics and pop the corresponding value from any defined results - names. A second default return value argument is supported, just as in - ``dict.pop()``. - - Example:: - - def remove_first(tokens): - tokens.pop(0) - print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - print(OneOrMore(Word(nums)).addParseAction(remove_first).parseString("0 123 321")) # -> ['123', '321'] - - label = Word(alphas) - patt = label("LABEL") + OneOrMore(Word(nums)) - print(patt.parseString("AAB 123 321").dump()) - - # Use pop() in a parse action to remove named result (note that corresponding value is not - # removed from list form of results) - def remove_LABEL(tokens): - tokens.pop("LABEL") - return tokens - patt.addParseAction(remove_LABEL) - print(patt.parseString("AAB 123 321").dump()) - - prints:: - - ['AAB', '123', '321'] - - LABEL: AAB - - ['AAB', '123', '321'] - """ - if not args: - args = [-1] - for k, v in kwargs.items(): - if k == 'default': - args = (args[0], v) - else: - raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) - or len(args) == 1 - or args[0] in self): - index = args[0] - ret = self[index] - del self[index] - return ret - else: - defaultvalue = args[1] - return defaultvalue - - def get(self, key, defaultValue=None): - """ - Returns named result matching the given key, or if there is no - such name, then returns the given ``defaultValue`` or ``None`` if no - ``defaultValue`` is specified. - - Similar to ``dict.get()``. - - Example:: - - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - result = date_str.parseString("1999/12/31") - print(result.get("year")) # -> '1999' - print(result.get("hour", "not specified")) # -> 'not specified' - print(result.get("hour")) # -> None - """ - if key in self: - return self[key] - else: - return defaultValue - - def insert(self, index, insStr): - """ - Inserts new element at location index in the list of parsed tokens. - - Similar to ``list.insert()``. - - Example:: - - print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - - # use a parse action to insert the parse location in the front of the parsed results - def insert_locn(locn, tokens): - tokens.insert(0, locn) - print(OneOrMore(Word(nums)).addParseAction(insert_locn).parseString("0 123 321")) # -> [0, '0', '123', '321'] - """ - self.__toklist.insert(index, insStr) - # fixup indices in token dictionary - for name, occurrences in self.__tokdict.items(): - for k, (value, position) in enumerate(occurrences): - occurrences[k] = _ParseResultsWithOffset(value, position + (position > index)) - - def append(self, item): - """ - Add single element to end of ParseResults list of elements. - - Example:: - - print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - - # use a parse action to compute the sum of the parsed integers, and add it to the end - def append_sum(tokens): - tokens.append(sum(map(int, tokens))) - print(OneOrMore(Word(nums)).addParseAction(append_sum).parseString("0 123 321")) # -> ['0', '123', '321', 444] - """ - self.__toklist.append(item) - - def extend(self, itemseq): - """ - Add sequence of elements to end of ParseResults list of elements. - - Example:: - - patt = OneOrMore(Word(alphas)) - - # use a parse action to append the reverse of the matched strings, to make a palindrome - def make_palindrome(tokens): - tokens.extend(reversed([t[::-1] for t in tokens])) - return ''.join(tokens) - print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl' - """ - if isinstance(itemseq, ParseResults): - self.__iadd__(itemseq) - else: - self.__toklist.extend(itemseq) - - def clear(self): - """ - Clear all elements and results names. - """ - del self.__toklist[:] - self.__tokdict.clear() - - def __getattr__(self, name): - try: - return self[name] - except KeyError: - return "" - - def __add__(self, other): - ret = self.copy() - ret += other - return ret - - def __iadd__(self, other): - if other.__tokdict: - offset = len(self.__toklist) - addoffset = lambda a: offset if a < 0 else a + offset - otheritems = other.__tokdict.items() - otherdictitems = [(k, _ParseResultsWithOffset(v[0], addoffset(v[1]))) - for k, vlist in otheritems for v in vlist] - for k, v in otherdictitems: - self[k] = v - if isinstance(v[0], ParseResults): - v[0].__parent = wkref(self) - - self.__toklist += other.__toklist - self.__accumNames.update(other.__accumNames) - return self - - def __radd__(self, other): - if isinstance(other, int) and other == 0: - # useful for merging many ParseResults using sum() builtin - return self.copy() - else: - # this may raise a TypeError - so be it - return other + self - - def __repr__(self): - return "(%s, %s)" % (repr(self.__toklist), repr(self.__tokdict)) - - def __str__(self): - return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']' - - def _asStringList(self, sep=''): - out = [] - for item in self.__toklist: - if out and sep: - out.append(sep) - if isinstance(item, ParseResults): - out += item._asStringList() - else: - out.append(_ustr(item)) - return out - - def asList(self): - """ - Returns the parse results as a nested list of matching tokens, all converted to strings. - - Example:: - - patt = OneOrMore(Word(alphas)) - result = patt.parseString("sldkj lsdkj sldkj") - # even though the result prints in string-like form, it is actually a pyparsing ParseResults - print(type(result), result) # -> ['sldkj', 'lsdkj', 'sldkj'] - - # Use asList() to create an actual list - result_list = result.asList() - print(type(result_list), result_list) # -> ['sldkj', 'lsdkj', 'sldkj'] - """ - return [res.asList() if isinstance(res, ParseResults) else res for res in self.__toklist] - - def asDict(self): - """ - Returns the named parse results as a nested dictionary. - - Example:: - - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - result = date_str.parseString('12/31/1999') - print(type(result), repr(result)) # -> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - - result_dict = result.asDict() - print(type(result_dict), repr(result_dict)) # -> {'day': '1999', 'year': '12', 'month': '31'} - - # even though a ParseResults supports dict-like access, sometime you just need to have a dict - import json - print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable - print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"} - """ - if PY_3: - item_fn = self.items - else: - item_fn = self.iteritems - - def toItem(obj): - if isinstance(obj, ParseResults): - if obj.haskeys(): - return obj.asDict() - else: - return [toItem(v) for v in obj] - else: - return obj - - return dict((k, toItem(v)) for k, v in item_fn()) - - def copy(self): - """ - Returns a new copy of a :class:`ParseResults` object. - """ - ret = ParseResults(self.__toklist) - ret.__tokdict = dict(self.__tokdict.items()) - ret.__parent = self.__parent - ret.__accumNames.update(self.__accumNames) - ret.__name = self.__name - return ret - - def asXML(self, doctag=None, namedItemsOnly=False, indent="", formatted=True): - """ - (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names. - """ - nl = "\n" - out = [] - namedItems = dict((v[1], k) for (k, vlist) in self.__tokdict.items() - for v in vlist) - nextLevelIndent = indent + " " - - # collapse out indents if formatting is not desired - if not formatted: - indent = "" - nextLevelIndent = "" - nl = "" - - selfTag = None - if doctag is not None: - selfTag = doctag - else: - if self.__name: - selfTag = self.__name - - if not selfTag: - if namedItemsOnly: - return "" - else: - selfTag = "ITEM" - - out += [nl, indent, "<", selfTag, ">"] - - for i, res in enumerate(self.__toklist): - if isinstance(res, ParseResults): - if i in namedItems: - out += [res.asXML(namedItems[i], - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] - else: - out += [res.asXML(None, - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] - else: - # individual token, see if there is a name for it - resTag = None - if i in namedItems: - resTag = namedItems[i] - if not resTag: - if namedItemsOnly: - continue - else: - resTag = "ITEM" - xmlBodyText = _xml_escape(_ustr(res)) - out += [nl, nextLevelIndent, "<", resTag, ">", - xmlBodyText, - ""] - - out += [nl, indent, ""] - return "".join(out) - - def __lookup(self, sub): - for k, vlist in self.__tokdict.items(): - for v, loc in vlist: - if sub is v: - return k - return None - - def getName(self): - r""" - Returns the results name for this token expression. Useful when several - different expressions might match at a particular location. - - Example:: - - integer = Word(nums) - ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") - house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") - | Group(ssn_expr)("ssn") - | Group(integer)("age")) - user_info = OneOrMore(user_data) - - result = user_info.parseString("22 111-22-3333 #221B") - for item in result: - print(item.getName(), ':', item[0]) - - prints:: - - age : 22 - ssn : 111-22-3333 - house_number : 221B - """ - if self.__name: - return self.__name - elif self.__parent: - par = self.__parent() - if par: - return par.__lookup(self) - else: - return None - elif (len(self) == 1 - and len(self.__tokdict) == 1 - and next(iter(self.__tokdict.values()))[0][1] in (0, -1)): - return next(iter(self.__tokdict.keys())) - else: - return None - - def dump(self, indent='', full=True, include_list=True, _depth=0): - """ - Diagnostic method for listing out the contents of - a :class:`ParseResults`. Accepts an optional ``indent`` argument so - that this string can be embedded in a nested display of other data. - - Example:: - - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - result = date_str.parseString('12/31/1999') - print(result.dump()) - - prints:: - - ['12', '/', '31', '/', '1999'] - - day: 1999 - - month: 31 - - year: 12 - """ - out = [] - NL = '\n' - if include_list: - out.append(indent + _ustr(self.asList())) - else: - out.append('') - - if full: - if self.haskeys(): - items = sorted((str(k), v) for k, v in self.items()) - for k, v in items: - if out: - out.append(NL) - out.append("%s%s- %s: " % (indent, (' ' * _depth), k)) - if isinstance(v, ParseResults): - if v: - out.append(v.dump(indent=indent, full=full, include_list=include_list, _depth=_depth + 1)) - else: - out.append(_ustr(v)) - else: - out.append(repr(v)) - elif any(isinstance(vv, ParseResults) for vv in self): - v = self - for i, vv in enumerate(v): - if isinstance(vv, ParseResults): - out.append("\n%s%s[%d]:\n%s%s%s" % (indent, - (' ' * (_depth)), - i, - indent, - (' ' * (_depth + 1)), - vv.dump(indent=indent, - full=full, - include_list=include_list, - _depth=_depth + 1))) - else: - out.append("\n%s%s[%d]:\n%s%s%s" % (indent, - (' ' * (_depth)), - i, - indent, - (' ' * (_depth + 1)), - _ustr(vv))) - - return "".join(out) - - def pprint(self, *args, **kwargs): - """ - Pretty-printer for parsed results as a list, using the - `pprint `_ module. - Accepts additional positional or keyword args as defined for - `pprint.pprint `_ . - - Example:: - - ident = Word(alphas, alphanums) - num = Word(nums) - func = Forward() - term = ident | num | Group('(' + func + ')') - func <<= ident + Group(Optional(delimitedList(term))) - result = func.parseString("fna a,b,(fnb c,d,200),100") - result.pprint(width=40) - - prints:: - - ['fna', - ['a', - 'b', - ['(', 'fnb', ['c', 'd', '200'], ')'], - '100']] - """ - pprint.pprint(self.asList(), *args, **kwargs) - - # add support for pickle protocol - def __getstate__(self): - return (self.__toklist, - (self.__tokdict.copy(), - self.__parent is not None and self.__parent() or None, - self.__accumNames, - self.__name)) - - def __setstate__(self, state): - self.__toklist = state[0] - self.__tokdict, par, inAccumNames, self.__name = state[1] - self.__accumNames = {} - self.__accumNames.update(inAccumNames) - if par is not None: - self.__parent = wkref(par) - else: - self.__parent = None - - def __getnewargs__(self): - return self.__toklist, self.__name, self.__asList, self.__modal - - def __dir__(self): - return dir(type(self)) + list(self.keys()) - - @classmethod - def from_dict(cls, other, name=None): - """ - Helper classmethod to construct a ParseResults from a dict, preserving the - name-value relations as results names. If an optional 'name' argument is - given, a nested ParseResults will be returned - """ - def is_iterable(obj): - try: - iter(obj) - except Exception: - return False - else: - if PY_3: - return not isinstance(obj, (str, bytes)) - else: - return not isinstance(obj, basestring) - - ret = cls([]) - for k, v in other.items(): - if isinstance(v, Mapping): - ret += cls.from_dict(v, name=k) - else: - ret += cls([v], name=k, asList=is_iterable(v)) - if name is not None: - ret = cls([ret], name=name) - return ret - -MutableMapping.register(ParseResults) - -def col (loc, strg): - """Returns current column within a string, counting newlines as line separators. - The first column is number 1. - - Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See - :class:`ParserElement.parseString` for more - information on parsing strings containing ```` s, and suggested - methods to maintain a consistent view of the parsed string, the parse - location, and line and column positions within the parsed string. - """ - s = strg - return 1 if 0 < loc < len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc) - -def lineno(loc, strg): - """Returns current line number within a string, counting newlines as line separators. - The first line is number 1. - - Note - the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See :class:`ParserElement.parseString` - for more information on parsing strings containing ```` s, and - suggested methods to maintain a consistent view of the parsed string, the - parse location, and line and column positions within the parsed string. - """ - return strg.count("\n", 0, loc) + 1 - -def line(loc, strg): - """Returns the line of text containing loc within a string, counting newlines as line separators. - """ - lastCR = strg.rfind("\n", 0, loc) - nextCR = strg.find("\n", loc) - if nextCR >= 0: - return strg[lastCR + 1:nextCR] - else: - return strg[lastCR + 1:] - -def _defaultStartDebugAction(instring, loc, expr): - print(("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % (lineno(loc, instring), col(loc, instring)))) - -def _defaultSuccessDebugAction(instring, startloc, endloc, expr, toks): - print("Matched " + _ustr(expr) + " -> " + str(toks.asList())) - -def _defaultExceptionDebugAction(instring, loc, expr, exc): - print("Exception raised:" + _ustr(exc)) - -def nullDebugAction(*args): - """'Do-nothing' debug action, to suppress debugging output during parsing.""" - pass - -# Only works on Python 3.x - nonlocal is toxic to Python 2 installs -#~ 'decorator to trim function calls to match the arity of the target' -#~ def _trim_arity(func, maxargs=3): - #~ if func in singleArgBuiltins: - #~ return lambda s,l,t: func(t) - #~ limit = 0 - #~ foundArity = False - #~ def wrapper(*args): - #~ nonlocal limit,foundArity - #~ while 1: - #~ try: - #~ ret = func(*args[limit:]) - #~ foundArity = True - #~ return ret - #~ except TypeError: - #~ if limit == maxargs or foundArity: - #~ raise - #~ limit += 1 - #~ continue - #~ return wrapper - -# this version is Python 2.x-3.x cross-compatible -'decorator to trim function calls to match the arity of the target' -def _trim_arity(func, maxargs=2): - if func in singleArgBuiltins: - return lambda s, l, t: func(t) - limit = [0] - foundArity = [False] - - # traceback return data structure changed in Py3.5 - normalize back to plain tuples - if system_version[:2] >= (3, 5): - def extract_stack(limit=0): - # special handling for Python 3.5.0 - extra deep call stack by 1 - offset = -3 if system_version == (3, 5, 0) else -2 - frame_summary = traceback.extract_stack(limit=-offset + limit - 1)[offset] - return [frame_summary[:2]] - def extract_tb(tb, limit=0): - frames = traceback.extract_tb(tb, limit=limit) - frame_summary = frames[-1] - return [frame_summary[:2]] - else: - extract_stack = traceback.extract_stack - extract_tb = traceback.extract_tb - - # synthesize what would be returned by traceback.extract_stack at the call to - # user's parse action 'func', so that we don't incur call penalty at parse time - - LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND - # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! - this_line = extract_stack(limit=2)[-1] - pa_call_line_synth = (this_line[0], this_line[1] + LINE_DIFF) - - def wrapper(*args): - while 1: - try: - ret = func(*args[limit[0]:]) - foundArity[0] = True - return ret - except TypeError: - # re-raise TypeErrors if they did not come from our arity testing - if foundArity[0]: - raise - else: - try: - tb = sys.exc_info()[-1] - if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth: - raise - finally: - try: - del tb - except NameError: - pass - - if limit[0] <= maxargs: - limit[0] += 1 - continue - raise - - # copy func name to wrapper for sensible debug output - func_name = "" - try: - func_name = getattr(func, '__name__', - getattr(func, '__class__').__name__) - except Exception: - func_name = str(func) - wrapper.__name__ = func_name - - return wrapper - - -class ParserElement(object): - """Abstract base level parser element class.""" - DEFAULT_WHITE_CHARS = " \n\t\r" - verbose_stacktrace = False - - @staticmethod - def setDefaultWhitespaceChars(chars): - r""" - Overrides the default whitespace chars - - Example:: - - # default whitespace chars are space, and newline - OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - - # change to just treat newline as significant - ParserElement.setDefaultWhitespaceChars(" \t") - OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] - """ - ParserElement.DEFAULT_WHITE_CHARS = chars - - @staticmethod - def inlineLiteralsUsing(cls): - """ - Set class to be used for inclusion of string literals into a parser. - - Example:: - - # default literal class used is Literal - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] - - - # change to Suppress - ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] - """ - ParserElement._literalStringClass = cls - - @classmethod - def _trim_traceback(cls, tb): - while tb.tb_next: - tb = tb.tb_next - return tb - - def __init__(self, savelist=False): - self.parseAction = list() - self.failAction = None - # ~ self.name = "" # don't define self.name, let subclasses try/except upcall - self.strRepr = None - self.resultsName = None - self.saveAsList = savelist - self.skipWhitespace = True - self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) - self.copyDefaultWhiteChars = True - self.mayReturnEmpty = False # used when checking for left-recursion - self.keepTabs = False - self.ignoreExprs = list() - self.debug = False - self.streamlined = False - self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index - self.errmsg = "" - self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) - self.debugActions = (None, None, None) # custom debug actions - self.re = None - self.callPreparse = True # used to avoid redundant calls to preParse - self.callDuringTry = False - - def copy(self): - """ - Make a copy of this :class:`ParserElement`. Useful for defining - different parse actions for the same parsing pattern, using copies of - the original parse element. - - Example:: - - integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - integerK = integer.copy().addParseAction(lambda toks: toks[0] * 1024) + Suppress("K") - integerM = integer.copy().addParseAction(lambda toks: toks[0] * 1024 * 1024) + Suppress("M") - - print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) - - prints:: - - [5120, 100, 655360, 268435456] - - Equivalent form of ``expr.copy()`` is just ``expr()``:: - - integerM = integer().addParseAction(lambda toks: toks[0] * 1024 * 1024) + Suppress("M") - """ - cpy = copy.copy(self) - cpy.parseAction = self.parseAction[:] - cpy.ignoreExprs = self.ignoreExprs[:] - if self.copyDefaultWhiteChars: - cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS - return cpy - - def setName(self, name): - """ - Define name for this expression, makes debugging and exception messages clearer. - - Example:: - - Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) - Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) - """ - self.name = name - self.errmsg = "Expected " + self.name - if __diag__.enable_debug_on_named_expressions: - self.setDebug() - return self - - def setResultsName(self, name, listAllMatches=False): - """ - Define name for referencing matching tokens as a nested attribute - of the returned parse results. - NOTE: this returns a *copy* of the original :class:`ParserElement` object; - this is so that the client can define a basic element, such as an - integer, and reference it in multiple places with different names. - - You can also set results names using the abbreviated syntax, - ``expr("name")`` in place of ``expr.setResultsName("name")`` - - see :class:`__call__`. - - Example:: - - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' - + integer.setResultsName("day")) - - # equivalent form: - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - """ - return self._setResultsName(name, listAllMatches) - - def _setResultsName(self, name, listAllMatches=False): - newself = self.copy() - if name.endswith("*"): - name = name[:-1] - listAllMatches = True - newself.resultsName = name - newself.modalResults = not listAllMatches - return newself - - def setBreak(self, breakFlag=True): - """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set ``breakFlag`` to True to enable, False to - disable. - """ - if breakFlag: - _parseMethod = self._parse - def breaker(instring, loc, doActions=True, callPreParse=True): - import pdb - # this call to pdb.set_trace() is intentional, not a checkin error - pdb.set_trace() - return _parseMethod(instring, loc, doActions, callPreParse) - breaker._originalParseMethod = _parseMethod - self._parse = breaker - else: - if hasattr(self._parse, "_originalParseMethod"): - self._parse = self._parse._originalParseMethod - return self - - def setParseAction(self, *fns, **kwargs): - """ - Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as ``fn(s, loc, toks)`` , - ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where: - - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object - - If the functions in fns modify the tokens, they can return them as the return - value from fn, and the modified list of tokens will replace the original. - Otherwise, fn does not need to return any value. - - If None is passed as the parse action, all previously added parse actions for this - expression are cleared. - - Optional keyword arguments: - - callDuringTry = (default= ``False``) indicate if parse action should be run during lookaheads and alternate testing - - Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See :class:`parseString for more - information on parsing strings containing ```` s, and suggested - methods to maintain a consistent view of the parsed string, the parse - location, and line and column positions within the parsed string. - - Example:: - - integer = Word(nums) - date_str = integer + '/' + integer + '/' + integer - - date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] - - # use parse action to convert to ints at parse time - integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - date_str = integer + '/' + integer + '/' + integer - - # note that integer fields are now ints, not strings - date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31] - """ - if list(fns) == [None,]: - self.parseAction = [] - else: - if not all(callable(fn) for fn in fns): - raise TypeError("parse actions must be callable") - self.parseAction = list(map(_trim_arity, list(fns))) - self.callDuringTry = kwargs.get("callDuringTry", False) - return self - - def addParseAction(self, *fns, **kwargs): - """ - Add one or more parse actions to expression's list of parse actions. See :class:`setParseAction`. - - See examples in :class:`copy`. - """ - self.parseAction += list(map(_trim_arity, list(fns))) - self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) - return self - - def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - :class:`setParseAction` for function call signatures. Unlike ``setParseAction``, - functions passed to ``addCondition`` need to return boolean success/fail of the condition. - - Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - - Example:: - - integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - year_int = integer.copy() - year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") - date_str = year_int + '/' + integer + '/' + integer - - result = date_str.parseString("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1) - """ - for fn in fns: - self.parseAction.append(conditionAsParseAction(fn, message=kwargs.get('message'), - fatal=kwargs.get('fatal', False))) - - self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) - return self - - def setFailAction(self, fn): - """Define action to perform if parsing fails at this expression. - Fail acton fn is a callable function that takes the arguments - ``fn(s, loc, expr, err)`` where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw :class:`ParseFatalException` - if it is desired to stop parsing immediately.""" - self.failAction = fn - return self - - def _skipIgnorables(self, instring, loc): - exprsFound = True - while exprsFound: - exprsFound = False - for e in self.ignoreExprs: - try: - while 1: - loc, dummy = e._parse(instring, loc) - exprsFound = True - except ParseException: - pass - return loc - - def preParse(self, instring, loc): - if self.ignoreExprs: - loc = self._skipIgnorables(instring, loc) - - if self.skipWhitespace: - wt = self.whiteChars - instrlen = len(instring) - while loc < instrlen and instring[loc] in wt: - loc += 1 - - return loc - - def parseImpl(self, instring, loc, doActions=True): - return loc, [] - - def postParse(self, instring, loc, tokenlist): - return tokenlist - - # ~ @profile - def _parseNoCache(self, instring, loc, doActions=True, callPreParse=True): - TRY, MATCH, FAIL = 0, 1, 2 - debugging = (self.debug) # and doActions) - - if debugging or self.failAction: - # ~ print ("Match", self, "at loc", loc, "(%d, %d)" % (lineno(loc, instring), col(loc, instring))) - if self.debugActions[TRY]: - self.debugActions[TRY](instring, loc, self) - try: - if callPreParse and self.callPreparse: - preloc = self.preParse(instring, loc) - else: - preloc = loc - tokensStart = preloc - if self.mayIndexError or preloc >= len(instring): - try: - loc, tokens = self.parseImpl(instring, preloc, doActions) - except IndexError: - raise ParseException(instring, len(instring), self.errmsg, self) - else: - loc, tokens = self.parseImpl(instring, preloc, doActions) - except Exception as err: - # ~ print ("Exception raised:", err) - if self.debugActions[FAIL]: - self.debugActions[FAIL](instring, tokensStart, self, err) - if self.failAction: - self.failAction(instring, tokensStart, self, err) - raise - else: - if callPreParse and self.callPreparse: - preloc = self.preParse(instring, loc) - else: - preloc = loc - tokensStart = preloc - if self.mayIndexError or preloc >= len(instring): - try: - loc, tokens = self.parseImpl(instring, preloc, doActions) - except IndexError: - raise ParseException(instring, len(instring), self.errmsg, self) - else: - loc, tokens = self.parseImpl(instring, preloc, doActions) - - tokens = self.postParse(instring, loc, tokens) - - retTokens = ParseResults(tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults) - if self.parseAction and (doActions or self.callDuringTry): - if debugging: - try: - for fn in self.parseAction: - try: - tokens = fn(instring, tokensStart, retTokens) - except IndexError as parse_action_exc: - exc = ParseException("exception raised in parse action") - exc.__cause__ = parse_action_exc - raise exc - - if tokens is not None and tokens is not retTokens: - retTokens = ParseResults(tokens, - self.resultsName, - asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), - modal=self.modalResults) - except Exception as err: - # ~ print "Exception raised in user parse action:", err - if self.debugActions[FAIL]: - self.debugActions[FAIL](instring, tokensStart, self, err) - raise - else: - for fn in self.parseAction: - try: - tokens = fn(instring, tokensStart, retTokens) - except IndexError as parse_action_exc: - exc = ParseException("exception raised in parse action") - exc.__cause__ = parse_action_exc - raise exc - - if tokens is not None and tokens is not retTokens: - retTokens = ParseResults(tokens, - self.resultsName, - asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), - modal=self.modalResults) - if debugging: - # ~ print ("Matched", self, "->", retTokens.asList()) - if self.debugActions[MATCH]: - self.debugActions[MATCH](instring, tokensStart, loc, self, retTokens) - - return loc, retTokens - - def tryParse(self, instring, loc): - try: - return self._parse(instring, loc, doActions=False)[0] - except ParseFatalException: - raise ParseException(instring, loc, self.errmsg, self) - - def canParseNext(self, instring, loc): - try: - self.tryParse(instring, loc) - except (ParseException, IndexError): - return False - else: - return True - - class _UnboundedCache(object): - def __init__(self): - cache = {} - self.not_in_cache = not_in_cache = object() - - def get(self, key): - return cache.get(key, not_in_cache) - - def set(self, key, value): - cache[key] = value - - def clear(self): - cache.clear() - - def cache_len(self): - return len(cache) - - self.get = types.MethodType(get, self) - self.set = types.MethodType(set, self) - self.clear = types.MethodType(clear, self) - self.__len__ = types.MethodType(cache_len, self) - - if _OrderedDict is not None: - class _FifoCache(object): - def __init__(self, size): - self.not_in_cache = not_in_cache = object() - - cache = _OrderedDict() - - def get(self, key): - return cache.get(key, not_in_cache) - - def set(self, key, value): - cache[key] = value - while len(cache) > size: - try: - cache.popitem(False) - except KeyError: - pass - - def clear(self): - cache.clear() - - def cache_len(self): - return len(cache) - - self.get = types.MethodType(get, self) - self.set = types.MethodType(set, self) - self.clear = types.MethodType(clear, self) - self.__len__ = types.MethodType(cache_len, self) - - else: - class _FifoCache(object): - def __init__(self, size): - self.not_in_cache = not_in_cache = object() - - cache = {} - key_fifo = collections.deque([], size) - - def get(self, key): - return cache.get(key, not_in_cache) - - def set(self, key, value): - cache[key] = value - while len(key_fifo) > size: - cache.pop(key_fifo.popleft(), None) - key_fifo.append(key) - - def clear(self): - cache.clear() - key_fifo.clear() - - def cache_len(self): - return len(cache) - - self.get = types.MethodType(get, self) - self.set = types.MethodType(set, self) - self.clear = types.MethodType(clear, self) - self.__len__ = types.MethodType(cache_len, self) - - # argument cache for optimizing repeated calls when backtracking through recursive expressions - packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail - packrat_cache_lock = RLock() - packrat_cache_stats = [0, 0] - - # this method gets repeatedly called during backtracking with the same arguments - - # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression - def _parseCache(self, instring, loc, doActions=True, callPreParse=True): - HIT, MISS = 0, 1 - lookup = (self, instring, loc, callPreParse, doActions) - with ParserElement.packrat_cache_lock: - cache = ParserElement.packrat_cache - value = cache.get(lookup) - if value is cache.not_in_cache: - ParserElement.packrat_cache_stats[MISS] += 1 - try: - value = self._parseNoCache(instring, loc, doActions, callPreParse) - except ParseBaseException as pe: - # cache a copy of the exception, without the traceback - cache.set(lookup, pe.__class__(*pe.args)) - raise - else: - cache.set(lookup, (value[0], value[1].copy())) - return value - else: - ParserElement.packrat_cache_stats[HIT] += 1 - if isinstance(value, Exception): - raise value - return value[0], value[1].copy() - - _parse = _parseNoCache - - @staticmethod - def resetCache(): - ParserElement.packrat_cache.clear() - ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats) - - _packratEnabled = False - @staticmethod - def enablePackrat(cache_size_limit=128): - """Enables "packrat" parsing, which adds memoizing to the parsing logic. - Repeated parse attempts at the same string location (which happens - often in many complex grammars) can immediately return a cached value, - instead of re-executing parsing/validating code. Memoizing is done of - both valid results and parsing exceptions. - - Parameters: - - - cache_size_limit - (default= ``128``) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - - This speedup may break existing programs that use parse actions that - have side-effects. For this reason, packrat parsing is disabled when - you first import pyparsing. To activate the packrat feature, your - program must call the class method :class:`ParserElement.enablePackrat`. - For best results, call ``enablePackrat()`` immediately after - importing pyparsing. - - Example:: - - import pyparsing - pyparsing.ParserElement.enablePackrat() - """ - if not ParserElement._packratEnabled: - ParserElement._packratEnabled = True - if cache_size_limit is None: - ParserElement.packrat_cache = ParserElement._UnboundedCache() - else: - ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit) - ParserElement._parse = ParserElement._parseCache - - def parseString(self, instring, parseAll=False): - """ - Execute the parse expression with the given string. - This is the main interface to the client code, once the complete - expression has been built. - - Returns the parsed data as a :class:`ParseResults` object, which may be - accessed as a list, or as a dict or object with attributes if the given parser - includes results names. - - If you want the grammar to require that the entire input string be - successfully parsed, then set ``parseAll`` to True (equivalent to ending - the grammar with ``StringEnd()``). - - Note: ``parseString`` implicitly calls ``expandtabs()`` on the input string, - in order to report proper column numbers in parse actions. - If the input string contains tabs and - the grammar uses parse actions that use the ``loc`` argument to index into the - string being parsed, you can ensure you have a consistent view of the input - string by: - - - calling ``parseWithTabs`` on your grammar before calling ``parseString`` - (see :class:`parseWithTabs`) - - define your parse action using the full ``(s, loc, toks)`` signature, and - reference the input string using the parse action's ``s`` argument - - explictly expand the tabs in your input string before calling - ``parseString`` - - Example:: - - Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] - Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text - """ - ParserElement.resetCache() - if not self.streamlined: - self.streamline() - # ~ self.saveAsList = True - for e in self.ignoreExprs: - e.streamline() - if not self.keepTabs: - instring = instring.expandtabs() - try: - loc, tokens = self._parse(instring, 0) - if parseAll: - loc = self.preParse(instring, loc) - se = Empty() + StringEnd() - se._parse(instring, loc) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clearing out pyparsing internal stack trace - if getattr(exc, '__traceback__', None) is not None: - exc.__traceback__ = self._trim_traceback(exc.__traceback__) - raise exc - else: - return tokens - - def scanString(self, instring, maxMatches=_MAX_INT, overlap=False): - """ - Scan the input string for expression matches. Each match will return the - matching tokens, start location, and end location. May be called with optional - ``maxMatches`` argument, to clip scanning after 'n' matches are found. If - ``overlap`` is specified, then overlapping matches will be reported. - - Note that the start and end locations are reported relative to the string - being parsed. See :class:`parseString` for more information on parsing - strings with embedded tabs. - - Example:: - - source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" - print(source) - for tokens, start, end in Word(alphas).scanString(source): - print(' '*start + '^'*(end-start)) - print(' '*start + tokens[0]) - - prints:: - - sldjf123lsdjjkf345sldkjf879lkjsfd987 - ^^^^^ - sldjf - ^^^^^^^ - lsdjjkf - ^^^^^^ - sldkjf - ^^^^^^ - lkjsfd - """ - if not self.streamlined: - self.streamline() - for e in self.ignoreExprs: - e.streamline() - - if not self.keepTabs: - instring = _ustr(instring).expandtabs() - instrlen = len(instring) - loc = 0 - preparseFn = self.preParse - parseFn = self._parse - ParserElement.resetCache() - matches = 0 - try: - while loc <= instrlen and matches < maxMatches: - try: - preloc = preparseFn(instring, loc) - nextLoc, tokens = parseFn(instring, preloc, callPreParse=False) - except ParseException: - loc = preloc + 1 - else: - if nextLoc > loc: - matches += 1 - yield tokens, preloc, nextLoc - if overlap: - nextloc = preparseFn(instring, loc) - if nextloc > loc: - loc = nextLoc - else: - loc += 1 - else: - loc = nextLoc - else: - loc = preloc + 1 - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clearing out pyparsing internal stack trace - if getattr(exc, '__traceback__', None) is not None: - exc.__traceback__ = self._trim_traceback(exc.__traceback__) - raise exc - - def transformString(self, instring): - """ - Extension to :class:`scanString`, to modify matching text with modified tokens that may - be returned from a parse action. To use ``transformString``, define a grammar and - attach a parse action to it that modifies the returned token list. - Invoking ``transformString()`` on a target string will then scan for matches, - and replace the matched text patterns according to the logic in the parse - action. ``transformString()`` returns the resulting transformed string. - - Example:: - - wd = Word(alphas) - wd.setParseAction(lambda toks: toks[0].title()) - - print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - - prints:: - - Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. - """ - out = [] - lastE = 0 - # force preservation of s, to minimize unwanted transformation of string, and to - # keep string locs straight between transformString and scanString - self.keepTabs = True - try: - for t, s, e in self.scanString(instring): - out.append(instring[lastE:s]) - if t: - if isinstance(t, ParseResults): - out += t.asList() - elif isinstance(t, list): - out += t - else: - out.append(t) - lastE = e - out.append(instring[lastE:]) - out = [o for o in out if o] - return "".join(map(_ustr, _flatten(out))) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clearing out pyparsing internal stack trace - if getattr(exc, '__traceback__', None) is not None: - exc.__traceback__ = self._trim_traceback(exc.__traceback__) - raise exc - - def searchString(self, instring, maxMatches=_MAX_INT): - """ - Another extension to :class:`scanString`, simplifying the access to the tokens found - to match the given parse expression. May be called with optional - ``maxMatches`` argument, to clip searching after 'n' matches are found. - - Example:: - - # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters - cap_word = Word(alphas.upper(), alphas.lower()) - - print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) - - # the sum() builtin can be used to merge results into a single ParseResults object - print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) - - prints:: - - [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] - ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] - """ - try: - return ParseResults([t for t, s, e in self.scanString(instring, maxMatches)]) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clearing out pyparsing internal stack trace - if getattr(exc, '__traceback__', None) is not None: - exc.__traceback__ = self._trim_traceback(exc.__traceback__) - raise exc - - def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): - """ - Generator method to split a string using the given expression as a separator. - May be called with optional ``maxsplit`` argument, to limit the number of splits; - and the optional ``includeSeparators`` argument (default= ``False``), if the separating - matching text should be included in the split results. - - Example:: - - punc = oneOf(list(".,;:/-!?")) - print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) - - prints:: - - ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] - """ - splits = 0 - last = 0 - for t, s, e in self.scanString(instring, maxMatches=maxsplit): - yield instring[last:s] - if includeSeparators: - yield t[0] - last = e - yield instring[last:] - - def __add__(self, other): - """ - Implementation of + operator - returns :class:`And`. Adding strings to a ParserElement - converts them to :class:`Literal`s by default. - - Example:: - - greet = Word(alphas) + "," + Word(alphas) + "!" - hello = "Hello, World!" - print (hello, "->", greet.parseString(hello)) - - prints:: - - Hello, World! -> ['Hello', ',', 'World', '!'] - - ``...`` may be used as a parse expression as a short form of :class:`SkipTo`. - - Literal('start') + ... + Literal('end') - - is equivalent to: - - Literal('start') + SkipTo('end')("_skipped*") + Literal('end') - - Note that the skipped text is returned with '_skipped' as a results name, - and to support having multiple skips in the same parser, the value returned is - a list of all skipped text. - """ - if other is Ellipsis: - return _PendingSkip(self) - - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return And([self, other]) - - def __radd__(self, other): - """ - Implementation of + operator when left operand is not a :class:`ParserElement` - """ - if other is Ellipsis: - return SkipTo(self)("_skipped*") + self - - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other + self - - def __sub__(self, other): - """ - Implementation of - operator, returns :class:`And` with error stop - """ - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return self + And._ErrorStop() + other - - def __rsub__(self, other): - """ - Implementation of - operator when left operand is not a :class:`ParserElement` - """ - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other - self - - def __mul__(self, other): - """ - Implementation of * operator, allows use of ``expr * 3`` in place of - ``expr + expr + expr``. Expressions may also me multiplied by a 2-integer - tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples - may also include ``None`` as in: - - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent - to ``expr*n + ZeroOrMore(expr)`` - (read as "at least n instances of ``expr``") - - ``expr*(None, n)`` is equivalent to ``expr*(0, n)`` - (read as "0 to n instances of ``expr``") - - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)`` - - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)`` - - Note that ``expr*(None, n)`` does not raise an exception if - more than n exprs exist in the input stream; that is, - ``expr*(None, n)`` does not enforce a maximum number of expr - occurrences. If this behavior is desired, then write - ``expr*(None, n) + ~expr`` - """ - if other is Ellipsis: - other = (0, None) - elif isinstance(other, tuple) and other[:1] == (Ellipsis,): - other = ((0, ) + other[1:] + (None,))[:2] - - if isinstance(other, int): - minElements, optElements = other, 0 - elif isinstance(other, tuple): - other = tuple(o if o is not Ellipsis else None for o in other) - other = (other + (None, None))[:2] - if other[0] is None: - other = (0, other[1]) - if isinstance(other[0], int) and other[1] is None: - if other[0] == 0: - return ZeroOrMore(self) - if other[0] == 1: - return OneOrMore(self) - else: - return self * other[0] + ZeroOrMore(self) - elif isinstance(other[0], int) and isinstance(other[1], int): - minElements, optElements = other - optElements -= minElements - else: - raise TypeError("cannot multiply 'ParserElement' and ('%s', '%s') objects", type(other[0]), type(other[1])) - else: - raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other)) - - if minElements < 0: - raise ValueError("cannot multiply ParserElement by negative value") - if optElements < 0: - raise ValueError("second tuple value must be greater or equal to first tuple value") - if minElements == optElements == 0: - raise ValueError("cannot multiply ParserElement by 0 or (0, 0)") - - if optElements: - def makeOptionalList(n): - if n > 1: - return Optional(self + makeOptionalList(n - 1)) - else: - return Optional(self) - if minElements: - if minElements == 1: - ret = self + makeOptionalList(optElements) - else: - ret = And([self] * minElements) + makeOptionalList(optElements) - else: - ret = makeOptionalList(optElements) - else: - if minElements == 1: - ret = self - else: - ret = And([self] * minElements) - return ret - - def __rmul__(self, other): - return self.__mul__(other) - - def __or__(self, other): - """ - Implementation of | operator - returns :class:`MatchFirst` - """ - if other is Ellipsis: - return _PendingSkip(self, must_skip=True) - - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return MatchFirst([self, other]) - - def __ror__(self, other): - """ - Implementation of | operator when left operand is not a :class:`ParserElement` - """ - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other | self - - def __xor__(self, other): - """ - Implementation of ^ operator - returns :class:`Or` - """ - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return Or([self, other]) - - def __rxor__(self, other): - """ - Implementation of ^ operator when left operand is not a :class:`ParserElement` - """ - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other ^ self - - def __and__(self, other): - """ - Implementation of & operator - returns :class:`Each` - """ - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return Each([self, other]) - - def __rand__(self, other): - """ - Implementation of & operator when left operand is not a :class:`ParserElement` - """ - if isinstance(other, basestring): - other = self._literalStringClass(other) - if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other & self - - def __invert__(self): - """ - Implementation of ~ operator - returns :class:`NotAny` - """ - return NotAny(self) - - def __iter__(self): - # must implement __iter__ to override legacy use of sequential access to __getitem__ to - # iterate over a sequence - raise TypeError('%r object is not iterable' % self.__class__.__name__) - - def __getitem__(self, key): - """ - use ``[]`` indexing notation as a short form for expression repetition: - - ``expr[n]`` is equivalent to ``expr*n`` - - ``expr[m, n]`` is equivalent to ``expr*(m, n)`` - - ``expr[n, ...]`` or ``expr[n,]`` is equivalent - to ``expr*n + ZeroOrMore(expr)`` - (read as "at least n instances of ``expr``") - - ``expr[..., n]`` is equivalent to ``expr*(0, n)`` - (read as "0 to n instances of ``expr``") - - ``expr[...]`` and ``expr[0, ...]`` are equivalent to ``ZeroOrMore(expr)`` - - ``expr[1, ...]`` is equivalent to ``OneOrMore(expr)`` - ``None`` may be used in place of ``...``. - - Note that ``expr[..., n]`` and ``expr[m, n]``do not raise an exception - if more than ``n`` ``expr``s exist in the input stream. If this behavior is - desired, then write ``expr[..., n] + ~expr``. - """ - - # convert single arg keys to tuples - try: - if isinstance(key, str): - key = (key,) - iter(key) - except TypeError: - key = (key, key) - - if len(key) > 2: - warnings.warn("only 1 or 2 index arguments supported ({0}{1})".format(key[:5], - '... [{0}]'.format(len(key)) - if len(key) > 5 else '')) - - # clip to 2 elements - ret = self * tuple(key[:2]) - return ret - - def __call__(self, name=None): - """ - Shortcut for :class:`setResultsName`, with ``listAllMatches=False``. - - If ``name`` is given with a trailing ``'*'`` character, then ``listAllMatches`` will be - passed as ``True``. - - If ``name` is omitted, same as calling :class:`copy`. - - Example:: - - # these are equivalent - userdata = Word(alphas).setResultsName("name") + Word(nums + "-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums + "-")("socsecno") - """ - if name is not None: - return self._setResultsName(name) - else: - return self.copy() - - def suppress(self): - """ - Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from - cluttering up returned output. - """ - return Suppress(self) - - def leaveWhitespace(self): - """ - Disables the skipping of whitespace before matching the characters in the - :class:`ParserElement`'s defined pattern. This is normally only used internally by - the pyparsing module, but may be needed in some whitespace-sensitive grammars. - """ - self.skipWhitespace = False - return self - - def setWhitespaceChars(self, chars): - """ - Overrides the default whitespace chars - """ - self.skipWhitespace = True - self.whiteChars = chars - self.copyDefaultWhiteChars = False - return self - - def parseWithTabs(self): - """ - Overrides default behavior to expand ````s to spaces before parsing the input string. - Must be called before ``parseString`` when the input grammar contains elements that - match ```` characters. - """ - self.keepTabs = True - return self - - def ignore(self, other): - """ - Define expression to be ignored (e.g., comments) while doing pattern - matching; may be called repeatedly, to define multiple comment or other - ignorable patterns. - - Example:: - - patt = OneOrMore(Word(alphas)) - patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - - patt.ignore(cStyleComment) - patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] - """ - if isinstance(other, basestring): - other = Suppress(other) - - if isinstance(other, Suppress): - if other not in self.ignoreExprs: - self.ignoreExprs.append(other) - else: - self.ignoreExprs.append(Suppress(other.copy())) - return self - - def setDebugActions(self, startAction, successAction, exceptionAction): - """ - Enable display of debugging messages while doing pattern matching. - """ - self.debugActions = (startAction or _defaultStartDebugAction, - successAction or _defaultSuccessDebugAction, - exceptionAction or _defaultExceptionDebugAction) - self.debug = True - return self - - def setDebug(self, flag=True): - """ - Enable display of debugging messages while doing pattern matching. - Set ``flag`` to True to enable, False to disable. - - Example:: - - wd = Word(alphas).setName("alphaword") - integer = Word(nums).setName("numword") - term = wd | integer - - # turn on debugging for wd - wd.setDebug() - - OneOrMore(term).parseString("abc 123 xyz 890") - - prints:: - - Match alphaword at loc 0(1,1) - Matched alphaword -> ['abc'] - Match alphaword at loc 3(1,4) - Exception raised:Expected alphaword (at char 4), (line:1, col:5) - Match alphaword at loc 7(1,8) - Matched alphaword -> ['xyz'] - Match alphaword at loc 11(1,12) - Exception raised:Expected alphaword (at char 12), (line:1, col:13) - Match alphaword at loc 15(1,16) - Exception raised:Expected alphaword (at char 15), (line:1, col:16) - - The output shown is that produced by the default debug actions - custom debug actions can be - specified using :class:`setDebugActions`. Prior to attempting - to match the ``wd`` expression, the debugging message ``"Match at loc (,)"`` - is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` - message is shown. Also note the use of :class:`setName` to assign a human-readable name to the expression, - which makes debugging and exception messages easier to understand - for instance, the default - name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. - """ - if flag: - self.setDebugActions(_defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction) - else: - self.debug = False - return self - - def __str__(self): - return self.name - - def __repr__(self): - return _ustr(self) - - def streamline(self): - self.streamlined = True - self.strRepr = None - return self - - def checkRecursion(self, parseElementList): - pass - - def validate(self, validateTrace=None): - """ - Check defined expressions for valid structure, check for infinite recursive definitions. - """ - self.checkRecursion([]) - - def parseFile(self, file_or_filename, parseAll=False): - """ - Execute the parse expression on the given file or filename. - If a filename is specified (instead of a file object), - the entire file is opened, read, and closed before parsing. - """ - try: - file_contents = file_or_filename.read() - except AttributeError: - with open(file_or_filename, "r") as f: - file_contents = f.read() - try: - return self.parseString(file_contents, parseAll) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clearing out pyparsing internal stack trace - if getattr(exc, '__traceback__', None) is not None: - exc.__traceback__ = self._trim_traceback(exc.__traceback__) - raise exc - - def __eq__(self, other): - if self is other: - return True - elif isinstance(other, basestring): - return self.matches(other) - elif isinstance(other, ParserElement): - return vars(self) == vars(other) - return False - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return id(self) - - def __req__(self, other): - return self == other - - def __rne__(self, other): - return not (self == other) - - def matches(self, testString, parseAll=True): - """ - Method for quick testing of a parser against a test string. Good for simple - inline microtests of sub expressions while building up larger parser. - - Parameters: - - testString - to test against this expression for a match - - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests - - Example:: - - expr = Word(nums) - assert expr.matches("100") - """ - try: - self.parseString(_ustr(testString), parseAll=parseAll) - return True - except ParseBaseException: - return False - - def runTests(self, tests, parseAll=True, comment='#', - fullDump=True, printResults=True, failureTests=False, postParse=None, - file=None): - """ - Execute the parse expression on a series of test strings, showing each - test, the parsed results or where the parse failed. Quick and easy way to - run a parse expression against a list of sample strings. - - Parameters: - - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests - - comment - (default= ``'#'``) - expression for indicating embedded comments in the test - string; pass None to disable comment filtering - - fullDump - (default= ``True``) - dump results as list followed by results names in nested outline; - if False, only dump nested list - - printResults - (default= ``True``) prints test output to stdout - - failureTests - (default= ``False``) indicates if these tests are expected to fail parsing - - postParse - (default= ``None``) optional callback for successful parse results; called as - `fn(test_string, parse_results)` and returns a string to be added to the test output - - file - (default=``None``) optional file-like object to which test output will be written; - if None, will default to ``sys.stdout`` - - Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if ``failureTests`` is True), and the results contain a list of lines of each - test's output - - Example:: - - number_expr = pyparsing_common.number.copy() - - result = number_expr.runTests(''' - # unsigned integer - 100 - # negative integer - -100 - # float with scientific notation - 6.02e23 - # integer with scientific notation - 1e-12 - ''') - print("Success" if result[0] else "Failed!") - - result = number_expr.runTests(''' - # stray character - 100Z - # missing leading digit before '.' - -.100 - # too many '.' - 3.14.159 - ''', failureTests=True) - print("Success" if result[0] else "Failed!") - - prints:: - - # unsigned integer - 100 - [100] - - # negative integer - -100 - [-100] - - # float with scientific notation - 6.02e23 - [6.02e+23] - - # integer with scientific notation - 1e-12 - [1e-12] - - Success - - # stray character - 100Z - ^ - FAIL: Expected end of text (at char 3), (line:1, col:4) - - # missing leading digit before '.' - -.100 - ^ - FAIL: Expected {real number with scientific notation | real number | signed integer} (at char 0), (line:1, col:1) - - # too many '.' - 3.14.159 - ^ - FAIL: Expected end of text (at char 4), (line:1, col:5) - - Success - - Each test string must be on a single line. If you want to test a string that spans multiple - lines, create a test like this:: - - expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - - (Note that this is a raw string literal, you must include the leading 'r'.) - """ - if isinstance(tests, basestring): - tests = list(map(str.strip, tests.rstrip().splitlines())) - if isinstance(comment, basestring): - comment = Literal(comment) - if file is None: - file = sys.stdout - print_ = file.write - - allResults = [] - comments = [] - success = True - NL = Literal(r'\n').addParseAction(replaceWith('\n')).ignore(quotedString) - BOM = u'\ufeff' - for t in tests: - if comment is not None and comment.matches(t, False) or comments and not t: - comments.append(t) - continue - if not t: - continue - out = ['\n' + '\n'.join(comments) if comments else '', t] - comments = [] - try: - # convert newline marks to actual newlines, and strip leading BOM if present - t = NL.transformString(t.lstrip(BOM)) - result = self.parseString(t, parseAll=parseAll) - except ParseBaseException as pe: - fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" - if '\n' in t: - out.append(line(pe.loc, t)) - out.append(' ' * (col(pe.loc, t) - 1) + '^' + fatal) - else: - out.append(' ' * pe.loc + '^' + fatal) - out.append("FAIL: " + str(pe)) - success = success and failureTests - result = pe - except Exception as exc: - out.append("FAIL-EXCEPTION: " + str(exc)) - success = success and failureTests - result = exc - else: - success = success and not failureTests - if postParse is not None: - try: - pp_value = postParse(t, result) - if pp_value is not None: - if isinstance(pp_value, ParseResults): - out.append(pp_value.dump()) - else: - out.append(str(pp_value)) - else: - out.append(result.dump()) - except Exception as e: - out.append(result.dump(full=fullDump)) - out.append("{0} failed: {1}: {2}".format(postParse.__name__, type(e).__name__, e)) - else: - out.append(result.dump(full=fullDump)) - - if printResults: - if fullDump: - out.append('') - print_('\n'.join(out)) - - allResults.append((t, result)) - - return success, allResults - - -class _PendingSkip(ParserElement): - # internal placeholder class to hold a place were '...' is added to a parser element, - # once another ParserElement is added, this placeholder will be replaced with a SkipTo - def __init__(self, expr, must_skip=False): - super(_PendingSkip, self).__init__() - self.strRepr = str(expr + Empty()).replace('Empty', '...') - self.name = self.strRepr - self.anchor = expr - self.must_skip = must_skip - - def __add__(self, other): - skipper = SkipTo(other).setName("...")("_skipped*") - if self.must_skip: - def must_skip(t): - if not t._skipped or t._skipped.asList() == ['']: - del t[0] - t.pop("_skipped", None) - def show_skip(t): - if t._skipped.asList()[-1:] == ['']: - skipped = t.pop('_skipped') - t['_skipped'] = 'missing <' + repr(self.anchor) + '>' - return (self.anchor + skipper().addParseAction(must_skip) - | skipper().addParseAction(show_skip)) + other - - return self.anchor + skipper + other - - def __repr__(self): - return self.strRepr - - def parseImpl(self, *args): - raise Exception("use of `...` expression without following SkipTo target expression") - - -class Token(ParserElement): - """Abstract :class:`ParserElement` subclass, for defining atomic - matching patterns. - """ - def __init__(self): - super(Token, self).__init__(savelist=False) - - -class Empty(Token): - """An empty token, will always match. - """ - def __init__(self): - super(Empty, self).__init__() - self.name = "Empty" - self.mayReturnEmpty = True - self.mayIndexError = False - - -class NoMatch(Token): - """A token that will never match. - """ - def __init__(self): - super(NoMatch, self).__init__() - self.name = "NoMatch" - self.mayReturnEmpty = True - self.mayIndexError = False - self.errmsg = "Unmatchable token" - - def parseImpl(self, instring, loc, doActions=True): - raise ParseException(instring, loc, self.errmsg, self) - - -class Literal(Token): - """Token to exactly match a specified string. - - Example:: - - Literal('blah').parseString('blah') # -> ['blah'] - Literal('blah').parseString('blahfooblah') # -> ['blah'] - Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use :class:`CaselessLiteral`. - - For keyword matching (force word break before and after the matched string), - use :class:`Keyword` or :class:`CaselessKeyword`. - """ - def __init__(self, matchString): - super(Literal, self).__init__() - self.match = matchString - self.matchLen = len(matchString) - try: - self.firstMatchChar = matchString[0] - except IndexError: - warnings.warn("null string passed to Literal; use Empty() instead", - SyntaxWarning, stacklevel=2) - self.__class__ = Empty - self.name = '"%s"' % _ustr(self.match) - self.errmsg = "Expected " + self.name - self.mayReturnEmpty = False - self.mayIndexError = False - - # Performance tuning: modify __class__ to select - # a parseImpl optimized for single-character check - if self.matchLen == 1 and type(self) is Literal: - self.__class__ = _SingleCharLiteral - - def parseImpl(self, instring, loc, doActions=True): - if instring[loc] == self.firstMatchChar and instring.startswith(self.match, loc): - return loc + self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) - -class _SingleCharLiteral(Literal): - def parseImpl(self, instring, loc, doActions=True): - if instring[loc] == self.firstMatchChar: - return loc + 1, self.match - raise ParseException(instring, loc, self.errmsg, self) - -_L = Literal -ParserElement._literalStringClass = Literal - -class Keyword(Token): - """Token to exactly match a specified string as a keyword, that is, - it must be immediately followed by a non-keyword character. Compare - with :class:`Literal`: - - - ``Literal("if")`` will match the leading ``'if'`` in - ``'ifAndOnlyIf'``. - - ``Keyword("if")`` will not; it will only match the leading - ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` - - Accepts two optional constructor arguments in addition to the - keyword string: - - - ``identChars`` is a string of characters that would be valid - identifier characters, defaulting to all alphanumerics + "_" and - "$" - - ``caseless`` allows case-insensitive matching, default is ``False``. - - Example:: - - Keyword("start").parseString("start") # -> ['start'] - Keyword("start").parseString("starting") # -> Exception - - For case-insensitive matching, use :class:`CaselessKeyword`. - """ - DEFAULT_KEYWORD_CHARS = alphanums + "_$" - - def __init__(self, matchString, identChars=None, caseless=False): - super(Keyword, self).__init__() - if identChars is None: - identChars = Keyword.DEFAULT_KEYWORD_CHARS - self.match = matchString - self.matchLen = len(matchString) - try: - self.firstMatchChar = matchString[0] - except IndexError: - warnings.warn("null string passed to Keyword; use Empty() instead", - SyntaxWarning, stacklevel=2) - self.name = '"%s"' % self.match - self.errmsg = "Expected " + self.name - self.mayReturnEmpty = False - self.mayIndexError = False - self.caseless = caseless - if caseless: - self.caselessmatch = matchString.upper() - identChars = identChars.upper() - self.identChars = set(identChars) - - def parseImpl(self, instring, loc, doActions=True): - if self.caseless: - if ((instring[loc:loc + self.matchLen].upper() == self.caselessmatch) - and (loc >= len(instring) - self.matchLen - or instring[loc + self.matchLen].upper() not in self.identChars) - and (loc == 0 - or instring[loc - 1].upper() not in self.identChars)): - return loc + self.matchLen, self.match - - else: - if instring[loc] == self.firstMatchChar: - if ((self.matchLen == 1 or instring.startswith(self.match, loc)) - and (loc >= len(instring) - self.matchLen - or instring[loc + self.matchLen] not in self.identChars) - and (loc == 0 or instring[loc - 1] not in self.identChars)): - return loc + self.matchLen, self.match - - raise ParseException(instring, loc, self.errmsg, self) - - def copy(self): - c = super(Keyword, self).copy() - c.identChars = Keyword.DEFAULT_KEYWORD_CHARS - return c - - @staticmethod - def setDefaultKeywordChars(chars): - """Overrides the default Keyword chars - """ - Keyword.DEFAULT_KEYWORD_CHARS = chars - -class CaselessLiteral(Literal): - """Token to match a specified string, ignoring case of letters. - Note: the matched results will always be in the case of the given - match string, NOT the case of the input text. - - Example:: - - OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for :class:`CaselessKeyword`.) - """ - def __init__(self, matchString): - super(CaselessLiteral, self).__init__(matchString.upper()) - # Preserve the defining literal. - self.returnString = matchString - self.name = "'%s'" % self.returnString - self.errmsg = "Expected " + self.name - - def parseImpl(self, instring, loc, doActions=True): - if instring[loc:loc + self.matchLen].upper() == self.match: - return loc + self.matchLen, self.returnString - raise ParseException(instring, loc, self.errmsg, self) - -class CaselessKeyword(Keyword): - """ - Caseless version of :class:`Keyword`. - - Example:: - - OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for :class:`CaselessLiteral`.) - """ - def __init__(self, matchString, identChars=None): - super(CaselessKeyword, self).__init__(matchString, identChars, caseless=True) - -class CloseMatch(Token): - """A variation on :class:`Literal` which matches "close" matches, - that is, strings with at most 'n' mismatching characters. - :class:`CloseMatch` takes parameters: - - - ``match_string`` - string to be matched - - ``maxMismatches`` - (``default=1``) maximum number of - mismatches allowed to count as a match - - The results from a successful parse will contain the matched text - from the input string and the following named results: - - - ``mismatches`` - a list of the positions within the - match_string where mismatches were found - - ``original`` - the original match_string used to compare - against the input string - - If ``mismatches`` is an empty list, then the match was an exact - match. - - Example:: - - patt = CloseMatch("ATCATCGAATGGA") - patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) - patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) - - # exact match - patt.parseString("ATCATCGAATGGA") # -> (['ATCATCGAATGGA'], {'mismatches': [[]], 'original': ['ATCATCGAATGGA']}) - - # close match allowing up to 2 mismatches - patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2) - patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']}) - """ - def __init__(self, match_string, maxMismatches=1): - super(CloseMatch, self).__init__() - self.name = match_string - self.match_string = match_string - self.maxMismatches = maxMismatches - self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches) - self.mayIndexError = False - self.mayReturnEmpty = False - - def parseImpl(self, instring, loc, doActions=True): - start = loc - instrlen = len(instring) - maxloc = start + len(self.match_string) - - if maxloc <= instrlen: - match_string = self.match_string - match_stringloc = 0 - mismatches = [] - maxMismatches = self.maxMismatches - - for match_stringloc, s_m in enumerate(zip(instring[loc:maxloc], match_string)): - src, mat = s_m - if src != mat: - mismatches.append(match_stringloc) - if len(mismatches) > maxMismatches: - break - else: - loc = match_stringloc + 1 - results = ParseResults([instring[start:loc]]) - results['original'] = match_string - results['mismatches'] = mismatches - return loc, results - - raise ParseException(instring, loc, self.errmsg, self) - - -class Word(Token): - """Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, an - optional string containing allowed body characters (if omitted, - defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for ``min`` is - 1 (a minimum value < 1 is not valid); the default values for - ``max`` and ``exact`` are 0, meaning no maximum or exact - length restriction. An optional ``excludeChars`` parameter can - list characters that might be found in the input ``bodyChars`` - string; useful to define a word of all printables except for one or - two characters, for instance. - - :class:`srange` is useful for defining custom character set strings - for defining ``Word`` expressions, using range notation from - regular expression character sets. - - A common mistake is to use :class:`Word` to match a specific literal - string, as in ``Word("Address")``. Remember that :class:`Word` - uses the string argument to define *sets* of matchable characters. - This expression would match "Add", "AAA", "dAred", or any other word - made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an - exact literal string, use :class:`Literal` or :class:`Keyword`. - - pyparsing includes helper strings for building Words: - - - :class:`alphas` - - :class:`nums` - - :class:`alphanums` - - :class:`hexnums` - - :class:`alphas8bit` (alphabetic characters in ASCII range 128-255 - - accented, tilded, umlauted, etc.) - - :class:`punc8bit` (non-alphabetic characters in ASCII range - 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - :class:`printables` (any non-whitespace character) - - Example:: - - # a word composed of digits - integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - - # a word with a leading capital, and zero or more lowercase - capital_word = Word(alphas.upper(), alphas.lower()) - - # hostnames are alphanumeric, with leading alpha, and '-' - hostname = Word(alphas, alphanums + '-') - - # roman numeral (not a strict parser, accepts invalid mix of characters) - roman = Word("IVXLCDM") - - # any string of non-whitespace characters, except for ',' - csv_value = Word(printables, excludeChars=",") - """ - def __init__(self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None): - super(Word, self).__init__() - if excludeChars: - excludeChars = set(excludeChars) - initChars = ''.join(c for c in initChars if c not in excludeChars) - if bodyChars: - bodyChars = ''.join(c for c in bodyChars if c not in excludeChars) - self.initCharsOrig = initChars - self.initChars = set(initChars) - if bodyChars: - self.bodyCharsOrig = bodyChars - self.bodyChars = set(bodyChars) - else: - self.bodyCharsOrig = initChars - self.bodyChars = set(initChars) - - self.maxSpecified = max > 0 - - if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted") - - self.minLen = min - - if max > 0: - self.maxLen = max - else: - self.maxLen = _MAX_INT - - if exact > 0: - self.maxLen = exact - self.minLen = exact - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayIndexError = False - self.asKeyword = asKeyword - - if ' ' not in self.initCharsOrig + self.bodyCharsOrig and (min == 1 and max == 0 and exact == 0): - if self.bodyCharsOrig == self.initCharsOrig: - self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig) - elif len(self.initCharsOrig) == 1: - self.reString = "%s[%s]*" % (re.escape(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) - else: - self.reString = "[%s][%s]*" % (_escapeRegexRangeChars(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) - if self.asKeyword: - self.reString = r"\b" + self.reString + r"\b" - - try: - self.re = re.compile(self.reString) - except Exception: - self.re = None - else: - self.re_match = self.re.match - self.__class__ = _WordRegex - - def parseImpl(self, instring, loc, doActions=True): - if instring[loc] not in self.initChars: - raise ParseException(instring, loc, self.errmsg, self) - - start = loc - loc += 1 - instrlen = len(instring) - bodychars = self.bodyChars - maxloc = start + self.maxLen - maxloc = min(maxloc, instrlen) - while loc < maxloc and instring[loc] in bodychars: - loc += 1 - - throwException = False - if loc - start < self.minLen: - throwException = True - elif self.maxSpecified and loc < instrlen and instring[loc] in bodychars: - throwException = True - elif self.asKeyword: - if (start > 0 and instring[start - 1] in bodychars - or loc < instrlen and instring[loc] in bodychars): - throwException = True - - if throwException: - raise ParseException(instring, loc, self.errmsg, self) - - return loc, instring[start:loc] - - def __str__(self): - try: - return super(Word, self).__str__() - except Exception: - pass - - if self.strRepr is None: - - def charsAsStr(s): - if len(s) > 4: - return s[:4] + "..." - else: - return s - - if self.initCharsOrig != self.bodyCharsOrig: - self.strRepr = "W:(%s, %s)" % (charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig)) - else: - self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig) - - return self.strRepr - -class _WordRegex(Word): - def parseImpl(self, instring, loc, doActions=True): - result = self.re_match(instring, loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - return loc, result.group() - - -class Char(_WordRegex): - """A short-cut class for defining ``Word(characters, exact=1)``, - when defining a match of any single character in a string of - characters. - """ - def __init__(self, charset, asKeyword=False, excludeChars=None): - super(Char, self).__init__(charset, exact=1, asKeyword=asKeyword, excludeChars=excludeChars) - self.reString = "[%s]" % _escapeRegexRangeChars(''.join(self.initChars)) - if asKeyword: - self.reString = r"\b%s\b" % self.reString - self.re = re.compile(self.reString) - self.re_match = self.re.match - - -class Regex(Token): - r"""Token for matching strings that match a given regular - expression. Defined with string specifying the regular expression in - a form recognized by the stdlib Python `re module `_. - If the given regex contains named groups (defined using ``(?P...)``), - these will be preserved as named parse results. - - If instead of the Python stdlib re module you wish to use a different RE module - (such as the `regex` module), you can replace it by either building your - Regex object with a compiled RE that was compiled using regex: - - Example:: - - realnum = Regex(r"[+-]?\d+\.\d*") - date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)') - # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") - - # use regex module instead of stdlib re module to construct a Regex using - # a compiled regular expression - import regex - parser = pp.Regex(regex.compile(r'[0-9]')) - - """ - def __init__(self, pattern, flags=0, asGroupList=False, asMatch=False): - """The parameters ``pattern`` and ``flags`` are passed - to the ``re.compile()`` function as-is. See the Python - `re module `_ module for an - explanation of the acceptable patterns and flags. - """ - super(Regex, self).__init__() - - if isinstance(pattern, basestring): - if not pattern: - warnings.warn("null string passed to Regex; use Empty() instead", - SyntaxWarning, stacklevel=2) - - self.pattern = pattern - self.flags = flags - - try: - self.re = re.compile(self.pattern, self.flags) - self.reString = self.pattern - except sre_constants.error: - warnings.warn("invalid pattern (%s) passed to Regex" % pattern, - SyntaxWarning, stacklevel=2) - raise - - elif hasattr(pattern, 'pattern') and hasattr(pattern, 'match'): - self.re = pattern - self.pattern = self.reString = pattern.pattern - self.flags = flags - - else: - raise TypeError("Regex may only be constructed with a string or a compiled RE object") - - self.re_match = self.re.match - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayIndexError = False - self.mayReturnEmpty = self.re_match("") is not None - self.asGroupList = asGroupList - self.asMatch = asMatch - if self.asGroupList: - self.parseImpl = self.parseImplAsGroupList - if self.asMatch: - self.parseImpl = self.parseImplAsMatch - - def parseImpl(self, instring, loc, doActions=True): - result = self.re_match(instring, loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - ret = ParseResults(result.group()) - d = result.groupdict() - if d: - for k, v in d.items(): - ret[k] = v - return loc, ret - - def parseImplAsGroupList(self, instring, loc, doActions=True): - result = self.re_match(instring, loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - ret = result.groups() - return loc, ret - - def parseImplAsMatch(self, instring, loc, doActions=True): - result = self.re_match(instring, loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - ret = result - return loc, ret - - def __str__(self): - try: - return super(Regex, self).__str__() - except Exception: - pass - - if self.strRepr is None: - self.strRepr = "Re:(%s)" % repr(self.pattern) - - return self.strRepr - - def sub(self, repl): - r""" - Return Regex with an attached parse action to transform the parsed - result as if called using `re.sub(expr, repl, string) `_. - - Example:: - - make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2") - print(make_html.transformString("h1:main title:")) - # prints "

main title

" - """ - if self.asGroupList: - warnings.warn("cannot use sub() with Regex(asGroupList=True)", - SyntaxWarning, stacklevel=2) - raise SyntaxError() - - if self.asMatch and callable(repl): - warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", - SyntaxWarning, stacklevel=2) - raise SyntaxError() - - if self.asMatch: - def pa(tokens): - return tokens[0].expand(repl) - else: - def pa(tokens): - return self.re.sub(repl, tokens[0]) - return self.addParseAction(pa) - -class QuotedString(Token): - r""" - Token for matching strings that are delimited by quoting characters. - - Defined with the following parameters: - - - quoteChar - string of one or more characters defining the - quote delimiting string - - escChar - character to escape quotes, typically backslash - (default= ``None``) - - escQuote - special quote sequence to escape an embedded quote - string (such as SQL's ``""`` to escape an embedded ``"``) - (default= ``None``) - - multiline - boolean indicating whether quotes can span - multiple lines (default= ``False``) - - unquoteResults - boolean indicating whether the matched text - should be unquoted (default= ``True``) - - endQuoteChar - string of one or more characters defining the - end of the quote delimited string (default= ``None`` => same as - quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace - (``'\t'``, ``'\n'``, etc.) to actual whitespace - (default= ``True``) - - Example:: - - qs = QuotedString('"') - print(qs.searchString('lsjdf "This is the quote" sldjf')) - complex_qs = QuotedString('{{', endQuoteChar='}}') - print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) - sql_qs = QuotedString('"', escQuote='""') - print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) - - prints:: - - [['This is the quote']] - [['This is the "quote"']] - [['This is the quote with "embedded" quotes']] - """ - def __init__(self, quoteChar, escChar=None, escQuote=None, multiline=False, - unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True): - super(QuotedString, self).__init__() - - # remove white space from quote chars - wont work anyway - quoteChar = quoteChar.strip() - if not quoteChar: - warnings.warn("quoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) - raise SyntaxError() - - if endQuoteChar is None: - endQuoteChar = quoteChar - else: - endQuoteChar = endQuoteChar.strip() - if not endQuoteChar: - warnings.warn("endQuoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) - raise SyntaxError() - - self.quoteChar = quoteChar - self.quoteCharLen = len(quoteChar) - self.firstQuoteChar = quoteChar[0] - self.endQuoteChar = endQuoteChar - self.endQuoteCharLen = len(endQuoteChar) - self.escChar = escChar - self.escQuote = escQuote - self.unquoteResults = unquoteResults - self.convertWhitespaceEscapes = convertWhitespaceEscapes - - if multiline: - self.flags = re.MULTILINE | re.DOTALL - self.pattern = r'%s(?:[^%s%s]' % (re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '')) - else: - self.flags = 0 - self.pattern = r'%s(?:[^%s\n\r%s]' % (re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '')) - if len(self.endQuoteChar) > 1: - self.pattern += ( - '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]), - _escapeRegexRangeChars(self.endQuoteChar[i])) - for i in range(len(self.endQuoteChar) - 1, 0, -1)) + ')') - - if escQuote: - self.pattern += (r'|(?:%s)' % re.escape(escQuote)) - if escChar: - self.pattern += (r'|(?:%s.)' % re.escape(escChar)) - self.escCharReplacePattern = re.escape(self.escChar) + "(.)" - self.pattern += (r')*%s' % re.escape(self.endQuoteChar)) - - try: - self.re = re.compile(self.pattern, self.flags) - self.reString = self.pattern - self.re_match = self.re.match - except sre_constants.error: - warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern, - SyntaxWarning, stacklevel=2) - raise - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayIndexError = False - self.mayReturnEmpty = True - - def parseImpl(self, instring, loc, doActions=True): - result = instring[loc] == self.firstQuoteChar and self.re_match(instring, loc) or None - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - ret = result.group() - - if self.unquoteResults: - - # strip off quotes - ret = ret[self.quoteCharLen: -self.endQuoteCharLen] - - if isinstance(ret, basestring): - # replace escaped whitespace - if '\\' in ret and self.convertWhitespaceEscapes: - ws_map = { - r'\t': '\t', - r'\n': '\n', - r'\f': '\f', - r'\r': '\r', - } - for wslit, wschar in ws_map.items(): - ret = ret.replace(wslit, wschar) - - # replace escaped characters - if self.escChar: - ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret) - - # replace escaped quotes - if self.escQuote: - ret = ret.replace(self.escQuote, self.endQuoteChar) - - return loc, ret - - def __str__(self): - try: - return super(QuotedString, self).__str__() - except Exception: - pass - - if self.strRepr is None: - self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar) - - return self.strRepr - - -class CharsNotIn(Token): - """Token for matching words composed of characters *not* in a given - set (will include whitespace in matched characters if not listed in - the provided exclusion set - see example). Defined with string - containing all disallowed characters, and an optional minimum, - maximum, and/or exact length. The default value for ``min`` is - 1 (a minimum value < 1 is not valid); the default values for - ``max`` and ``exact`` are 0, meaning no maximum or exact - length restriction. - - Example:: - - # define a comma-separated-value as anything that is not a ',' - csv_value = CharsNotIn(',') - print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) - - prints:: - - ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] - """ - def __init__(self, notChars, min=1, max=0, exact=0): - super(CharsNotIn, self).__init__() - self.skipWhitespace = False - self.notChars = notChars - - if min < 1: - raise ValueError("cannot specify a minimum length < 1; use " - "Optional(CharsNotIn()) if zero-length char group is permitted") - - self.minLen = min - - if max > 0: - self.maxLen = max - else: - self.maxLen = _MAX_INT - - if exact > 0: - self.maxLen = exact - self.minLen = exact - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayReturnEmpty = (self.minLen == 0) - self.mayIndexError = False - - def parseImpl(self, instring, loc, doActions=True): - if instring[loc] in self.notChars: - raise ParseException(instring, loc, self.errmsg, self) - - start = loc - loc += 1 - notchars = self.notChars - maxlen = min(start + self.maxLen, len(instring)) - while loc < maxlen and instring[loc] not in notchars: - loc += 1 - - if loc - start < self.minLen: - raise ParseException(instring, loc, self.errmsg, self) - - return loc, instring[start:loc] - - def __str__(self): - try: - return super(CharsNotIn, self).__str__() - except Exception: - pass - - if self.strRepr is None: - if len(self.notChars) > 4: - self.strRepr = "!W:(%s...)" % self.notChars[:4] - else: - self.strRepr = "!W:(%s)" % self.notChars - - return self.strRepr - -class White(Token): - """Special matching class for matching whitespace. Normally, - whitespace is ignored by pyparsing grammars. This class is included - when some whitespace structures are significant. Define with - a string containing the whitespace characters to be matched; default - is ``" \\t\\r\\n"``. Also takes optional ``min``, - ``max``, and ``exact`` arguments, as defined for the - :class:`Word` class. - """ - whiteStrs = { - ' ' : '', - '\t': '', - '\n': '', - '\r': '', - '\f': '', - u'\u00A0': '', - u'\u1680': '', - u'\u180E': '', - u'\u2000': '', - u'\u2001': '', - u'\u2002': '', - u'\u2003': '', - u'\u2004': '', - u'\u2005': '', - u'\u2006': '', - u'\u2007': '', - u'\u2008': '', - u'\u2009': '', - u'\u200A': '', - u'\u200B': '', - u'\u202F': '', - u'\u205F': '', - u'\u3000': '', - } - def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): - super(White, self).__init__() - self.matchWhite = ws - self.setWhitespaceChars("".join(c for c in self.whiteChars if c not in self.matchWhite)) - # ~ self.leaveWhitespace() - self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite)) - self.mayReturnEmpty = True - self.errmsg = "Expected " + self.name - - self.minLen = min - - if max > 0: - self.maxLen = max - else: - self.maxLen = _MAX_INT - - if exact > 0: - self.maxLen = exact - self.minLen = exact - - def parseImpl(self, instring, loc, doActions=True): - if instring[loc] not in self.matchWhite: - raise ParseException(instring, loc, self.errmsg, self) - start = loc - loc += 1 - maxloc = start + self.maxLen - maxloc = min(maxloc, len(instring)) - while loc < maxloc and instring[loc] in self.matchWhite: - loc += 1 - - if loc - start < self.minLen: - raise ParseException(instring, loc, self.errmsg, self) - - return loc, instring[start:loc] - - -class _PositionToken(Token): - def __init__(self): - super(_PositionToken, self).__init__() - self.name = self.__class__.__name__ - self.mayReturnEmpty = True - self.mayIndexError = False - -class GoToColumn(_PositionToken): - """Token to advance to a specific column of input text; useful for - tabular report scraping. - """ - def __init__(self, colno): - super(GoToColumn, self).__init__() - self.col = colno - - def preParse(self, instring, loc): - if col(loc, instring) != self.col: - instrlen = len(instring) - if self.ignoreExprs: - loc = self._skipIgnorables(instring, loc) - while loc < instrlen and instring[loc].isspace() and col(loc, instring) != self.col: - loc += 1 - return loc - - def parseImpl(self, instring, loc, doActions=True): - thiscol = col(loc, instring) - if thiscol > self.col: - raise ParseException(instring, loc, "Text not in expected column", self) - newloc = loc + self.col - thiscol - ret = instring[loc: newloc] - return newloc, ret - - -class LineStart(_PositionToken): - r"""Matches if current position is at the beginning of a line within - the parse string - - Example:: - - test = '''\ - AAA this line - AAA and this line - AAA but not this one - B AAA and definitely not this one - ''' - - for t in (LineStart() + 'AAA' + restOfLine).searchString(test): - print(t) - - prints:: - - ['AAA', ' this line'] - ['AAA', ' and this line'] - - """ - def __init__(self): - super(LineStart, self).__init__() - self.errmsg = "Expected start of line" - - def parseImpl(self, instring, loc, doActions=True): - if col(loc, instring) == 1: - return loc, [] - raise ParseException(instring, loc, self.errmsg, self) - -class LineEnd(_PositionToken): - """Matches if current position is at the end of a line within the - parse string - """ - def __init__(self): - super(LineEnd, self).__init__() - self.setWhitespaceChars(ParserElement.DEFAULT_WHITE_CHARS.replace("\n", "")) - self.errmsg = "Expected end of line" - - def parseImpl(self, instring, loc, doActions=True): - if loc < len(instring): - if instring[loc] == "\n": - return loc + 1, "\n" - else: - raise ParseException(instring, loc, self.errmsg, self) - elif loc == len(instring): - return loc + 1, [] - else: - raise ParseException(instring, loc, self.errmsg, self) - -class StringStart(_PositionToken): - """Matches if current position is at the beginning of the parse - string - """ - def __init__(self): - super(StringStart, self).__init__() - self.errmsg = "Expected start of text" - - def parseImpl(self, instring, loc, doActions=True): - if loc != 0: - # see if entire string up to here is just whitespace and ignoreables - if loc != self.preParse(instring, 0): - raise ParseException(instring, loc, self.errmsg, self) - return loc, [] - -class StringEnd(_PositionToken): - """Matches if current position is at the end of the parse string - """ - def __init__(self): - super(StringEnd, self).__init__() - self.errmsg = "Expected end of text" - - def parseImpl(self, instring, loc, doActions=True): - if loc < len(instring): - raise ParseException(instring, loc, self.errmsg, self) - elif loc == len(instring): - return loc + 1, [] - elif loc > len(instring): - return loc, [] - else: - raise ParseException(instring, loc, self.errmsg, self) - -class WordStart(_PositionToken): - """Matches if the current position is at the beginning of a Word, - and is not preceded by any character in a given set of - ``wordChars`` (default= ``printables``). To emulate the - ``\b`` behavior of regular expressions, use - ``WordStart(alphanums)``. ``WordStart`` will also match at - the beginning of the string being parsed, or at the beginning of - a line. - """ - def __init__(self, wordChars=printables): - super(WordStart, self).__init__() - self.wordChars = set(wordChars) - self.errmsg = "Not at the start of a word" - - def parseImpl(self, instring, loc, doActions=True): - if loc != 0: - if (instring[loc - 1] in self.wordChars - or instring[loc] not in self.wordChars): - raise ParseException(instring, loc, self.errmsg, self) - return loc, [] - -class WordEnd(_PositionToken): - """Matches if the current position is at the end of a Word, and is - not followed by any character in a given set of ``wordChars`` - (default= ``printables``). To emulate the ``\b`` behavior of - regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` - will also match at the end of the string being parsed, or at the end - of a line. - """ - def __init__(self, wordChars=printables): - super(WordEnd, self).__init__() - self.wordChars = set(wordChars) - self.skipWhitespace = False - self.errmsg = "Not at the end of a word" - - def parseImpl(self, instring, loc, doActions=True): - instrlen = len(instring) - if instrlen > 0 and loc < instrlen: - if (instring[loc] in self.wordChars or - instring[loc - 1] not in self.wordChars): - raise ParseException(instring, loc, self.errmsg, self) - return loc, [] - - -class ParseExpression(ParserElement): - """Abstract subclass of ParserElement, for combining and - post-processing parsed tokens. - """ - def __init__(self, exprs, savelist=False): - super(ParseExpression, self).__init__(savelist) - if isinstance(exprs, _generatorType): - exprs = list(exprs) - - if isinstance(exprs, basestring): - self.exprs = [self._literalStringClass(exprs)] - elif isinstance(exprs, ParserElement): - self.exprs = [exprs] - elif isinstance(exprs, Iterable): - exprs = list(exprs) - # if sequence of strings provided, wrap with Literal - if any(isinstance(expr, basestring) for expr in exprs): - exprs = (self._literalStringClass(e) if isinstance(e, basestring) else e for e in exprs) - self.exprs = list(exprs) - else: - try: - self.exprs = list(exprs) - except TypeError: - self.exprs = [exprs] - self.callPreparse = False - - def append(self, other): - self.exprs.append(other) - self.strRepr = None - return self - - def leaveWhitespace(self): - """Extends ``leaveWhitespace`` defined in base class, and also invokes ``leaveWhitespace`` on - all contained expressions.""" - self.skipWhitespace = False - self.exprs = [e.copy() for e in self.exprs] - for e in self.exprs: - e.leaveWhitespace() - return self - - def ignore(self, other): - if isinstance(other, Suppress): - if other not in self.ignoreExprs: - super(ParseExpression, self).ignore(other) - for e in self.exprs: - e.ignore(self.ignoreExprs[-1]) - else: - super(ParseExpression, self).ignore(other) - for e in self.exprs: - e.ignore(self.ignoreExprs[-1]) - return self - - def __str__(self): - try: - return super(ParseExpression, self).__str__() - except Exception: - pass - - if self.strRepr is None: - self.strRepr = "%s:(%s)" % (self.__class__.__name__, _ustr(self.exprs)) - return self.strRepr - - def streamline(self): - super(ParseExpression, self).streamline() - - for e in self.exprs: - e.streamline() - - # collapse nested And's of the form And(And(And(a, b), c), d) to And(a, b, c, d) - # but only if there are no parse actions or resultsNames on the nested And's - # (likewise for Or's and MatchFirst's) - if len(self.exprs) == 2: - other = self.exprs[0] - if (isinstance(other, self.__class__) - and not other.parseAction - and other.resultsName is None - and not other.debug): - self.exprs = other.exprs[:] + [self.exprs[1]] - self.strRepr = None - self.mayReturnEmpty |= other.mayReturnEmpty - self.mayIndexError |= other.mayIndexError - - other = self.exprs[-1] - if (isinstance(other, self.__class__) - and not other.parseAction - and other.resultsName is None - and not other.debug): - self.exprs = self.exprs[:-1] + other.exprs[:] - self.strRepr = None - self.mayReturnEmpty |= other.mayReturnEmpty - self.mayIndexError |= other.mayIndexError - - self.errmsg = "Expected " + _ustr(self) - - return self - - def validate(self, validateTrace=None): - tmp = (validateTrace if validateTrace is not None else [])[:] + [self] - for e in self.exprs: - e.validate(tmp) - self.checkRecursion([]) - - def copy(self): - ret = super(ParseExpression, self).copy() - ret.exprs = [e.copy() for e in self.exprs] - return ret - - def _setResultsName(self, name, listAllMatches=False): - if __diag__.warn_ungrouped_named_tokens_in_collection: - for e in self.exprs: - if isinstance(e, ParserElement) and e.resultsName: - warnings.warn("{0}: setting results name {1!r} on {2} expression " - "collides with {3!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", - name, - type(self).__name__, - e.resultsName), - stacklevel=3) - - return super(ParseExpression, self)._setResultsName(name, listAllMatches) - - -class And(ParseExpression): - """ - Requires all given :class:`ParseExpression` s to be found in the given order. - Expressions may be separated by whitespace. - May be constructed using the ``'+'`` operator. - May also be constructed using the ``'-'`` operator, which will - suppress backtracking. - - Example:: - - integer = Word(nums) - name_expr = OneOrMore(Word(alphas)) - - expr = And([integer("id"), name_expr("name"), integer("age")]) - # more easily written as: - expr = integer("id") + name_expr("name") + integer("age") - """ - - class _ErrorStop(Empty): - def __init__(self, *args, **kwargs): - super(And._ErrorStop, self).__init__(*args, **kwargs) - self.name = '-' - self.leaveWhitespace() - - def __init__(self, exprs, savelist=True): - exprs = list(exprs) - if exprs and Ellipsis in exprs: - tmp = [] - for i, expr in enumerate(exprs): - if expr is Ellipsis: - if i < len(exprs) - 1: - skipto_arg = (Empty() + exprs[i + 1]).exprs[-1] - tmp.append(SkipTo(skipto_arg)("_skipped*")) - else: - raise Exception("cannot construct And with sequence ending in ...") - else: - tmp.append(expr) - exprs[:] = tmp - super(And, self).__init__(exprs, savelist) - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - self.setWhitespaceChars(self.exprs[0].whiteChars) - self.skipWhitespace = self.exprs[0].skipWhitespace - self.callPreparse = True - - def streamline(self): - # collapse any _PendingSkip's - if self.exprs: - if any(isinstance(e, ParseExpression) and e.exprs and isinstance(e.exprs[-1], _PendingSkip) - for e in self.exprs[:-1]): - for i, e in enumerate(self.exprs[:-1]): - if e is None: - continue - if (isinstance(e, ParseExpression) - and e.exprs and isinstance(e.exprs[-1], _PendingSkip)): - e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1] - self.exprs[i + 1] = None - self.exprs = [e for e in self.exprs if e is not None] - - super(And, self).streamline() - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - return self - - def parseImpl(self, instring, loc, doActions=True): - # pass False as last arg to _parse for first element, since we already - # pre-parsed the string as part of our And pre-parsing - loc, resultlist = self.exprs[0]._parse(instring, loc, doActions, callPreParse=False) - errorStop = False - for e in self.exprs[1:]: - if isinstance(e, And._ErrorStop): - errorStop = True - continue - if errorStop: - try: - loc, exprtokens = e._parse(instring, loc, doActions) - except ParseSyntaxException: - raise - except ParseBaseException as pe: - pe.__traceback__ = None - raise ParseSyntaxException._from_exception(pe) - except IndexError: - raise ParseSyntaxException(instring, len(instring), self.errmsg, self) - else: - loc, exprtokens = e._parse(instring, loc, doActions) - if exprtokens or exprtokens.haskeys(): - resultlist += exprtokens - return loc, resultlist - - def __iadd__(self, other): - if isinstance(other, basestring): - other = self._literalStringClass(other) - return self.append(other) # And([self, other]) - - def checkRecursion(self, parseElementList): - subRecCheckList = parseElementList[:] + [self] - for e in self.exprs: - e.checkRecursion(subRecCheckList) - if not e.mayReturnEmpty: - break - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + " ".join(_ustr(e) for e in self.exprs) + "}" - - return self.strRepr - - -class Or(ParseExpression): - """Requires that at least one :class:`ParseExpression` is found. If - two expressions match, the expression that matches the longest - string will be used. May be constructed using the ``'^'`` - operator. - - Example:: - - # construct Or using '^' operator - - number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) - print(number.searchString("123 3.1416 789")) - - prints:: - - [['123'], ['3.1416'], ['789']] - """ - def __init__(self, exprs, savelist=False): - super(Or, self).__init__(exprs, savelist) - if self.exprs: - self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) - else: - self.mayReturnEmpty = True - - def streamline(self): - super(Or, self).streamline() - if __compat__.collect_all_And_tokens: - self.saveAsList = any(e.saveAsList for e in self.exprs) - return self - - def parseImpl(self, instring, loc, doActions=True): - maxExcLoc = -1 - maxException = None - matches = [] - for e in self.exprs: - try: - loc2 = e.tryParse(instring, loc) - except ParseException as err: - err.__traceback__ = None - if err.loc > maxExcLoc: - maxException = err - maxExcLoc = err.loc - except IndexError: - if len(instring) > maxExcLoc: - maxException = ParseException(instring, len(instring), e.errmsg, self) - maxExcLoc = len(instring) - else: - # save match among all matches, to retry longest to shortest - matches.append((loc2, e)) - - if matches: - # re-evaluate all matches in descending order of length of match, in case attached actions - # might change whether or how much they match of the input. - matches.sort(key=itemgetter(0), reverse=True) - - if not doActions: - # no further conditions or parse actions to change the selection of - # alternative, so the first match will be the best match - best_expr = matches[0][1] - return best_expr._parse(instring, loc, doActions) - - longest = -1, None - for loc1, expr1 in matches: - if loc1 <= longest[0]: - # already have a longer match than this one will deliver, we are done - return longest - - try: - loc2, toks = expr1._parse(instring, loc, doActions) - except ParseException as err: - err.__traceback__ = None - if err.loc > maxExcLoc: - maxException = err - maxExcLoc = err.loc - else: - if loc2 >= loc1: - return loc2, toks - # didn't match as much as before - elif loc2 > longest[0]: - longest = loc2, toks - - if longest != (-1, None): - return longest - - if maxException is not None: - maxException.msg = self.errmsg - raise maxException - else: - raise ParseException(instring, loc, "no defined alternatives to match", self) - - - def __ixor__(self, other): - if isinstance(other, basestring): - other = self._literalStringClass(other) - return self.append(other) # Or([self, other]) - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}" - - return self.strRepr - - def checkRecursion(self, parseElementList): - subRecCheckList = parseElementList[:] + [self] - for e in self.exprs: - e.checkRecursion(subRecCheckList) - - def _setResultsName(self, name, listAllMatches=False): - if (not __compat__.collect_all_And_tokens - and __diag__.warn_multiple_tokens_in_named_alternation): - if any(isinstance(e, And) for e in self.exprs): - warnings.warn("{0}: setting results name {1!r} on {2} expression " - "may only return a single token for an And alternative, " - "in future will return the full list of tokens".format( - "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), - stacklevel=3) - - return super(Or, self)._setResultsName(name, listAllMatches) - - -class MatchFirst(ParseExpression): - """Requires that at least one :class:`ParseExpression` is found. If - two expressions match, the first one listed is the one that will - match. May be constructed using the ``'|'`` operator. - - Example:: - - # construct MatchFirst using '|' operator - - # watch the order of expressions to match - number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) - print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] - - # put more selective expression first - number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums) - print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']] - """ - def __init__(self, exprs, savelist=False): - super(MatchFirst, self).__init__(exprs, savelist) - if self.exprs: - self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) - else: - self.mayReturnEmpty = True - - def streamline(self): - super(MatchFirst, self).streamline() - if __compat__.collect_all_And_tokens: - self.saveAsList = any(e.saveAsList for e in self.exprs) - return self - - def parseImpl(self, instring, loc, doActions=True): - maxExcLoc = -1 - maxException = None - for e in self.exprs: - try: - ret = e._parse(instring, loc, doActions) - return ret - except ParseException as err: - if err.loc > maxExcLoc: - maxException = err - maxExcLoc = err.loc - except IndexError: - if len(instring) > maxExcLoc: - maxException = ParseException(instring, len(instring), e.errmsg, self) - maxExcLoc = len(instring) - - # only got here if no expression matched, raise exception for match that made it the furthest - else: - if maxException is not None: - maxException.msg = self.errmsg - raise maxException - else: - raise ParseException(instring, loc, "no defined alternatives to match", self) - - def __ior__(self, other): - if isinstance(other, basestring): - other = self._literalStringClass(other) - return self.append(other) # MatchFirst([self, other]) - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}" - - return self.strRepr - - def checkRecursion(self, parseElementList): - subRecCheckList = parseElementList[:] + [self] - for e in self.exprs: - e.checkRecursion(subRecCheckList) - - def _setResultsName(self, name, listAllMatches=False): - if (not __compat__.collect_all_And_tokens - and __diag__.warn_multiple_tokens_in_named_alternation): - if any(isinstance(e, And) for e in self.exprs): - warnings.warn("{0}: setting results name {1!r} on {2} expression " - "may only return a single token for an And alternative, " - "in future will return the full list of tokens".format( - "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), - stacklevel=3) - - return super(MatchFirst, self)._setResultsName(name, listAllMatches) - - -class Each(ParseExpression): - """Requires all given :class:`ParseExpression` s to be found, but in - any order. Expressions may be separated by whitespace. - - May be constructed using the ``'&'`` operator. - - Example:: - - color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") - shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") - integer = Word(nums) - shape_attr = "shape:" + shape_type("shape") - posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn") - color_attr = "color:" + color("color") - size_attr = "size:" + integer("size") - - # use Each (using operator '&') to accept attributes in any order - # (shape and posn are required, color and size are optional) - shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) - - shape_spec.runTests(''' - shape: SQUARE color: BLACK posn: 100, 120 - shape: CIRCLE size: 50 color: BLUE posn: 50,80 - color:GREEN size:20 shape:TRIANGLE posn:20,40 - ''' - ) - - prints:: - - shape: SQUARE color: BLACK posn: 100, 120 - ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - - color: BLACK - - posn: ['100', ',', '120'] - - x: 100 - - y: 120 - - shape: SQUARE - - - shape: CIRCLE size: 50 color: BLUE posn: 50,80 - ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', 'posn:', ['50', ',', '80']] - - color: BLUE - - posn: ['50', ',', '80'] - - x: 50 - - y: 80 - - shape: CIRCLE - - size: 50 - - - color: GREEN size: 20 shape: TRIANGLE posn: 20,40 - ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', 'posn:', ['20', ',', '40']] - - color: GREEN - - posn: ['20', ',', '40'] - - x: 20 - - y: 40 - - shape: TRIANGLE - - size: 20 - """ - def __init__(self, exprs, savelist=True): - super(Each, self).__init__(exprs, savelist) - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - self.skipWhitespace = True - self.initExprGroups = True - self.saveAsList = True - - def streamline(self): - super(Each, self).streamline() - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - return self - - def parseImpl(self, instring, loc, doActions=True): - if self.initExprGroups: - self.opt1map = dict((id(e.expr), e) for e in self.exprs if isinstance(e, Optional)) - opt1 = [e.expr for e in self.exprs if isinstance(e, Optional)] - opt2 = [e for e in self.exprs if e.mayReturnEmpty and not isinstance(e, (Optional, Regex))] - self.optionals = opt1 + opt2 - self.multioptionals = [e.expr for e in self.exprs if isinstance(e, ZeroOrMore)] - self.multirequired = [e.expr for e in self.exprs if isinstance(e, OneOrMore)] - self.required = [e for e in self.exprs if not isinstance(e, (Optional, ZeroOrMore, OneOrMore))] - self.required += self.multirequired - self.initExprGroups = False - tmpLoc = loc - tmpReqd = self.required[:] - tmpOpt = self.optionals[:] - matchOrder = [] - - keepMatching = True - while keepMatching: - tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired - failed = [] - for e in tmpExprs: - try: - tmpLoc = e.tryParse(instring, tmpLoc) - except ParseException: - failed.append(e) - else: - matchOrder.append(self.opt1map.get(id(e), e)) - if e in tmpReqd: - tmpReqd.remove(e) - elif e in tmpOpt: - tmpOpt.remove(e) - if len(failed) == len(tmpExprs): - keepMatching = False - - if tmpReqd: - missing = ", ".join(_ustr(e) for e in tmpReqd) - raise ParseException(instring, loc, "Missing one or more required elements (%s)" % missing) - - # add any unmatched Optionals, in case they have default values defined - matchOrder += [e for e in self.exprs if isinstance(e, Optional) and e.expr in tmpOpt] - - resultlist = [] - for e in matchOrder: - loc, results = e._parse(instring, loc, doActions) - resultlist.append(results) - - finalResults = sum(resultlist, ParseResults([])) - return loc, finalResults - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}" - - return self.strRepr - - def checkRecursion(self, parseElementList): - subRecCheckList = parseElementList[:] + [self] - for e in self.exprs: - e.checkRecursion(subRecCheckList) - - -class ParseElementEnhance(ParserElement): - """Abstract subclass of :class:`ParserElement`, for combining and - post-processing parsed tokens. - """ - def __init__(self, expr, savelist=False): - super(ParseElementEnhance, self).__init__(savelist) - if isinstance(expr, basestring): - if issubclass(self._literalStringClass, Token): - expr = self._literalStringClass(expr) - else: - expr = self._literalStringClass(Literal(expr)) - self.expr = expr - self.strRepr = None - if expr is not None: - self.mayIndexError = expr.mayIndexError - self.mayReturnEmpty = expr.mayReturnEmpty - self.setWhitespaceChars(expr.whiteChars) - self.skipWhitespace = expr.skipWhitespace - self.saveAsList = expr.saveAsList - self.callPreparse = expr.callPreparse - self.ignoreExprs.extend(expr.ignoreExprs) - - def parseImpl(self, instring, loc, doActions=True): - if self.expr is not None: - return self.expr._parse(instring, loc, doActions, callPreParse=False) - else: - raise ParseException("", loc, self.errmsg, self) - - def leaveWhitespace(self): - self.skipWhitespace = False - self.expr = self.expr.copy() - if self.expr is not None: - self.expr.leaveWhitespace() - return self - - def ignore(self, other): - if isinstance(other, Suppress): - if other not in self.ignoreExprs: - super(ParseElementEnhance, self).ignore(other) - if self.expr is not None: - self.expr.ignore(self.ignoreExprs[-1]) - else: - super(ParseElementEnhance, self).ignore(other) - if self.expr is not None: - self.expr.ignore(self.ignoreExprs[-1]) - return self - - def streamline(self): - super(ParseElementEnhance, self).streamline() - if self.expr is not None: - self.expr.streamline() - return self - - def checkRecursion(self, parseElementList): - if self in parseElementList: - raise RecursiveGrammarException(parseElementList + [self]) - subRecCheckList = parseElementList[:] + [self] - if self.expr is not None: - self.expr.checkRecursion(subRecCheckList) - - def validate(self, validateTrace=None): - if validateTrace is None: - validateTrace = [] - tmp = validateTrace[:] + [self] - if self.expr is not None: - self.expr.validate(tmp) - self.checkRecursion([]) - - def __str__(self): - try: - return super(ParseElementEnhance, self).__str__() - except Exception: - pass - - if self.strRepr is None and self.expr is not None: - self.strRepr = "%s:(%s)" % (self.__class__.__name__, _ustr(self.expr)) - return self.strRepr - - -class FollowedBy(ParseElementEnhance): - """Lookahead matching of the given parse expression. - ``FollowedBy`` does *not* advance the parsing position within - the input string, it only verifies that the specified parse - expression matches at the current position. ``FollowedBy`` - always returns a null token list. If any results names are defined - in the lookahead expression, those *will* be returned for access by - name. - - Example:: - - # use FollowedBy to match a label only if it is followed by a ':' - data_word = Word(alphas) - label = data_word + FollowedBy(':') - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - - OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() - - prints:: - - [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] - """ - def __init__(self, expr): - super(FollowedBy, self).__init__(expr) - self.mayReturnEmpty = True - - def parseImpl(self, instring, loc, doActions=True): - # by using self._expr.parse and deleting the contents of the returned ParseResults list - # we keep any named results that were defined in the FollowedBy expression - _, ret = self.expr._parse(instring, loc, doActions=doActions) - del ret[:] - - return loc, ret - - -class PrecededBy(ParseElementEnhance): - """Lookbehind matching of the given parse expression. - ``PrecededBy`` does not advance the parsing position within the - input string, it only verifies that the specified parse expression - matches prior to the current position. ``PrecededBy`` always - returns a null token list, but if a results name is defined on the - given expression, it is returned. - - Parameters: - - - expr - expression that must match prior to the current parse - location - - retreat - (default= ``None``) - (int) maximum number of characters - to lookbehind prior to the current parse location - - If the lookbehind expression is a string, Literal, Keyword, or - a Word or CharsNotIn with a specified exact or maximum length, then - the retreat parameter is not required. Otherwise, retreat must be - specified to give a maximum number of characters to look back from - the current parse position for a lookbehind match. - - Example:: - - # VB-style variable names with type prefixes - int_var = PrecededBy("#") + pyparsing_common.identifier - str_var = PrecededBy("$") + pyparsing_common.identifier - - """ - def __init__(self, expr, retreat=None): - super(PrecededBy, self).__init__(expr) - self.expr = self.expr().leaveWhitespace() - self.mayReturnEmpty = True - self.mayIndexError = False - self.exact = False - if isinstance(expr, str): - retreat = len(expr) - self.exact = True - elif isinstance(expr, (Literal, Keyword)): - retreat = expr.matchLen - self.exact = True - elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: - retreat = expr.maxLen - self.exact = True - elif isinstance(expr, _PositionToken): - retreat = 0 - self.exact = True - self.retreat = retreat - self.errmsg = "not preceded by " + str(expr) - self.skipWhitespace = False - self.parseAction.append(lambda s, l, t: t.__delitem__(slice(None, None))) - - def parseImpl(self, instring, loc=0, doActions=True): - if self.exact: - if loc < self.retreat: - raise ParseException(instring, loc, self.errmsg) - start = loc - self.retreat - _, ret = self.expr._parse(instring, start) - else: - # retreat specified a maximum lookbehind window, iterate - test_expr = self.expr + StringEnd() - instring_slice = instring[max(0, loc - self.retreat):loc] - last_expr = ParseException(instring, loc, self.errmsg) - for offset in range(1, min(loc, self.retreat + 1)+1): - try: - # print('trying', offset, instring_slice, repr(instring_slice[loc - offset:])) - _, ret = test_expr._parse(instring_slice, len(instring_slice) - offset) - except ParseBaseException as pbe: - last_expr = pbe - else: - break - else: - raise last_expr - return loc, ret - - -class NotAny(ParseElementEnhance): - """Lookahead to disallow matching with the given parse expression. - ``NotAny`` does *not* advance the parsing position within the - input string, it only verifies that the specified parse expression - does *not* match at the current position. Also, ``NotAny`` does - *not* skip over leading whitespace. ``NotAny`` always returns - a null token list. May be constructed using the '~' operator. - - Example:: - - AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) - - # take care not to mistake keywords for identifiers - ident = ~(AND | OR | NOT) + Word(alphas) - boolean_term = Optional(NOT) + ident - - # very crude boolean expression - to support parenthesis groups and - # operation hierarchy, use infixNotation - boolean_expr = boolean_term + ZeroOrMore((AND | OR) + boolean_term) - - # integers that are followed by "." are actually floats - integer = Word(nums) + ~Char(".") - """ - def __init__(self, expr): - super(NotAny, self).__init__(expr) - # ~ self.leaveWhitespace() - self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs - self.mayReturnEmpty = True - self.errmsg = "Found unwanted token, " + _ustr(self.expr) - - def parseImpl(self, instring, loc, doActions=True): - if self.expr.canParseNext(instring, loc): - raise ParseException(instring, loc, self.errmsg, self) - return loc, [] - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "~{" + _ustr(self.expr) + "}" - - return self.strRepr - -class _MultipleMatch(ParseElementEnhance): - def __init__(self, expr, stopOn=None): - super(_MultipleMatch, self).__init__(expr) - self.saveAsList = True - ender = stopOn - if isinstance(ender, basestring): - ender = self._literalStringClass(ender) - self.stopOn(ender) - - def stopOn(self, ender): - if isinstance(ender, basestring): - ender = self._literalStringClass(ender) - self.not_ender = ~ender if ender is not None else None - return self - - def parseImpl(self, instring, loc, doActions=True): - self_expr_parse = self.expr._parse - self_skip_ignorables = self._skipIgnorables - check_ender = self.not_ender is not None - if check_ender: - try_not_ender = self.not_ender.tryParse - - # must be at least one (but first see if we are the stopOn sentinel; - # if so, fail) - if check_ender: - try_not_ender(instring, loc) - loc, tokens = self_expr_parse(instring, loc, doActions, callPreParse=False) - try: - hasIgnoreExprs = (not not self.ignoreExprs) - while 1: - if check_ender: - try_not_ender(instring, loc) - if hasIgnoreExprs: - preloc = self_skip_ignorables(instring, loc) - else: - preloc = loc - loc, tmptokens = self_expr_parse(instring, preloc, doActions) - if tmptokens or tmptokens.haskeys(): - tokens += tmptokens - except (ParseException, IndexError): - pass - - return loc, tokens - - def _setResultsName(self, name, listAllMatches=False): - if __diag__.warn_ungrouped_named_tokens_in_collection: - for e in [self.expr] + getattr(self.expr, 'exprs', []): - if isinstance(e, ParserElement) and e.resultsName: - warnings.warn("{0}: setting results name {1!r} on {2} expression " - "collides with {3!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", - name, - type(self).__name__, - e.resultsName), - stacklevel=3) - - return super(_MultipleMatch, self)._setResultsName(name, listAllMatches) - - -class OneOrMore(_MultipleMatch): - """Repetition of one or more of the given expression. - - Parameters: - - expr - expression that must match one or more times - - stopOn - (default= ``None``) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) - - Example:: - - data_word = Word(alphas) - label = data_word + FollowedBy(':') - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) - - text = "shape: SQUARE posn: upper left color: BLACK" - OneOrMore(attr_expr).parseString(text).pprint() # Fail! read 'color' as data instead of next label -> [['shape', 'SQUARE color']] - - # use stopOn attribute for OneOrMore to avoid reading label string as part of the data - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - - # could also be written as - (attr_expr * (1,)).parseString(text).pprint() - """ - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + _ustr(self.expr) + "}..." - - return self.strRepr - -class ZeroOrMore(_MultipleMatch): - """Optional repetition of zero or more of the given expression. - - Parameters: - - expr - expression that must match zero or more times - - stopOn - (default= ``None``) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) - - Example: similar to :class:`OneOrMore` - """ - def __init__(self, expr, stopOn=None): - super(ZeroOrMore, self).__init__(expr, stopOn=stopOn) - self.mayReturnEmpty = True - - def parseImpl(self, instring, loc, doActions=True): - try: - return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) - except (ParseException, IndexError): - return loc, [] - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "[" + _ustr(self.expr) + "]..." - - return self.strRepr - - -class _NullToken(object): - def __bool__(self): - return False - __nonzero__ = __bool__ - def __str__(self): - return "" - -class Optional(ParseElementEnhance): - """Optional matching of the given expression. - - Parameters: - - expr - expression that must match zero or more times - - default (optional) - value to be returned if the optional expression is not found. - - Example:: - - # US postal code can be a 5-digit zip, plus optional 4-digit qualifier - zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) - zip.runTests(''' - # traditional ZIP code - 12345 - - # ZIP+4 form - 12101-0001 - - # invalid ZIP - 98765- - ''') - - prints:: - - # traditional ZIP code - 12345 - ['12345'] - - # ZIP+4 form - 12101-0001 - ['12101-0001'] - - # invalid ZIP - 98765- - ^ - FAIL: Expected end of text (at char 5), (line:1, col:6) - """ - __optionalNotMatched = _NullToken() - - def __init__(self, expr, default=__optionalNotMatched): - super(Optional, self).__init__(expr, savelist=False) - self.saveAsList = self.expr.saveAsList - self.defaultValue = default - self.mayReturnEmpty = True - - def parseImpl(self, instring, loc, doActions=True): - try: - loc, tokens = self.expr._parse(instring, loc, doActions, callPreParse=False) - except (ParseException, IndexError): - if self.defaultValue is not self.__optionalNotMatched: - if self.expr.resultsName: - tokens = ParseResults([self.defaultValue]) - tokens[self.expr.resultsName] = self.defaultValue - else: - tokens = [self.defaultValue] - else: - tokens = [] - return loc, tokens - - def __str__(self): - if hasattr(self, "name"): - return self.name - - if self.strRepr is None: - self.strRepr = "[" + _ustr(self.expr) + "]" - - return self.strRepr - -class SkipTo(ParseElementEnhance): - """Token for skipping over all undefined text until the matched - expression is found. - - Parameters: - - expr - target expression marking the end of the data to be skipped - - include - (default= ``False``) if True, the target expression is also parsed - (the skipped text and target expression are returned as a 2-element list). - - ignore - (default= ``None``) used to define grammars (typically quoted strings and - comments) that might contain false matches to the target expression - - failOn - (default= ``None``) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, - the SkipTo is not a match - - Example:: - - report = ''' - Outstanding Issues Report - 1 Jan 2000 - - # | Severity | Description | Days Open - -----+----------+-------------------------------------------+----------- - 101 | Critical | Intermittent system crash | 6 - 94 | Cosmetic | Spelling error on Login ('log|n') | 14 - 79 | Minor | System slow when running too many reports | 47 - ''' - integer = Word(nums) - SEP = Suppress('|') - # use SkipTo to simply match everything up until the next SEP - # - ignore quoted strings, so that a '|' character inside a quoted string does not match - # - parse action will call token.strip() for each matched token, i.e., the description body - string_data = SkipTo(SEP, ignore=quotedString) - string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP - + integer("days_open")) - - for tkt in ticket_expr.searchString(report): - print tkt.dump() - - prints:: - - ['101', 'Critical', 'Intermittent system crash', '6'] - - days_open: 6 - - desc: Intermittent system crash - - issue_num: 101 - - sev: Critical - ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14'] - - days_open: 14 - - desc: Spelling error on Login ('log|n') - - issue_num: 94 - - sev: Cosmetic - ['79', 'Minor', 'System slow when running too many reports', '47'] - - days_open: 47 - - desc: System slow when running too many reports - - issue_num: 79 - - sev: Minor - """ - def __init__(self, other, include=False, ignore=None, failOn=None): - super(SkipTo, self).__init__(other) - self.ignoreExpr = ignore - self.mayReturnEmpty = True - self.mayIndexError = False - self.includeMatch = include - self.saveAsList = False - if isinstance(failOn, basestring): - self.failOn = self._literalStringClass(failOn) - else: - self.failOn = failOn - self.errmsg = "No match found for " + _ustr(self.expr) - - def parseImpl(self, instring, loc, doActions=True): - startloc = loc - instrlen = len(instring) - expr = self.expr - expr_parse = self.expr._parse - self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None - self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - - tmploc = loc - while tmploc <= instrlen: - if self_failOn_canParseNext is not None: - # break if failOn expression matches - if self_failOn_canParseNext(instring, tmploc): - break - - if self_ignoreExpr_tryParse is not None: - # advance past ignore expressions - while 1: - try: - tmploc = self_ignoreExpr_tryParse(instring, tmploc) - except ParseBaseException: - break - - try: - expr_parse(instring, tmploc, doActions=False, callPreParse=False) - except (ParseException, IndexError): - # no match, advance loc in string - tmploc += 1 - else: - # matched skipto expr, done - break - - else: - # ran off the end of the input string without matching skipto expr, fail - raise ParseException(instring, loc, self.errmsg, self) - - # build up return values - loc = tmploc - skiptext = instring[startloc:loc] - skipresult = ParseResults(skiptext) - - if self.includeMatch: - loc, mat = expr_parse(instring, loc, doActions, callPreParse=False) - skipresult += mat - - return loc, skipresult - -class Forward(ParseElementEnhance): - """Forward declaration of an expression to be defined later - - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the ``Forward`` - variable using the '<<' operator. - - Note: take care when assigning to ``Forward`` not to overlook - precedence of operators. - - Specifically, '|' has a lower precedence than '<<', so that:: - - fwdExpr << a | b | c - - will actually be evaluated as:: - - (fwdExpr << a) | b | c - - thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the ``Forward``:: - - fwdExpr << (a | b | c) - - Converting to use the '<<=' operator instead will avoid this problem. - - See :class:`ParseResults.pprint` for an example of a recursive - parser created using ``Forward``. - """ - def __init__(self, other=None): - super(Forward, self).__init__(other, savelist=False) - - def __lshift__(self, other): - if isinstance(other, basestring): - other = self._literalStringClass(other) - self.expr = other - self.strRepr = None - self.mayIndexError = self.expr.mayIndexError - self.mayReturnEmpty = self.expr.mayReturnEmpty - self.setWhitespaceChars(self.expr.whiteChars) - self.skipWhitespace = self.expr.skipWhitespace - self.saveAsList = self.expr.saveAsList - self.ignoreExprs.extend(self.expr.ignoreExprs) - return self - - def __ilshift__(self, other): - return self << other - - def leaveWhitespace(self): - self.skipWhitespace = False - return self - - def streamline(self): - if not self.streamlined: - self.streamlined = True - if self.expr is not None: - self.expr.streamline() - return self - - def validate(self, validateTrace=None): - if validateTrace is None: - validateTrace = [] - - if self not in validateTrace: - tmp = validateTrace[:] + [self] - if self.expr is not None: - self.expr.validate(tmp) - self.checkRecursion([]) - - def __str__(self): - if hasattr(self, "name"): - return self.name - if self.strRepr is not None: - return self.strRepr - - # Avoid infinite recursion by setting a temporary strRepr - self.strRepr = ": ..." - - # Use the string representation of main expression. - retString = '...' - try: - if self.expr is not None: - retString = _ustr(self.expr)[:1000] - else: - retString = "None" - finally: - self.strRepr = self.__class__.__name__ + ": " + retString - return self.strRepr - - def copy(self): - if self.expr is not None: - return super(Forward, self).copy() - else: - ret = Forward() - ret <<= self - return ret - - def _setResultsName(self, name, listAllMatches=False): - if __diag__.warn_name_set_on_empty_Forward: - if self.expr is None: - warnings.warn("{0}: setting results name {0!r} on {1} expression " - "that has no contained expression".format("warn_name_set_on_empty_Forward", - name, - type(self).__name__), - stacklevel=3) - - return super(Forward, self)._setResultsName(name, listAllMatches) - -class TokenConverter(ParseElementEnhance): - """ - Abstract subclass of :class:`ParseExpression`, for converting parsed results. - """ - def __init__(self, expr, savelist=False): - super(TokenConverter, self).__init__(expr) # , savelist) - self.saveAsList = False - -class Combine(TokenConverter): - """Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the - input string; this can be disabled by specifying - ``'adjacent=False'`` in the constructor. - - Example:: - - real = Word(nums) + '.' + Word(nums) - print(real.parseString('3.1416')) # -> ['3', '.', '1416'] - # will also erroneously match the following - print(real.parseString('3. 1416')) # -> ['3', '.', '1416'] - - real = Combine(Word(nums) + '.' + Word(nums)) - print(real.parseString('3.1416')) # -> ['3.1416'] - # no match when there are internal spaces - print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...) - """ - def __init__(self, expr, joinString="", adjacent=True): - super(Combine, self).__init__(expr) - # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself - if adjacent: - self.leaveWhitespace() - self.adjacent = adjacent - self.skipWhitespace = True - self.joinString = joinString - self.callPreparse = True - - def ignore(self, other): - if self.adjacent: - ParserElement.ignore(self, other) - else: - super(Combine, self).ignore(other) - return self - - def postParse(self, instring, loc, tokenlist): - retToks = tokenlist.copy() - del retToks[:] - retToks += ParseResults(["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults) - - if self.resultsName and retToks.haskeys(): - return [retToks] - else: - return retToks - -class Group(TokenConverter): - """Converter to return the matched tokens as a list - useful for - returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. - - Example:: - - ident = Word(alphas) - num = Word(nums) - term = ident | num - func = ident + Optional(delimitedList(term)) - print(func.parseString("fn a, b, 100")) # -> ['fn', 'a', 'b', '100'] - - func = ident + Group(Optional(delimitedList(term))) - print(func.parseString("fn a, b, 100")) # -> ['fn', ['a', 'b', '100']] - """ - def __init__(self, expr): - super(Group, self).__init__(expr) - self.saveAsList = True - - def postParse(self, instring, loc, tokenlist): - return [tokenlist] - -class Dict(TokenConverter): - """Converter to return a repetitive expression as a list, but also - as a dictionary. Each element can also be referenced using the first - token in the expression as its key. Useful for tabular report - scraping when the first column can be used as a item key. - - Example:: - - data_word = Word(alphas) - label = data_word + FollowedBy(':') - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) - - text = "shape: SQUARE posn: upper left color: light blue texture: burlap" - attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - - # print attributes as plain groups - print(OneOrMore(attr_expr).parseString(text).dump()) - - # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names - result = Dict(OneOrMore(Group(attr_expr))).parseString(text) - print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - - prints:: - - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] - [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - - color: light blue - - posn: upper left - - shape: SQUARE - - texture: burlap - SQUARE - {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - - See more examples at :class:`ParseResults` of accessing fields by results name. - """ - def __init__(self, expr): - super(Dict, self).__init__(expr) - self.saveAsList = True - - def postParse(self, instring, loc, tokenlist): - for i, tok in enumerate(tokenlist): - if len(tok) == 0: - continue - ikey = tok[0] - if isinstance(ikey, int): - ikey = _ustr(tok[0]).strip() - if len(tok) == 1: - tokenlist[ikey] = _ParseResultsWithOffset("", i) - elif len(tok) == 2 and not isinstance(tok[1], ParseResults): - tokenlist[ikey] = _ParseResultsWithOffset(tok[1], i) - else: - dictvalue = tok.copy() # ParseResults(i) - del dictvalue[0] - if len(dictvalue) != 1 or (isinstance(dictvalue, ParseResults) and dictvalue.haskeys()): - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i) - else: - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i) - - if self.resultsName: - return [tokenlist] - else: - return tokenlist - - -class Suppress(TokenConverter): - """Converter for ignoring the results of a parsed expression. - - Example:: - - source = "a, b, c,d" - wd = Word(alphas) - wd_list1 = wd + ZeroOrMore(',' + wd) - print(wd_list1.parseString(source)) - - # often, delimiters that are useful during parsing are just in the - # way afterward - use Suppress to keep them out of the parsed output - wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) - print(wd_list2.parseString(source)) - - prints:: - - ['a', ',', 'b', ',', 'c', ',', 'd'] - ['a', 'b', 'c', 'd'] - - (See also :class:`delimitedList`.) - """ - def postParse(self, instring, loc, tokenlist): - return [] - - def suppress(self): - return self - - -class OnlyOnce(object): - """Wrapper for parse actions, to ensure they are only called once. - """ - def __init__(self, methodCall): - self.callable = _trim_arity(methodCall) - self.called = False - def __call__(self, s, l, t): - if not self.called: - results = self.callable(s, l, t) - self.called = True - return results - raise ParseException(s, l, "") - def reset(self): - self.called = False - -def traceParseAction(f): - """Decorator for debugging parse actions. - - When the parse action is called, this decorator will print - ``">> entering method-name(line:, , )"``. - When the parse action completes, the decorator will print - ``"<<"`` followed by the returned value, or any exception that the parse action raised. - - Example:: - - wd = Word(alphas) - - @traceParseAction - def remove_duplicate_chars(tokens): - return ''.join(sorted(set(''.join(tokens)))) - - wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) - print(wds.parseString("slkdjs sld sldd sdlf sdljf")) - - prints:: - - >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) - < 3: - thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc - sys.stderr.write(">>entering %s(line: '%s', %d, %r)\n" % (thisFunc, line(l, s), l, t)) - try: - ret = f(*paArgs) - except Exception as exc: - sys.stderr.write("< ['aa', 'bb', 'cc'] - delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] - """ - dlName = _ustr(expr) + " [" + _ustr(delim) + " " + _ustr(expr) + "]..." - if combine: - return Combine(expr + ZeroOrMore(delim + expr)).setName(dlName) - else: - return (expr + ZeroOrMore(Suppress(delim) + expr)).setName(dlName) - -def countedArray(expr, intExpr=None): - """Helper to define a counted list of expressions. - - This helper defines a pattern of the form:: - - integer expr expr expr... - - where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the - leading count token is suppressed. - - If ``intExpr`` is specified, it should be a pyparsing expression - that produces an integer value. - - Example:: - - countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] - - # in this parser, the leading integer value is given in binary, - # '10' indicating that 2 values are in the array - binaryConstant = Word('01').setParseAction(lambda t: int(t[0], 2)) - countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd'] - """ - arrayExpr = Forward() - def countFieldParseAction(s, l, t): - n = t[0] - arrayExpr << (n and Group(And([expr] * n)) or Group(empty)) - return [] - if intExpr is None: - intExpr = Word(nums).setParseAction(lambda t: int(t[0])) - else: - intExpr = intExpr.copy() - intExpr.setName("arrayLen") - intExpr.addParseAction(countFieldParseAction, callDuringTry=True) - return (intExpr + arrayExpr).setName('(len) ' + _ustr(expr) + '...') - -def _flatten(L): - ret = [] - for i in L: - if isinstance(i, list): - ret.extend(_flatten(i)) - else: - ret.append(i) - return ret - -def matchPreviousLiteral(expr): - """Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks for - a 'repeat' of a previous expression. For example:: - - first = Word(nums) - second = matchPreviousLiteral(first) - matchExpr = first + ":" + second - - will match ``"1:1"``, but not ``"1:2"``. Because this - matches a previous literal, will also match the leading - ``"1:1"`` in ``"1:10"``. If this is not desired, use - :class:`matchPreviousExpr`. Do *not* use with packrat parsing - enabled. - """ - rep = Forward() - def copyTokenToRepeater(s, l, t): - if t: - if len(t) == 1: - rep << t[0] - else: - # flatten t tokens - tflat = _flatten(t.asList()) - rep << And(Literal(tt) for tt in tflat) - else: - rep << Empty() - expr.addParseAction(copyTokenToRepeater, callDuringTry=True) - rep.setName('(prev) ' + _ustr(expr)) - return rep - -def matchPreviousExpr(expr): - """Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks for - a 'repeat' of a previous expression. For example:: - - first = Word(nums) - second = matchPreviousExpr(first) - matchExpr = first + ":" + second - - will match ``"1:1"``, but not ``"1:2"``. Because this - matches by expressions, will *not* match the leading ``"1:1"`` - in ``"1:10"``; the expressions are evaluated first, and then - compared, so ``"1"`` is compared with ``"10"``. Do *not* use - with packrat parsing enabled. - """ - rep = Forward() - e2 = expr.copy() - rep <<= e2 - def copyTokenToRepeater(s, l, t): - matchTokens = _flatten(t.asList()) - def mustMatchTheseTokens(s, l, t): - theseTokens = _flatten(t.asList()) - if theseTokens != matchTokens: - raise ParseException('', 0, '') - rep.setParseAction(mustMatchTheseTokens, callDuringTry=True) - expr.addParseAction(copyTokenToRepeater, callDuringTry=True) - rep.setName('(prev) ' + _ustr(expr)) - return rep - -def _escapeRegexRangeChars(s): - # ~ escape these chars: ^-[] - for c in r"\^-[]": - s = s.replace(c, _bslash + c) - s = s.replace("\n", r"\n") - s = s.replace("\t", r"\t") - return _ustr(s) - -def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): - """Helper to quickly define a set of alternative Literals, and makes - sure to do longest-first testing when there is a conflict, - regardless of the input order, but returns - a :class:`MatchFirst` for best performance. - - Parameters: - - - strs - a string of space-delimited literals, or a collection of - string literals - - caseless - (default= ``False``) - treat all literals as - caseless - - useRegex - (default= ``True``) - as an optimization, will - generate a Regex object; otherwise, will generate - a :class:`MatchFirst` object (if ``caseless=True`` or ``asKeyword=True``, or if - creating a :class:`Regex` raises an exception) - - asKeyword - (default=``False``) - enforce Keyword-style matching on the - generated expressions - - Example:: - - comp_oper = oneOf("< = > <= >= !=") - var = Word(alphas) - number = Word(nums) - term = var | number - comparison_expr = term + comp_oper + term - print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) - - prints:: - - [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] - """ - if isinstance(caseless, basestring): - warnings.warn("More than one string argument passed to oneOf, pass " - "choices as a list or space-delimited string", stacklevel=2) - - if caseless: - isequal = (lambda a, b: a.upper() == b.upper()) - masks = (lambda a, b: b.upper().startswith(a.upper())) - parseElementClass = CaselessKeyword if asKeyword else CaselessLiteral - else: - isequal = (lambda a, b: a == b) - masks = (lambda a, b: b.startswith(a)) - parseElementClass = Keyword if asKeyword else Literal - - symbols = [] - if isinstance(strs, basestring): - symbols = strs.split() - elif isinstance(strs, Iterable): - symbols = list(strs) - else: - warnings.warn("Invalid argument to oneOf, expected string or iterable", - SyntaxWarning, stacklevel=2) - if not symbols: - return NoMatch() - - if not asKeyword: - # if not producing keywords, need to reorder to take care to avoid masking - # longer choices with shorter ones - i = 0 - while i < len(symbols) - 1: - cur = symbols[i] - for j, other in enumerate(symbols[i + 1:]): - if isequal(other, cur): - del symbols[i + j + 1] - break - elif masks(cur, other): - del symbols[i + j + 1] - symbols.insert(i, other) - break - else: - i += 1 - - if not (caseless or asKeyword) and useRegex: - # ~ print (strs, "->", "|".join([_escapeRegexChars(sym) for sym in symbols])) - try: - if len(symbols) == len("".join(symbols)): - return Regex("[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols)).setName(' | '.join(symbols)) - else: - return Regex("|".join(re.escape(sym) for sym in symbols)).setName(' | '.join(symbols)) - except Exception: - warnings.warn("Exception creating Regex for oneOf, building MatchFirst", - SyntaxWarning, stacklevel=2) - - # last resort, just use MatchFirst - return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) - -def dictOf(key, value): - """Helper to easily and clearly define a dictionary by specifying - the respective patterns for the key and value. Takes care of - defining the :class:`Dict`, :class:`ZeroOrMore`, and - :class:`Group` tokens in the proper order. The key pattern - can include delimiting markers or punctuation, as long as they are - suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the :class:`Dict` results - can include named token fields. - - Example:: - - text = "shape: SQUARE posn: upper left color: light blue texture: burlap" - attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - print(OneOrMore(attr_expr).parseString(text).dump()) - - attr_label = label - attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) - - # similar to Dict, but simpler call format - result = dictOf(attr_label, attr_value).parseString(text) - print(result.dump()) - print(result['shape']) - print(result.shape) # object attribute access works too - print(result.asDict()) - - prints:: - - [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - - color: light blue - - posn: upper left - - shape: SQUARE - - texture: burlap - SQUARE - SQUARE - {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} - """ - return Dict(OneOrMore(Group(key + value))) - -def originalTextFor(expr, asString=True): - """Helper to return the original, untokenized text for a given - expression. Useful to restore the parsed fields of an HTML start - tag into the raw tag text itself, or to revert separate tokens with - intervening whitespace back to the original matching input text. By - default, returns astring containing the original parsed text. - - If the optional ``asString`` argument is passed as - ``False``, then the return value is - a :class:`ParseResults` containing any results names that - were originally matched, and a single token containing the original - matched text from the input string. So if the expression passed to - :class:`originalTextFor` contains expressions with defined - results names, you must set ``asString`` to ``False`` if you - want to preserve those results name values. - - Example:: - - src = "this is test bold text normal text " - for tag in ("b", "i"): - opener, closer = makeHTMLTags(tag) - patt = originalTextFor(opener + SkipTo(closer) + closer) - print(patt.searchString(src)[0]) - - prints:: - - [' bold text '] - ['text'] - """ - locMarker = Empty().setParseAction(lambda s, loc, t: loc) - endlocMarker = locMarker.copy() - endlocMarker.callPreparse = False - matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") - if asString: - extractText = lambda s, l, t: s[t._original_start: t._original_end] - else: - def extractText(s, l, t): - t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]] - matchExpr.setParseAction(extractText) - matchExpr.ignoreExprs = expr.ignoreExprs - return matchExpr - -def ungroup(expr): - """Helper to undo pyparsing's default grouping of And expressions, - even if all but one are non-empty. - """ - return TokenConverter(expr).addParseAction(lambda t: t[0]) - -def locatedExpr(expr): - """Helper to decorate a returned token with its starting and ending - locations in the input string. - - This helper adds the following results names: - - - locn_start = location where matched expression begins - - locn_end = location where matched expression ends - - value = the actual parsed results - - Be careful if the input text contains ```` characters, you - may want to call :class:`ParserElement.parseWithTabs` - - Example:: - - wd = Word(alphas) - for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): - print(match) - - prints:: - - [[0, 'ljsdf', 5]] - [[8, 'lksdjjf', 15]] - [[18, 'lkkjj', 23]] - """ - locator = Empty().setParseAction(lambda s, l, t: l) - return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end")) - - -# convenience constants for positional expressions -empty = Empty().setName("empty") -lineStart = LineStart().setName("lineStart") -lineEnd = LineEnd().setName("lineEnd") -stringStart = StringStart().setName("stringStart") -stringEnd = StringEnd().setName("stringEnd") - -_escapedPunc = Word(_bslash, r"\[]-*.$+^?()~ ", exact=2).setParseAction(lambda s, l, t: t[0][1]) -_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s, l, t: unichr(int(t[0].lstrip(r'\0x'), 16))) -_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s, l, t: unichr(int(t[0][1:], 8))) -_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) -_charRange = Group(_singleChar + Suppress("-") + _singleChar) -_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group(OneOrMore(_charRange | _singleChar)).setResultsName("body") + "]" - -def srange(s): - r"""Helper to easily define string ranges for use in Word - construction. Borrows syntax from regexp '[]' string range - definitions:: - - srange("[0-9]") -> "0123456789" - srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" - srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - - The input string must be enclosed in []'s, and the returned string - is the expanded character set joined into a single string. The - values enclosed in the []'s may be: - - - a single character - - an escaped character with a leading backslash (such as ``\-`` - or ``\]``) - - an escaped hex character with a leading ``'\x'`` - (``\x21``, which is a ``'!'`` character) (``\0x##`` - is also supported for backwards compatibility) - - an escaped octal character with a leading ``'\0'`` - (``\041``, which is a ``'!'`` character) - - a range of any of the above, separated by a dash (``'a-z'``, - etc.) - - any combination of the above (``'aeiouy'``, - ``'a-zA-Z0-9_$'``, etc.) - """ - _expanded = lambda p: p if not isinstance(p, ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) - try: - return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body) - except Exception: - return "" - -def matchOnlyAtCol(n): - """Helper method for defining parse actions that require matching at - a specific column in the input text. - """ - def verifyCol(strg, locn, toks): - if col(locn, strg) != n: - raise ParseException(strg, locn, "matched token not at column %d" % n) - return verifyCol - -def replaceWith(replStr): - """Helper method for common parse actions that simply return - a literal value. Especially useful when used with - :class:`transformString` (). - - Example:: - - num = Word(nums).setParseAction(lambda toks: int(toks[0])) - na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) - term = na | num - - OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] - """ - return lambda s, l, t: [replStr] - -def removeQuotes(s, l, t): - """Helper parse action for removing quotation marks from parsed - quoted strings. - - Example:: - - # by default, quotation marks are included in parsed results - quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] - - # use removeQuotes to strip quotation marks from parsed results - quotedString.setParseAction(removeQuotes) - quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["Now is the Winter of our Discontent"] - """ - return t[0][1:-1] - -def tokenMap(func, *args): - """Helper to define a parse action by mapping a function to all - elements of a ParseResults list. If any additional args are passed, - they are forwarded to the given function as additional arguments - after the token, as in - ``hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))``, - which will convert the parsed data to an integer using base 16. - - Example (compare the last to example in :class:`ParserElement.transformString`:: - - hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) - hex_ints.runTests(''' - 00 11 22 aa FF 0a 0d 1a - ''') - - upperword = Word(alphas).setParseAction(tokenMap(str.upper)) - OneOrMore(upperword).runTests(''' - my kingdom for a horse - ''') - - wd = Word(alphas).setParseAction(tokenMap(str.title)) - OneOrMore(wd).setParseAction(' '.join).runTests(''' - now is the winter of our discontent made glorious summer by this sun of york - ''') - - prints:: - - 00 11 22 aa FF 0a 0d 1a - [0, 17, 34, 170, 255, 10, 13, 26] - - my kingdom for a horse - ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE'] - - now is the winter of our discontent made glorious summer by this sun of york - ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York'] - """ - def pa(s, l, t): - return [func(tokn, *args) for tokn in t] - - try: - func_name = getattr(func, '__name__', - getattr(func, '__class__').__name__) - except Exception: - func_name = str(func) - pa.__name__ = func_name - - return pa - -upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. -Deprecated in favor of :class:`pyparsing_common.upcaseTokens`""" - -downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. -Deprecated in favor of :class:`pyparsing_common.downcaseTokens`""" - -def _makeTags(tagStr, xml, - suppress_LT=Suppress("<"), - suppress_GT=Suppress(">")): - """Internal helper to construct opening and closing tag expressions, given a tag name""" - if isinstance(tagStr, basestring): - resname = tagStr - tagStr = Keyword(tagStr, caseless=not xml) - else: - resname = tagStr.name - - tagAttrName = Word(alphas, alphanums + "_-:") - if xml: - tagAttrValue = dblQuotedString.copy().setParseAction(removeQuotes) - openTag = (suppress_LT - + tagStr("tag") - + Dict(ZeroOrMore(Group(tagAttrName + Suppress("=") + tagAttrValue))) - + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') - + suppress_GT) - else: - tagAttrValue = quotedString.copy().setParseAction(removeQuotes) | Word(printables, excludeChars=">") - openTag = (suppress_LT - + tagStr("tag") - + Dict(ZeroOrMore(Group(tagAttrName.setParseAction(downcaseTokens) - + Optional(Suppress("=") + tagAttrValue)))) - + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') - + suppress_GT) - closeTag = Combine(_L("", adjacent=False) - - openTag.setName("<%s>" % resname) - # add start results name in parse action now that ungrouped names are not reported at two levels - openTag.addParseAction(lambda t: t.__setitem__("start" + "".join(resname.replace(":", " ").title().split()), t.copy())) - closeTag = closeTag("end" + "".join(resname.replace(":", " ").title().split())).setName("" % resname) - openTag.tag = resname - closeTag.tag = resname - openTag.tag_body = SkipTo(closeTag()) - return openTag, closeTag - -def makeHTMLTags(tagStr): - """Helper to construct opening and closing tag expressions for HTML, - given a tag name. Matches tags in either upper or lower case, - attributes with namespaces and with quoted or unquoted values. - - Example:: - - text = 'More info at the pyparsing wiki page' - # makeHTMLTags returns pyparsing expressions for the opening and - # closing tags as a 2-tuple - a, a_end = makeHTMLTags("A") - link_expr = a + SkipTo(a_end)("link_text") + a_end - - for link in link_expr.searchString(text): - # attributes in the tag (like "href" shown here) are - # also accessible as named results - print(link.link_text, '->', link.href) - - prints:: - - pyparsing -> https://github.com/pyparsing/pyparsing/wiki - """ - return _makeTags(tagStr, False) - -def makeXMLTags(tagStr): - """Helper to construct opening and closing tag expressions for XML, - given a tag name. Matches tags only in the given upper/lower case. - - Example: similar to :class:`makeHTMLTags` - """ - return _makeTags(tagStr, True) - -def withAttribute(*args, **attrDict): - """Helper to create a validating parse action to be used with start - tags created with :class:`makeXMLTags` or - :class:`makeHTMLTags`. Use ``withAttribute`` to qualify - a starting tag with a required attribute value, to avoid false - matches on common tags such as ```` or ``
``. - - Call ``withAttribute`` with a series of attribute names and - values. Specify the list of filter attributes names and values as: - - - keyword arguments, as in ``(align="right")``, or - - as an explicit dict with ``**`` operator, when an attribute - name is also a Python reserved word, as in ``**{"class":"Customer", "align":"right"}`` - - a list of name-value tuples, as in ``(("ns1:class", "Customer"), ("ns2:align", "right"))`` - - For attribute names with a namespace prefix, you must use the second - form. Attribute names are matched insensitive to upper/lower case. - - If just testing for ``class`` (with or without a namespace), use - :class:`withClass`. - - To verify that the attribute exists, but without specifying a value, - pass ``withAttribute.ANY_VALUE`` as the value. - - Example:: - - html = ''' -
- Some text -
1 4 0 1 0
-
1,3 2,3 1,1
-
this has no type
-
- - ''' - div,div_end = makeHTMLTags("div") - - # only match div tag having a type attribute with value "grid" - div_grid = div().setParseAction(withAttribute(type="grid")) - grid_expr = div_grid + SkipTo(div | div_end)("body") - for grid_header in grid_expr.searchString(html): - print(grid_header.body) - - # construct a match with any div tag having a type attribute, regardless of the value - div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) - div_expr = div_any_type + SkipTo(div | div_end)("body") - for div_header in div_expr.searchString(html): - print(div_header.body) - - prints:: - - 1 4 0 1 0 - - 1 4 0 1 0 - 1,3 2,3 1,1 - """ - if args: - attrs = args[:] - else: - attrs = attrDict.items() - attrs = [(k, v) for k, v in attrs] - def pa(s, l, tokens): - for attrName, attrValue in attrs: - if attrName not in tokens: - raise ParseException(s, l, "no matching attribute " + attrName) - if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue: - raise ParseException(s, l, "attribute '%s' has value '%s', must be '%s'" % - (attrName, tokens[attrName], attrValue)) - return pa -withAttribute.ANY_VALUE = object() - -def withClass(classname, namespace=''): - """Simplified version of :class:`withAttribute` when - matching on a div class - made difficult because ``class`` is - a reserved word in Python. - - Example:: - - html = ''' -
- Some text -
1 4 0 1 0
-
1,3 2,3 1,1
-
this <div> has no class
-
- - ''' - div,div_end = makeHTMLTags("div") - div_grid = div().setParseAction(withClass("grid")) - - grid_expr = div_grid + SkipTo(div | div_end)("body") - for grid_header in grid_expr.searchString(html): - print(grid_header.body) - - div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) - div_expr = div_any_type + SkipTo(div | div_end)("body") - for div_header in div_expr.searchString(html): - print(div_header.body) - - prints:: - - 1 4 0 1 0 - - 1 4 0 1 0 - 1,3 2,3 1,1 - """ - classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr: classname}) - -opAssoc = SimpleNamespace() -opAssoc.LEFT = object() -opAssoc.RIGHT = object() - -def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): - """Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary - or binary, left- or right-associative. Parse actions can also be - attached to operator expressions. The generated parser will also - recognize the use of parentheses to override operator precedences - (see example below). - - Note: if you define a deep operator list, you may see performance - issues when using infixNotation. See - :class:`ParserElement.enablePackrat` for a mechanism to potentially - improve your parser performance. - - Parameters: - - baseExpr - expression representing the most basic element for the - nested - - opList - list of tuples, one for each operator precedence level - in the expression grammar; each tuple is of the form ``(opExpr, - numTerms, rightLeftAssoc, parseAction)``, where: - - - opExpr is the pyparsing expression for the operator; may also - be a string, which will be converted to a Literal; if numTerms - is 3, opExpr is a tuple of two expressions, for the two - operators separating the 3 terms - - numTerms is the number of terms for this operator (must be 1, - 2, or 3) - - rightLeftAssoc is the indicator whether the operator is right - or left associative, using the pyparsing-defined constants - ``opAssoc.RIGHT`` and ``opAssoc.LEFT``. - - parseAction is the parse action to be associated with - expressions matching this operator expression (the parse action - tuple member may be omitted); if the parse action is passed - a tuple or list of functions, this is equivalent to calling - ``setParseAction(*fn)`` - (:class:`ParserElement.setParseAction`) - - lpar - expression for matching left-parentheses - (default= ``Suppress('(')``) - - rpar - expression for matching right-parentheses - (default= ``Suppress(')')``) - - Example:: - - # simple example of four-function arithmetic with ints and - # variable names - integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - - arith_expr = infixNotation(integer | varname, - [ - ('-', 1, opAssoc.RIGHT), - (oneOf('* /'), 2, opAssoc.LEFT), - (oneOf('+ -'), 2, opAssoc.LEFT), - ]) - - arith_expr.runTests(''' - 5+3*6 - (5+3)*6 - -2--11 - ''', fullDump=False) - - prints:: - - 5+3*6 - [[5, '+', [3, '*', 6]]] - - (5+3)*6 - [[[5, '+', 3], '*', 6]] - - -2--11 - [[['-', 2], '-', ['-', 11]]] - """ - # captive version of FollowedBy that does not do parse actions or capture results names - class _FB(FollowedBy): - def parseImpl(self, instring, loc, doActions=True): - self.expr.tryParse(instring, loc) - return loc, [] - - ret = Forward() - lastExpr = baseExpr | (lpar + ret + rpar) - for i, operDef in enumerate(opList): - opExpr, arity, rightLeftAssoc, pa = (operDef + (None, ))[:4] - termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr - if arity == 3: - if opExpr is None or len(opExpr) != 2: - raise ValueError( - "if numterms=3, opExpr must be a tuple or list of two expressions") - opExpr1, opExpr2 = opExpr - thisExpr = Forward().setName(termName) - if rightLeftAssoc == opAssoc.LEFT: - if arity == 1: - matchExpr = _FB(lastExpr + opExpr) + Group(lastExpr + OneOrMore(opExpr)) - elif arity == 2: - if opExpr is not None: - matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group(lastExpr + OneOrMore(opExpr + lastExpr)) - else: - matchExpr = _FB(lastExpr + lastExpr) + Group(lastExpr + OneOrMore(lastExpr)) - elif arity == 3: - matchExpr = (_FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) - + Group(lastExpr + OneOrMore(opExpr1 + lastExpr + opExpr2 + lastExpr))) - else: - raise ValueError("operator must be unary (1), binary (2), or ternary (3)") - elif rightLeftAssoc == opAssoc.RIGHT: - if arity == 1: - # try to avoid LR with this extra test - if not isinstance(opExpr, Optional): - opExpr = Optional(opExpr) - matchExpr = _FB(opExpr.expr + thisExpr) + Group(opExpr + thisExpr) - elif arity == 2: - if opExpr is not None: - matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group(lastExpr + OneOrMore(opExpr + thisExpr)) - else: - matchExpr = _FB(lastExpr + thisExpr) + Group(lastExpr + OneOrMore(thisExpr)) - elif arity == 3: - matchExpr = (_FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) - + Group(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr)) - else: - raise ValueError("operator must be unary (1), binary (2), or ternary (3)") - else: - raise ValueError("operator must indicate right or left associativity") - if pa: - if isinstance(pa, (tuple, list)): - matchExpr.setParseAction(*pa) - else: - matchExpr.setParseAction(pa) - thisExpr <<= (matchExpr.setName(termName) | lastExpr) - lastExpr = thisExpr - ret <<= lastExpr - return ret - -operatorPrecedence = infixNotation -"""(Deprecated) Former name of :class:`infixNotation`, will be -dropped in a future release.""" - -dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("string enclosed in double quotes") -sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("string enclosed in single quotes") -quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' - | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("quotedString using single or double quotes") -unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") - -def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """Helper method for defining nested lists enclosed in opening and - closing delimiters ("(" and ")" are the default). - - Parameters: - - opener - opening character for a nested list - (default= ``"("``); can also be a pyparsing expression - - closer - closing character for a nested list - (default= ``")"``); can also be a pyparsing expression - - content - expression for items within the nested lists - (default= ``None``) - - ignoreExpr - expression for ignoring opening and closing - delimiters (default= :class:`quotedString`) - - If an expression is not provided for the content argument, the - nested expression will capture all whitespace-delimited content - between delimiters as a list of separate values. - - Use the ``ignoreExpr`` argument to define expressions that may - contain opening or closing characters that should not be treated as - opening or closing characters for nesting, such as quotedString or - a comment expression. Specify multiple expressions using an - :class:`Or` or :class:`MatchFirst`. The default is - :class:`quotedString`, but if no expressions are to be ignored, then - pass ``None`` for this argument. - - Example:: - - data_type = oneOf("void int short long char float double") - decl_data_type = Combine(data_type + Optional(Word('*'))) - ident = Word(alphas+'_', alphanums+'_') - number = pyparsing_common.number - arg = Group(decl_data_type + ident) - LPAR, RPAR = map(Suppress, "()") - - code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - - c_function = (decl_data_type("type") - + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR - + code_body("body")) - c_function.ignore(cStyleComment) - - source_code = ''' - int is_odd(int x) { - return (x%2); - } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { - return (10+ord(hchar)-ord('A')); - } - } - ''' - for func in c_function.searchString(source_code): - print("%(name)s (%(type)s) args: %(args)s" % func) - - - prints:: - - is_odd (int) args: [['int', 'x']] - dec_to_hex (int) args: [['char', 'hchar']] - """ - if opener == closer: - raise ValueError("opening and closing strings cannot be the same") - if content is None: - if isinstance(opener, basestring) and isinstance(closer, basestring): - if len(opener) == 1 and len(closer) == 1: - if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr - + CharsNotIn(opener - + closer - + ParserElement.DEFAULT_WHITE_CHARS, exact=1) - ) - ).setParseAction(lambda t: t[0].strip())) - else: - content = (empty.copy() + CharsNotIn(opener - + closer - + ParserElement.DEFAULT_WHITE_CHARS - ).setParseAction(lambda t: t[0].strip())) - else: - if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr - + ~Literal(opener) - + ~Literal(closer) - + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) - ).setParseAction(lambda t: t[0].strip())) - else: - content = (Combine(OneOrMore(~Literal(opener) - + ~Literal(closer) - + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) - ).setParseAction(lambda t: t[0].strip())) - else: - raise ValueError("opening and closing arguments must be strings if no content expression is given") - ret = Forward() - if ignoreExpr is not None: - ret <<= Group(Suppress(opener) + ZeroOrMore(ignoreExpr | ret | content) + Suppress(closer)) - else: - ret <<= Group(Suppress(opener) + ZeroOrMore(ret | content) + Suppress(closer)) - ret.setName('nested %s%s expression' % (opener, closer)) - return ret - -def indentedBlock(blockStatementExpr, indentStack, indent=True): - """Helper method for defining space-delimited indentation blocks, - such as those used to define block statements in Python source code. - - Parameters: - - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single - grammar should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond - the current level; set to False for block of left-most - statements (default= ``True``) - - A valid block must contain at least one ``blockStatement``. - - Example:: - - data = ''' - def A(z): - A1 - B = 100 - G = A2 - A2 - A3 - B - def BB(a,b,c): - BB1 - def BBA(): - bba1 - bba2 - bba3 - C - D - def spam(x,y): - def eggs(z): - pass - ''' - - - indentStack = [1] - stmt = Forward() - - identifier = Word(alphas, alphanums) - funcDecl = ("def" + identifier + Group("(" + Optional(delimitedList(identifier)) + ")") + ":") - func_body = indentedBlock(stmt, indentStack) - funcDef = Group(funcDecl + func_body) - - rvalue = Forward() - funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")") - rvalue << (funcCall | identifier | Word(nums)) - assignment = Group(identifier + "=" + rvalue) - stmt << (funcDef | assignment | identifier) - - module_body = OneOrMore(stmt) - - parseTree = module_body.parseString(data) - parseTree.pprint() - - prints:: - - [['def', - 'A', - ['(', 'z', ')'], - ':', - [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]], - 'B', - ['def', - 'BB', - ['(', 'a', 'b', 'c', ')'], - ':', - [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]], - 'C', - 'D', - ['def', - 'spam', - ['(', 'x', 'y', ')'], - ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] - """ - backup_stack = indentStack[:] - - def reset_stack(): - indentStack[:] = backup_stack - - def checkPeerIndent(s, l, t): - if l >= len(s): return - curCol = col(l, s) - if curCol != indentStack[-1]: - if curCol > indentStack[-1]: - raise ParseException(s, l, "illegal nesting") - raise ParseException(s, l, "not a peer entry") - - def checkSubIndent(s, l, t): - curCol = col(l, s) - if curCol > indentStack[-1]: - indentStack.append(curCol) - else: - raise ParseException(s, l, "not a subentry") - - def checkUnindent(s, l, t): - if l >= len(s): return - curCol = col(l, s) - if not(indentStack and curCol in indentStack): - raise ParseException(s, l, "not an unindent") - if curCol < indentStack[-1]: - indentStack.pop() - - NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress(), stopOn=StringEnd()) - INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT') - PEER = Empty().setParseAction(checkPeerIndent).setName('') - UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT') - if indent: - smExpr = Group(Optional(NL) - + INDENT - + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL), stopOn=StringEnd()) - + UNDENT) - else: - smExpr = Group(Optional(NL) - + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL), stopOn=StringEnd()) - + UNDENT) - smExpr.setFailAction(lambda a, b, c, d: reset_stack()) - blockStatementExpr.ignore(_bslash + LineEnd()) - return smExpr.setName('indented block') - -alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") -punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") - -anyOpenTag, anyCloseTag = makeHTMLTags(Word(alphas, alphanums + "_:").setName('any tag')) -_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(), '><& "\'')) -commonHTMLEntity = Regex('&(?P' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity") -def replaceHTMLEntity(t): - """Helper parser action to replace common HTML entities with their special characters""" - return _htmlEntityMap.get(t.entity) - -# it's easy to get these comment structures wrong - they're very common, so may as well make them available -cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form ``/* ... */``" - -htmlComment = Regex(r"").setName("HTML comment") -"Comment of the form ````" - -restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") -dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form ``// ... (to end of line)``" - -cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/' | dblSlashComment).setName("C++ style comment") -"Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" - -javaStyleComment = cppStyleComment -"Same as :class:`cppStyleComment`" - -pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form ``# ... (to end of line)``" - -_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') - + Optional(Word(" \t") - + ~Literal(",") + ~LineEnd()))).streamline().setName("commaItem") -commaSeparatedList = delimitedList(Optional(quotedString.copy() | _commasepitem, default="")).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or -quoted strings, separated by commas. - -This expression is deprecated in favor of :class:`pyparsing_common.comma_separated_list`. -""" - -# some other useful expressions - using lower-case class name since we are really using this as a namespace -class pyparsing_common: - """Here are some common low-level expressions that may be useful in - jump-starting parser development: - - - numeric forms (:class:`integers`, :class:`reals`, - :class:`scientific notation`) - - common :class:`programming identifiers` - - network addresses (:class:`MAC`, - :class:`IPv4`, :class:`IPv6`) - - ISO8601 :class:`dates` and - :class:`datetime` - - :class:`UUID` - - :class:`comma-separated list` - - Parse actions: - - - :class:`convertToInteger` - - :class:`convertToFloat` - - :class:`convertToDate` - - :class:`convertToDatetime` - - :class:`stripHTMLTags` - - :class:`upcaseTokens` - - :class:`downcaseTokens` - - Example:: - - pyparsing_common.number.runTests(''' - # any int or real number, returned as the appropriate type - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - ''') - - pyparsing_common.fnumber.runTests(''' - # any int or real number, returned as float - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - ''') - - pyparsing_common.hex_integer.runTests(''' - # hex numbers - 100 - FF - ''') - - pyparsing_common.fraction.runTests(''' - # fractions - 1/2 - -3/4 - ''') - - pyparsing_common.mixed_integer.runTests(''' - # mixed fractions - 1 - 1/2 - -3/4 - 1-3/4 - ''') - - import uuid - pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) - pyparsing_common.uuid.runTests(''' - # uuid - 12345678-1234-5678-1234-567812345678 - ''') - - prints:: - - # any int or real number, returned as the appropriate type - 100 - [100] - - -100 - [-100] - - +100 - [100] - - 3.14159 - [3.14159] - - 6.02e23 - [6.02e+23] - - 1e-12 - [1e-12] - - # any int or real number, returned as float - 100 - [100.0] - - -100 - [-100.0] - - +100 - [100.0] - - 3.14159 - [3.14159] - - 6.02e23 - [6.02e+23] - - 1e-12 - [1e-12] - - # hex numbers - 100 - [256] - - FF - [255] - - # fractions - 1/2 - [0.5] - - -3/4 - [-0.75] - - # mixed fractions - 1 - [1] - - 1/2 - [0.5] - - -3/4 - [-0.75] - - 1-3/4 - [1.75] - - # uuid - 12345678-1234-5678-1234-567812345678 - [UUID('12345678-1234-5678-1234-567812345678')] - """ - - convertToInteger = tokenMap(int) - """ - Parse action for converting parsed integers to Python int - """ - - convertToFloat = tokenMap(float) - """ - Parse action for converting parsed numbers to Python float - """ - - integer = Word(nums).setName("integer").setParseAction(convertToInteger) - """expression that parses an unsigned integer, returns an int""" - - hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int, 16)) - """expression that parses a hexadecimal integer, returns an int""" - - signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger) - """expression that parses an integer with optional leading sign, returns an int""" - - fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction") - """fractional expression of an integer divided by an integer, returns a float""" - fraction.addParseAction(lambda t: t[0]/t[-1]) - - mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction") - """mixed integer of the form 'integer - fraction', with optional leading integer, returns float""" - mixed_integer.addParseAction(sum) - - real = Regex(r'[+-]?(?:\d+\.\d*|\.\d+)').setName("real number").setParseAction(convertToFloat) - """expression that parses a floating point number and returns a float""" - - sci_real = Regex(r'[+-]?(?:\d+(?:[eE][+-]?\d+)|(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional - scientific notation and returns a float""" - - # streamlining this expression makes the docs nicer-looking - number = (sci_real | real | signed_integer).streamline() - """any numeric expression, returns the corresponding Python type""" - - fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) - """any int or real number, returned as float""" - - identifier = Word(alphas + '_', alphanums + '_').setName("identifier") - """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - - ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (``0.0.0.0 - 255.255.255.255``)" - - _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") - _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part) * 7).setName("full IPv6 address") - _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) - + "::" - + Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) - ).setName("short IPv6 address") - _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8) - _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") - ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") - "IPv6 address (long, short, or mixed form)" - - mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") - "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" - - @staticmethod - def convertToDate(fmt="%Y-%m-%d"): - """ - Helper to create a parse action for converting parsed date string to Python datetime.date - - Params - - - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) - - Example:: - - date_expr = pyparsing_common.iso8601_date.copy() - date_expr.setParseAction(pyparsing_common.convertToDate()) - print(date_expr.parseString("1999-12-31")) - - prints:: - - [datetime.date(1999, 12, 31)] - """ - def cvt_fn(s, l, t): - try: - return datetime.strptime(t[0], fmt).date() - except ValueError as ve: - raise ParseException(s, l, str(ve)) - return cvt_fn - - @staticmethod - def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """Helper to create a parse action for converting parsed - datetime string to Python datetime.datetime - - Params - - - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) - - Example:: - - dt_expr = pyparsing_common.iso8601_datetime.copy() - dt_expr.setParseAction(pyparsing_common.convertToDatetime()) - print(dt_expr.parseString("1999-12-31T23:59:59.999")) - - prints:: - - [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] - """ - def cvt_fn(s, l, t): - try: - return datetime.strptime(t[0], fmt) - except ValueError as ve: - raise ParseException(s, l, str(ve)) - return cvt_fn - - iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (``yyyy-mm-dd``)" - - iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" - - uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" - - _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() - @staticmethod - def stripHTMLTags(s, l, tokens): - """Parse action to remove HTML tags from web page HTML source - - Example:: - - # strip HTML links from normal text - text = 'More info at the pyparsing wiki page' - td, td_end = makeHTMLTags("TD") - table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - print(table_text.parseString(text).body) - - Prints:: - - More info at the pyparsing wiki page - """ - return pyparsing_common._html_stripper.transformString(tokens[0]) - - _commasepitem = Combine(OneOrMore(~Literal(",") - + ~LineEnd() - + Word(printables, excludeChars=',') - + Optional(White(" \t")))).streamline().setName("commaItem") - comma_separated_list = delimitedList(Optional(quotedString.copy() - | _commasepitem, default='') - ).setName("comma separated list") - """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" - - upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper())) - """Parse action to convert tokens to upper case.""" - - downcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).lower())) - """Parse action to convert tokens to lower case.""" - - -class _lazyclassproperty(object): - def __init__(self, fn): - self.fn = fn - self.__doc__ = fn.__doc__ - self.__name__ = fn.__name__ - - def __get__(self, obj, cls): - if cls is None: - cls = type(obj) - if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) - for superclass in cls.__mro__[1:]): - cls._intern = {} - attrname = self.fn.__name__ - if attrname not in cls._intern: - cls._intern[attrname] = self.fn(cls) - return cls._intern[attrname] - - -class unicode_set(object): - """ - A set of Unicode characters, for language-specific strings for - ``alphas``, ``nums``, ``alphanums``, and ``printables``. - A unicode_set is defined by a list of ranges in the Unicode character - set, in a class attribute ``_ranges``, such as:: - - _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] - - A unicode set can also be defined using multiple inheritance of other unicode sets:: - - class CJK(Chinese, Japanese, Korean): - pass - """ - _ranges = [] - - @classmethod - def _get_chars_for_ranges(cls): - ret = [] - for cc in cls.__mro__: - if cc is unicode_set: - break - for rr in cc._ranges: - ret.extend(range(rr[0], rr[-1] + 1)) - return [unichr(c) for c in sorted(set(ret))] - - @_lazyclassproperty - def printables(cls): - "all non-whitespace characters in this range" - return u''.join(filterfalse(unicode.isspace, cls._get_chars_for_ranges())) - - @_lazyclassproperty - def alphas(cls): - "all alphabetic characters in this range" - return u''.join(filter(unicode.isalpha, cls._get_chars_for_ranges())) - - @_lazyclassproperty - def nums(cls): - "all numeric digit characters in this range" - return u''.join(filter(unicode.isdigit, cls._get_chars_for_ranges())) - - @_lazyclassproperty - def alphanums(cls): - "all alphanumeric characters in this range" - return cls.alphas + cls.nums - - -class pyparsing_unicode(unicode_set): - """ - A namespace class for defining common language unicode_sets. - """ - _ranges = [(32, sys.maxunicode)] - - class Latin1(unicode_set): - "Unicode set for Latin-1 Unicode Character Range" - _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] - - class LatinA(unicode_set): - "Unicode set for Latin-A Unicode Character Range" - _ranges = [(0x0100, 0x017f),] - - class LatinB(unicode_set): - "Unicode set for Latin-B Unicode Character Range" - _ranges = [(0x0180, 0x024f),] - - class Greek(unicode_set): - "Unicode set for Greek Unicode Character Ranges" - _ranges = [ - (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), - (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), - (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), - ] - - class Cyrillic(unicode_set): - "Unicode set for Cyrillic Unicode Character Range" - _ranges = [(0x0400, 0x04ff)] - - class Chinese(unicode_set): - "Unicode set for Chinese Unicode Character Range" - _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f),] - - class Japanese(unicode_set): - "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" - _ranges = [] - - class Kanji(unicode_set): - "Unicode set for Kanji Unicode Character Range" - _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f),] - - class Hiragana(unicode_set): - "Unicode set for Hiragana Unicode Character Range" - _ranges = [(0x3040, 0x309f),] - - class Katakana(unicode_set): - "Unicode set for Katakana Unicode Character Range" - _ranges = [(0x30a0, 0x30ff),] - - class Korean(unicode_set): - "Unicode set for Korean Unicode Character Range" - _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f),] - - class CJK(Chinese, Japanese, Korean): - "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" - pass - - class Thai(unicode_set): - "Unicode set for Thai Unicode Character Range" - _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b),] - - class Arabic(unicode_set): - "Unicode set for Arabic Unicode Character Range" - _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f),] - - class Hebrew(unicode_set): - "Unicode set for Hebrew Unicode Character Range" - _ranges = [(0x0590, 0x05ff),] - - class Devanagari(unicode_set): - "Unicode set for Devanagari Unicode Character Range" - _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] - -pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges - + pyparsing_unicode.Japanese.Hiragana._ranges - + pyparsing_unicode.Japanese.Katakana._ranges) - -# define ranges in language character sets -if PY_3: - setattr(pyparsing_unicode, u"العربية", pyparsing_unicode.Arabic) - setattr(pyparsing_unicode, u"中文", pyparsing_unicode.Chinese) - setattr(pyparsing_unicode, u"кириллица", pyparsing_unicode.Cyrillic) - setattr(pyparsing_unicode, u"Ελληνικά", pyparsing_unicode.Greek) - setattr(pyparsing_unicode, u"עִברִית", pyparsing_unicode.Hebrew) - setattr(pyparsing_unicode, u"日本語", pyparsing_unicode.Japanese) - setattr(pyparsing_unicode.Japanese, u"漢字", pyparsing_unicode.Japanese.Kanji) - setattr(pyparsing_unicode.Japanese, u"カタカナ", pyparsing_unicode.Japanese.Katakana) - setattr(pyparsing_unicode.Japanese, u"ひらがな", pyparsing_unicode.Japanese.Hiragana) - setattr(pyparsing_unicode, u"한국어", pyparsing_unicode.Korean) - setattr(pyparsing_unicode, u"ไทย", pyparsing_unicode.Thai) - setattr(pyparsing_unicode, u"देवनागरी", pyparsing_unicode.Devanagari) - - -class pyparsing_test: - """ - namespace class for classes useful in writing unit tests - """ - - class reset_pyparsing_context: - """ - Context manager to be used when writing unit tests that modify pyparsing config values: - - packrat parsing - - default whitespace characters. - - default keyword characters - - literal string auto-conversion class - - __diag__ settings - - Example: - with reset_pyparsing_context(): - # test that literals used to construct a grammar are automatically suppressed - ParserElement.inlineLiteralsUsing(Suppress) - - term = Word(alphas) | Word(nums) - group = Group('(' + term[...] + ')') - - # assert that the '()' characters are not included in the parsed tokens - self.assertParseAndCheckLisst(group, "(abc 123 def)", ['abc', '123', 'def']) - - # after exiting context manager, literals are converted to Literal expressions again - """ - - def __init__(self): - self._save_context = {} - - def save(self): - self._save_context["default_whitespace"] = ParserElement.DEFAULT_WHITE_CHARS - self._save_context["default_keyword_chars"] = Keyword.DEFAULT_KEYWORD_CHARS - self._save_context[ - "literal_string_class" - ] = ParserElement._literalStringClass - self._save_context["packrat_enabled"] = ParserElement._packratEnabled - self._save_context["packrat_parse"] = ParserElement._parse - self._save_context["__diag__"] = { - name: getattr(__diag__, name) for name in __diag__._all_names - } - self._save_context["__compat__"] = { - "collect_all_And_tokens": __compat__.collect_all_And_tokens - } - return self - - def restore(self): - # reset pyparsing global state - if ( - ParserElement.DEFAULT_WHITE_CHARS - != self._save_context["default_whitespace"] - ): - ParserElement.setDefaultWhitespaceChars( - self._save_context["default_whitespace"] - ) - Keyword.DEFAULT_KEYWORD_CHARS = self._save_context["default_keyword_chars"] - ParserElement.inlineLiteralsUsing( - self._save_context["literal_string_class"] - ) - for name, value in self._save_context["__diag__"].items(): - setattr(__diag__, name, value) - ParserElement._packratEnabled = self._save_context["packrat_enabled"] - ParserElement._parse = self._save_context["packrat_parse"] - __compat__.collect_all_And_tokens = self._save_context["__compat__"] - - def __enter__(self): - return self.save() - - def __exit__(self, *args): - return self.restore() - - class TestParseResultsAsserts: - """ - A mixin class to add parse results assertion methods to normal unittest.TestCase classes. - """ - def assertParseResultsEquals( - self, result, expected_list=None, expected_dict=None, msg=None - ): - """ - Unit test assertion to compare a ParseResults object with an optional expected_list, - and compare any defined results names with an optional expected_dict. - """ - if expected_list is not None: - self.assertEqual(expected_list, result.asList(), msg=msg) - if expected_dict is not None: - self.assertEqual(expected_dict, result.asDict(), msg=msg) - - def assertParseAndCheckList( - self, expr, test_string, expected_list, msg=None, verbose=True - ): - """ - Convenience wrapper assert to test a parser element and input string, and assert that - the resulting ParseResults.asList() is equal to the expected_list. - """ - result = expr.parseString(test_string, parseAll=True) - if verbose: - print(result.dump()) - self.assertParseResultsEquals(result, expected_list=expected_list, msg=msg) - - def assertParseAndCheckDict( - self, expr, test_string, expected_dict, msg=None, verbose=True - ): - """ - Convenience wrapper assert to test a parser element and input string, and assert that - the resulting ParseResults.asDict() is equal to the expected_dict. - """ - result = expr.parseString(test_string, parseAll=True) - if verbose: - print(result.dump()) - self.assertParseResultsEquals(result, expected_dict=expected_dict, msg=msg) - - def assertRunTestResults( - self, run_tests_report, expected_parse_results=None, msg=None - ): - """ - Unit test assertion to evaluate output of ParserElement.runTests(). If a list of - list-dict tuples is given as the expected_parse_results argument, then these are zipped - with the report tuples returned by runTests and evaluated using assertParseResultsEquals. - Finally, asserts that the overall runTests() success value is True. - - :param run_tests_report: tuple(bool, [tuple(str, ParseResults or Exception)]) returned from runTests - :param expected_parse_results (optional): [tuple(str, list, dict, Exception)] - """ - run_test_success, run_test_results = run_tests_report - - if expected_parse_results is not None: - merged = [ - (rpt[0], rpt[1], expected) - for rpt, expected in zip(run_test_results, expected_parse_results) - ] - for test_string, result, expected in merged: - # expected should be a tuple containing a list and/or a dict or an exception, - # and optional failure message string - # an empty tuple will skip any result validation - fail_msg = next( - (exp for exp in expected if isinstance(exp, str)), None - ) - expected_exception = next( - ( - exp - for exp in expected - if isinstance(exp, type) and issubclass(exp, Exception) - ), - None, - ) - if expected_exception is not None: - with self.assertRaises( - expected_exception=expected_exception, msg=fail_msg or msg - ): - if isinstance(result, Exception): - raise result - else: - expected_list = next( - (exp for exp in expected if isinstance(exp, list)), None - ) - expected_dict = next( - (exp for exp in expected if isinstance(exp, dict)), None - ) - if (expected_list, expected_dict) != (None, None): - self.assertParseResultsEquals( - result, - expected_list=expected_list, - expected_dict=expected_dict, - msg=fail_msg or msg, - ) - else: - # warning here maybe? - print("no validation for {!r}".format(test_string)) - - # do this last, in case some specific test results can be reported instead - self.assertTrue( - run_test_success, msg=msg if msg is not None else "failed runTests" - ) - - @contextmanager - def assertRaisesParseException(self, exc_type=ParseException, msg=None): - with self.assertRaises(exc_type, msg=msg): - yield - - -if __name__ == "__main__": - - selectToken = CaselessLiteral("select") - fromToken = CaselessLiteral("from") - - ident = Word(alphas, alphanums + "_$") - - columnName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) - columnNameList = Group(delimitedList(columnName)).setName("columns") - columnSpec = ('*' | columnNameList) - - tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) - tableNameList = Group(delimitedList(tableName)).setName("tables") - - simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") - - # demo runTests method, including embedded comments in test string - simpleSQL.runTests(""" - # '*' as column list and dotted table name - select * from SYS.XYZZY - - # caseless match on "SELECT", and casts back to "select" - SELECT * from XYZZY, ABC - - # list of column names, and mixed case SELECT keyword - Select AA,BB,CC from Sys.dual - - # multiple tables - Select A, B, C from Sys.dual, Table2 - - # invalid SELECT keyword - should fail - Xelect A, B, C from Sys.dual - - # incomplete command - should fail - Select - - # invalid column name - should fail - Select ^^^ frox Sys.dual - - """) - - pyparsing_common.number.runTests(""" - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - """) - - # any int or real number, returned as float - pyparsing_common.fnumber.runTests(""" - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - """) - - pyparsing_common.hex_integer.runTests(""" - 100 - FF - """) - - import uuid - pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) - pyparsing_common.uuid.runTests(""" - 12345678-1234-5678-1234-567812345678 - """) diff --git a/third_party/python/requirements.in b/third_party/python/requirements.in index 6f228449acd4..d102213e635a 100644 --- a/third_party/python/requirements.in +++ b/third_party/python/requirements.in @@ -5,6 +5,7 @@ attrs==23.1.0 blessed==1.19.1 cbor2==4.0.1 certifi==2022.12.7 +click==8.1.6 colorama==0.4.5 compare-locales==9.0.1 cookies==2.2.1 @@ -22,7 +23,7 @@ jsonschema==4.17.3 looseversion==1.0.1 mozilla-repo-urls==0.1.1 mozilla-version==2.0.0 -packaging==21.3 +packaging==23.1 pathspec==0.9.0 pip==23.0.1 pip-tools==5.5.0 diff --git a/third_party/python/requirements.txt b/third_party/python/requirements.txt index 0dd57a1bffab..b194fd48734a 100644 --- a/third_party/python/requirements.txt +++ b/third_party/python/requirements.txt @@ -66,9 +66,9 @@ certifi==2022.12.7 ; python_version >= "3.7" and python_version < "4.0" \ chardet==4.0.0 ; python_version >= "3.7" and python_version < "4.0" \ --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 -click==7.1.2 ; python_version >= "3.7" and python_version < "4.0" \ - --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ - --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc +click==8.1.6 ; python_version >= "3.7" and python_version < "4.0" \ + --hash=sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd \ + --hash=sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5 colorama==0.4.5 ; python_version >= "3.7" and python_version < "4.0" \ --hash=sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da \ --hash=sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4 @@ -252,9 +252,9 @@ multidict==5.1.0 ; python_version >= "3.7" and python_version < "4.0" \ --hash=sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5 \ --hash=sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281 \ --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80 -packaging==21.3 ; python_version >= "3.7" and python_version < "4.0" \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 +packaging==23.1 ; python_version >= "3.7" and python_version < "4.0" \ + --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ + --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f pathspec==0.9.0 ; python_version >= "3.7" and python_version < "4.0" \ --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 @@ -277,9 +277,6 @@ pyasn1==0.4.8 ; python_version >= "3.7" and python_version < "4.0" \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba pylru==1.0.9 ; python_version >= "3.7" and python_version < "4.0" \ --hash=sha256:71376192671f0ad1690b2a7427d39a29b1df994c8469a9b46b03ed7e28c0172c -pyparsing==2.4.7 ; python_version >= "3.7" and python_version < "4.0" \ - --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ - --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b pyrsistent==0.16.0 ; python_version >= "3.7" and python_version < "4.0" \ --hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 python-dateutil==2.8.2 ; python_version >= "3.7" and python_version < "4.0" \ diff --git a/tools/lint/python/black_requirements.in b/tools/lint/python/black_requirements.in index e5efa47492bd..1f50369f7a1f 100644 --- a/tools/lint/python/black_requirements.in +++ b/tools/lint/python/black_requirements.in @@ -1,4 +1,4 @@ -black==21.11b1 +black==23.3.0 typing-extensions==3.10.0.2 dataclasses==0.6 diff --git a/tools/lint/python/black_requirements.txt b/tools/lint/python/black_requirements.txt index ffa9ef3564a4..8b473adb5d37 100644 --- a/tools/lint/python/black_requirements.txt +++ b/tools/lint/python/black_requirements.txt @@ -1,12 +1,35 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: # -# pip-compile --generate-hashes --output-file=tools/lint/python/black_requirements.txt tools/lint/python/black_requirements.in +# pip-compile --config=pyproject.toml --generate-hashes --output-file=tools/lint/python/black_requirements.txt tools/lint/python/black_requirements.in # -black==21.11b1 \ - --hash=sha256:802c6c30b637b28645b7fde282ed2569c0cd777dbe493a41b6a03c1d903f99ac \ - --hash=sha256:a042adbb18b3262faad5aff4e834ff186bb893f95ba3a8013f09de1e5569def2 +black==23.3.0 \ + --hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \ + --hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \ + --hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \ + --hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \ + --hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \ + --hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \ + --hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \ + --hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \ + --hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \ + --hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \ + --hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \ + --hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \ + --hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \ + --hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \ + --hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \ + --hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \ + --hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \ + --hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \ + --hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \ + --hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \ + --hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \ + --hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \ + --hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \ + --hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \ + --hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3 # via -r tools/lint/python/black_requirements.in click==8.0.3 \ --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \ @@ -16,10 +39,18 @@ dataclasses==0.6 \ --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \ --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 # via -r tools/lint/python/black_requirements.in +importlib-metadata==6.7.0 \ + --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \ + --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5 + # via click mypy-extensions==0.4.3 \ --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 # via black +packaging==23.1 \ + --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ + --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f + # via black pathspec==0.9.0 \ --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 @@ -28,86 +59,53 @@ platformdirs==2.4.0 \ --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \ --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d # via black -regex==2021.11.10 \ - --hash=sha256:05b7d6d7e64efe309972adab77fc2af8907bb93217ec60aa9fe12a0dad35874f \ - --hash=sha256:0617383e2fe465732af4509e61648b77cbe3aee68b6ac8c0b6fe934db90be5cc \ - --hash=sha256:07856afef5ffcc052e7eccf3213317fbb94e4a5cd8177a2caa69c980657b3cb4 \ - --hash=sha256:162abfd74e88001d20cb73ceaffbfe601469923e875caf9118333b1a4aaafdc4 \ - --hash=sha256:2207ae4f64ad3af399e2d30dde66f0b36ae5c3129b52885f1bffc2f05ec505c8 \ - --hash=sha256:30ab804ea73972049b7a2a5c62d97687d69b5a60a67adca07eb73a0ddbc9e29f \ - --hash=sha256:3b5df18db1fccd66de15aa59c41e4f853b5df7550723d26aa6cb7f40e5d9da5a \ - --hash=sha256:3c5fb32cc6077abad3bbf0323067636d93307c9fa93e072771cf9a64d1c0f3ef \ - --hash=sha256:416c5f1a188c91e3eb41e9c8787288e707f7d2ebe66e0a6563af280d9b68478f \ - --hash=sha256:432bd15d40ed835a51617521d60d0125867f7b88acf653e4ed994a1f8e4995dc \ - --hash=sha256:4aaa4e0705ef2b73dd8e36eeb4c868f80f8393f5f4d855e94025ce7ad8525f50 \ - --hash=sha256:537ca6a3586931b16a85ac38c08cc48f10fc870a5b25e51794c74df843e9966d \ - --hash=sha256:53db2c6be8a2710b359bfd3d3aa17ba38f8aa72a82309a12ae99d3c0c3dcd74d \ - --hash=sha256:5537f71b6d646f7f5f340562ec4c77b6e1c915f8baae822ea0b7e46c1f09b733 \ - --hash=sha256:6650f16365f1924d6014d2ea770bde8555b4a39dc9576abb95e3cd1ff0263b36 \ - --hash=sha256:666abff54e474d28ff42756d94544cdfd42e2ee97065857413b72e8a2d6a6345 \ - --hash=sha256:68a067c11463de2a37157930d8b153005085e42bcb7ad9ca562d77ba7d1404e0 \ - --hash=sha256:780b48456a0f0ba4d390e8b5f7c661fdd218934388cde1a974010a965e200e12 \ - --hash=sha256:788aef3549f1924d5c38263104dae7395bf020a42776d5ec5ea2b0d3d85d6646 \ - --hash=sha256:7ee1227cf08b6716c85504aebc49ac827eb88fcc6e51564f010f11a406c0a667 \ - --hash=sha256:7f301b11b9d214f83ddaf689181051e7f48905568b0c7017c04c06dfd065e244 \ - --hash=sha256:83ee89483672b11f8952b158640d0c0ff02dc43d9cb1b70c1564b49abe92ce29 \ - --hash=sha256:85bfa6a5413be0ee6c5c4a663668a2cad2cbecdee367630d097d7823041bdeec \ - --hash=sha256:9345b6f7ee578bad8e475129ed40123d265464c4cfead6c261fd60fc9de00bcf \ - --hash=sha256:93a5051fcf5fad72de73b96f07d30bc29665697fb8ecdfbc474f3452c78adcf4 \ - --hash=sha256:962b9a917dd7ceacbe5cd424556914cb0d636001e393b43dc886ba31d2a1e449 \ - --hash=sha256:98ba568e8ae26beb726aeea2273053c717641933836568c2a0278a84987b2a1a \ - --hash=sha256:a3feefd5e95871872673b08636f96b61ebef62971eab044f5124fb4dea39919d \ - --hash=sha256:b43c2b8a330a490daaef5a47ab114935002b13b3f9dc5da56d5322ff218eeadb \ - --hash=sha256:b483c9d00a565633c87abd0aaf27eb5016de23fed952e054ecc19ce32f6a9e7e \ - --hash=sha256:ba05430e819e58544e840a68b03b28b6d328aff2e41579037e8bab7653b37d83 \ - --hash=sha256:ca5f18a75e1256ce07494e245cdb146f5a9267d3c702ebf9b65c7f8bd843431e \ - --hash=sha256:d5ca078bb666c4a9d1287a379fe617a6dccd18c3e8a7e6c7e1eb8974330c626a \ - --hash=sha256:da1a90c1ddb7531b1d5ff1e171b4ee61f6345119be7351104b67ff413843fe94 \ - --hash=sha256:dba70f30fd81f8ce6d32ddeef37d91c8948e5d5a4c63242d16a2b2df8143aafc \ - --hash=sha256:dd33eb9bdcfbabab3459c9ee651d94c842bc8a05fabc95edf4ee0c15a072495e \ - --hash=sha256:e0538c43565ee6e703d3a7c3bdfe4037a5209250e8502c98f20fea6f5fdf2965 \ - --hash=sha256:e1f54b9b4b6c53369f40028d2dd07a8c374583417ee6ec0ea304e710a20f80a0 \ - --hash=sha256:e32d2a2b02ccbef10145df9135751abea1f9f076e67a4e261b05f24b94219e36 \ - --hash=sha256:e71255ba42567d34a13c03968736c5d39bb4a97ce98188fafb27ce981115beec \ - --hash=sha256:ed2e07c6a26ed4bea91b897ee2b0835c21716d9a469a96c3e878dc5f8c55bb23 \ - --hash=sha256:eef2afb0fd1747f33f1ee3e209bce1ed582d1896b240ccc5e2697e3275f037c7 \ - --hash=sha256:f23222527b307970e383433daec128d769ff778d9b29343fb3496472dc20dabe \ - --hash=sha256:f341ee2df0999bfdf7a95e448075effe0db212a59387de1a70690e4acb03d4c6 \ - --hash=sha256:f7f325be2804246a75a4f45c72d4ce80d2443ab815063cdf70ee8fb2ca59ee1b \ - --hash=sha256:f8af619e3be812a2059b212064ea7a640aff0568d972cd1b9e920837469eb3cb \ - --hash=sha256:fa8c626d6441e2d04b6ee703ef2d1e17608ad44c7cb75258c09dd42bacdfc64b \ - --hash=sha256:fbb9dc00e39f3e6c0ef48edee202f9520dafb233e8b51b06b8428cfcb92abd30 \ - --hash=sha256:fff55f3ce50a3ff63ec8e2a8d3dd924f1941b250b0aac3d3d42b687eeff07a8e - # via black tomli==1.2.2 \ --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade # via black -typed-ast==1.5.4 \ - --hash=sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2 \ - --hash=sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1 \ - --hash=sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6 \ - --hash=sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62 \ - --hash=sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac \ - --hash=sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d \ - --hash=sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc \ - --hash=sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2 \ - --hash=sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97 \ - --hash=sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35 \ - --hash=sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6 \ - --hash=sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1 \ - --hash=sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4 \ - --hash=sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c \ - --hash=sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e \ - --hash=sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec \ - --hash=sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f \ - --hash=sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72 \ - --hash=sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47 \ - --hash=sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72 \ - --hash=sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe \ - --hash=sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6 \ - --hash=sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3 \ - --hash=sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66 +typed-ast==1.5.5 \ + --hash=sha256:042eb665ff6bf020dd2243307d11ed626306b82812aba21836096d229fdc6a10 \ + --hash=sha256:045f9930a1550d9352464e5149710d56a2aed23a2ffe78946478f7b5416f1ede \ + --hash=sha256:0635900d16ae133cab3b26c607586131269f88266954eb04ec31535c9a12ef1e \ + --hash=sha256:118c1ce46ce58fda78503eae14b7664163aa735b620b64b5b725453696f2a35c \ + --hash=sha256:16f7313e0a08c7de57f2998c85e2a69a642e97cb32f87eb65fbfe88381a5e44d \ + --hash=sha256:1efebbbf4604ad1283e963e8915daa240cb4bf5067053cf2f0baadc4d4fb51b8 \ + --hash=sha256:2188bc33d85951ea4ddad55d2b35598b2709d122c11c75cffd529fbc9965508e \ + --hash=sha256:2b946ef8c04f77230489f75b4b5a4a6f24c078be4aed241cfabe9cbf4156e7e5 \ + --hash=sha256:335f22ccb244da2b5c296e6f96b06ee9bed46526db0de38d2f0e5a6597b81155 \ + --hash=sha256:381eed9c95484ceef5ced626355fdc0765ab51d8553fec08661dce654a935db4 \ + --hash=sha256:429ae404f69dc94b9361bb62291885894b7c6fb4640d561179548c849f8492ba \ + --hash=sha256:44f214394fc1af23ca6d4e9e744804d890045d1643dd7e8229951e0ef39429b5 \ + --hash=sha256:48074261a842acf825af1968cd912f6f21357316080ebaca5f19abbb11690c8a \ + --hash=sha256:4bc1efe0ce3ffb74784e06460f01a223ac1f6ab31c6bc0376a21184bf5aabe3b \ + --hash=sha256:57bfc3cf35a0f2fdf0a88a3044aafaec1d2f24d8ae8cd87c4f58d615fb5b6311 \ + --hash=sha256:597fc66b4162f959ee6a96b978c0435bd63791e31e4f410622d19f1686d5e769 \ + --hash=sha256:5f7a8c46a8b333f71abd61d7ab9255440d4a588f34a21f126bbfc95f6049e686 \ + --hash=sha256:5fe83a9a44c4ce67c796a1b466c270c1272e176603d5e06f6afbc101a572859d \ + --hash=sha256:61443214d9b4c660dcf4b5307f15c12cb30bdfe9588ce6158f4a005baeb167b2 \ + --hash=sha256:622e4a006472b05cf6ef7f9f2636edc51bda670b7bbffa18d26b255269d3d814 \ + --hash=sha256:6eb936d107e4d474940469e8ec5b380c9b329b5f08b78282d46baeebd3692dc9 \ + --hash=sha256:7f58fabdde8dcbe764cef5e1a7fcb440f2463c1bbbec1cf2a86ca7bc1f95184b \ + --hash=sha256:83509f9324011c9a39faaef0922c6f720f9623afe3fe220b6d0b15638247206b \ + --hash=sha256:8c524eb3024edcc04e288db9541fe1f438f82d281e591c548903d5b77ad1ddd4 \ + --hash=sha256:94282f7a354f36ef5dbce0ef3467ebf6a258e370ab33d5b40c249fa996e590dd \ + --hash=sha256:b445c2abfecab89a932b20bd8261488d574591173d07827c1eda32c457358b18 \ + --hash=sha256:be4919b808efa61101456e87f2d4c75b228f4e52618621c77f1ddcaae15904fa \ + --hash=sha256:bfd39a41c0ef6f31684daff53befddae608f9daf6957140228a08e51f312d7e6 \ + --hash=sha256:c631da9710271cb67b08bd3f3813b7af7f4c69c319b75475436fcab8c3d21bee \ + --hash=sha256:cc95ffaaab2be3b25eb938779e43f513e0e538a84dd14a5d844b8f2932593d88 \ + --hash=sha256:d09d930c2d1d621f717bb217bf1fe2584616febb5138d9b3e8cdd26506c3f6d4 \ + --hash=sha256:d40c10326893ecab8a80a53039164a224984339b2c32a6baf55ecbd5b1df6431 \ + --hash=sha256:d41b7a686ce653e06c2609075d397ebd5b969d821b9797d029fccd71fdec8e04 \ + --hash=sha256:d5c0c112a74c0e5db2c75882a0adf3133adedcdbfd8cf7c9d6ed77365ab90a1d \ + --hash=sha256:e1a976ed4cc2d71bb073e1b2a250892a6e968ff02aa14c1f40eba4f365ffec02 \ + --hash=sha256:e48bf27022897577d8479eaed64701ecaf0467182448bd95759883300ca818c8 \ + --hash=sha256:ed4a1a42df8a3dfb6b40c3d2de109e935949f2f66b19703eafade03173f8f437 \ + --hash=sha256:f0aefdd66f1784c58f65b502b6cf8b121544680456d1cebbd300c2c813899274 \ + --hash=sha256:fc2b8c4e1bc5cd96c1a823a885e6b158f8451cf6f5530e1829390b4d27d0807f \ + --hash=sha256:fd946abf3c31fb50eee07451a6aedbfff912fcd13cf357363f5b4e834cc5e71a \ + --hash=sha256:fe58ef6a764de7b4b36edfc8592641f56e69b7163bba9f9c8089838ee596bfb2 + # via black typing-extensions==3.10.0.2 \ --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ @@ -115,3 +113,8 @@ typing-extensions==3.10.0.2 \ # via # -r tools/lint/python/black_requirements.in # black + # importlib-metadata +zipp==3.15.0 \ + --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ + --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 + # via importlib-metadata