diff --git a/third_party/python/glean_parser/.circleci/config.yml b/third_party/python/glean_parser/.circleci/config.yml index 06df8753be08..8cd7f41930d4 100644 --- a/third_party/python/glean_parser/.circleci/config.yml +++ b/third_party/python/glean_parser/.circleci/config.yml @@ -62,14 +62,14 @@ commands: jobs: build-36: docker: - - image: circleci/python:3.6.9 + - image: circleci/python:3.6.12 steps: - test-start - test-python-version build-36-min: docker: - - image: circleci/python:3.6.9 + - image: circleci/python:3.6.12 steps: - test-start - test-min-requirements @@ -77,7 +77,7 @@ jobs: build-37: docker: - - image: circleci/python:3.7.5 + - image: circleci/python:3.7.9 steps: - test-start - test-python-version @@ -92,19 +92,26 @@ jobs: build-38: docker: - - image: circleci/python:3.8.0 + - image: circleci/python:3.8.5 steps: - test-start - test-python-version build-38-min: docker: - - image: circleci/python:3.8.0 + - image: circleci/python:3.8.5 steps: - test-start - test-min-requirements - test-python-version + build-39: + docker: + - image: circleci/python:3.9.0rc1 + steps: + - test-start + - test-python-version + docs-deploy: docker: - image: node:8.10.0 @@ -174,6 +181,10 @@ workflows: filters: tags: only: /.*/ + - build-39: + filters: + tags: + only: /.*/ - docs-deploy: requires: - build-37 diff --git a/third_party/python/glean_parser/CONTRIBUTING.rst b/third_party/python/glean_parser/CONTRIBUTING.rst index 6b205a00bb75..28a1f95feac0 100644 --- a/third_party/python/glean_parser/CONTRIBUTING.rst +++ b/third_party/python/glean_parser/CONTRIBUTING.rst @@ -141,7 +141,7 @@ Get a clean main branch with all of the changes from `upstream`:: - Make sure all your changes are committed. -- Push the changes upstream:: +- Push the changes upstream. (Normally pushing directly without review is frowned upon, but the `main` branch is protected from force pushes and release tagging requires the same permissions as pushing to `main`):: $ git push upstream main @@ -149,8 +149,12 @@ Get a clean main branch with all of the changes from `upstream`:: - Make the release on GitHub using [this link](https://github.com/mozilla/glean_parser/releases/new) -- Enter the new version in the form `vX.Y.Z`. +- Both the tag and the release title should be in the form `vX.Y.Z`. - Copy and paste the relevant part of the `HISTORY.rst` file into the description. +- Tagging the release will trigger a CI workflow which will build the distribution of `glean_parser` and publish it to PyPI. + The continuous integration system will then automatically deploy to PyPI. + +See also the [instructions for updating the version of `glean_parser` used by the Glean SDK](https://mozilla.github.io/glean/book/dev/upgrading-glean-parser.html). diff --git a/third_party/python/glean_parser/HISTORY.rst b/third_party/python/glean_parser/HISTORY.rst index 0846789b8d5d..48630541b092 100644 --- a/third_party/python/glean_parser/HISTORY.rst +++ b/third_party/python/glean_parser/HISTORY.rst @@ -5,6 +5,18 @@ History Unreleased ---------- +1.29.0 (2020-10-07) +------------------- + +* **Breaking change:** `glean_parser` will now return an error code when any of the input files do not exist (unless the `--allow-missing-files` flag is passed). +* Generated code now includes a comment next to each metric containing the name of the metric in its original `snake_case` form. +* When metrics don't provide a `unit` parameter, it is not included in the output (as provided by probe-scraper). + +1.28.6 (2020-09-24) +------------------- + +* BUGFIX: Ensure Kotlin arguments are deterministically ordered + 1.28.5 (2020-09-14) ------------------- diff --git a/third_party/python/glean_parser/Makefile b/third_party/python/glean_parser/Makefile index 21a59713196d..d88a0eef0ae9 100644 --- a/third_party/python/glean_parser/Makefile +++ b/third_party/python/glean_parser/Makefile @@ -36,9 +36,7 @@ clean-test: ## remove test and coverage artifacts lint: ## check style with flake8 python3 -m flake8 glean_parser tests - if python3 --version | grep 'Python 3\.[678]\..*'; then \ - python3 -m black --check glean_parser tests setup.py; \ - fi + python3 -m black --check glean_parser tests setup.py python3 -m yamllint glean_parser tests python3 -m mypy glean_parser diff --git a/third_party/python/glean_parser/PKG-INFO b/third_party/python/glean_parser/PKG-INFO index d6d46295b3df..9b1f7381bc37 100644 --- a/third_party/python/glean_parser/PKG-INFO +++ b/third_party/python/glean_parser/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: glean_parser -Version: 1.28.5 +Version: 1.29.0 Summary: Parser tools for Mozilla's Glean telemetry Home-page: https://github.com/mozilla/glean_parser Author: Michael Droettboom @@ -69,6 +69,18 @@ Description: ============ Unreleased ---------- + 1.29.0 (2020-10-07) + ------------------- + + * **Breaking change:** `glean_parser` will now return an error code when any of the input files do not exist (unless the `--allow-missing-files` flag is passed). + * Generated code now includes a comment next to each metric containing the name of the metric in its original `snake_case` form. + * When metrics don't provide a `unit` parameter, it is not included in the output (as provided by probe-scraper). + + 1.28.6 (2020-09-24) + ------------------- + + * BUGFIX: Ensure Kotlin arguments are deterministically ordered + 1.28.5 (2020-09-14) ------------------- @@ -475,3 +487,4 @@ Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 diff --git a/third_party/python/glean_parser/glean_parser/__main__.py b/third_party/python/glean_parser/glean_parser/__main__.py index 6729a38d61bf..844fc1541988 100644 --- a/third_party/python/glean_parser/glean_parser/__main__.py +++ b/third_party/python/glean_parser/glean_parser/__main__.py @@ -53,7 +53,12 @@ from . import validate_ping "Should only be set when building the Glean library itself." ), ) -def translate(input, format, output, option, allow_reserved): +@click.option( + "--allow-missing-files", + is_flag=True, + help=("Do not treat missing input files as an error."), +) +def translate(input, format, output, option, allow_reserved, allow_missing_files): """ Translate metrics.yaml and pings.yaml files to other formats. """ @@ -68,7 +73,10 @@ def translate(input, format, output, option, allow_reserved): format, Path(output), option_dict, - {"allow_reserved": allow_reserved}, + { + "allow_reserved": allow_reserved, + "allow_missing_files": allow_missing_files, + }, ) ) @@ -112,11 +120,24 @@ def check(schema): "Should only be set when building the Glean library itself." ), ) -def glinter(input, allow_reserved): +@click.option( + "--allow-missing-files", + is_flag=True, + help=("Do not treat missing input files as an error."), +) +def glinter(input, allow_reserved, allow_missing_files): """ Runs a linter over the metrics. """ - sys.exit(lint.glinter([Path(x) for x in input], {"allow_reserved": allow_reserved})) + sys.exit( + lint.glinter( + [Path(x) for x in input], + { + "allow_reserved": allow_reserved, + "allow_missing_files": allow_missing_files, + }, + ) + ) @click.group() @@ -131,5 +152,18 @@ main.add_command(check) main.add_command(glinter) +def main_wrapper(args=None): + """ + A simple wrapper around click's `main` to display the glean_parser version + when there is an error. + """ + try: + main(args=args) + except SystemExit as e: + if e.code != 0: + print(f"ERROR running glean_parser v{glean_parser.__version__}") + raise + + if __name__ == "__main__": - sys.exit(main()) # pragma: no cover + main_wrapper() # pragma: no cover diff --git a/third_party/python/glean_parser/glean_parser/csharp.py b/third_party/python/glean_parser/glean_parser/csharp.py index 52a53962adbb..6ed8cb0338ea 100644 --- a/third_party/python/glean_parser/glean_parser/csharp.py +++ b/third_party/python/glean_parser/glean_parser/csharp.py @@ -11,7 +11,7 @@ Outputter to generate C# code for metrics. import enum import json from pathlib import Path -from typing import Any, Dict, List, Union # noqa +from typing import Any, Dict, List, Optional, Union # noqa from . import metrics from . import pings @@ -104,7 +104,7 @@ def class_name(obj_type: str) -> str: def output_csharp( - objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {} + objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None ) -> None: """ Given a tree of objects, output C# code to `output_dir`. @@ -120,6 +120,9 @@ def output_csharp( This is where glean objects will be imported from in the generated code. """ + if options is None: + options = {} + template = util.get_jinja2_template( "csharp.jinja2", filters=( diff --git a/third_party/python/glean_parser/glean_parser/kotlin.py b/third_party/python/glean_parser/glean_parser/kotlin.py index f12466b9738b..566049a8921b 100644 --- a/third_party/python/glean_parser/glean_parser/kotlin.py +++ b/third_party/python/glean_parser/glean_parser/kotlin.py @@ -12,7 +12,7 @@ from collections import OrderedDict import enum import json from pathlib import Path -from typing import Any, Dict, List, Union # noqa +from typing import Any, Dict, List, Optional, Union # noqa from . import metrics from . import pings @@ -102,7 +102,7 @@ def class_name(obj_type: str) -> str: def output_gecko_lookup( - objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {} + objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None ) -> None: """ Given a tree of objects, generate a Kotlin map between Gecko histograms and @@ -119,6 +119,9 @@ def output_gecko_lookup( This is where glean objects will be imported from in the generated code. """ + if options is None: + options = {} + template = util.get_jinja2_template( "kotlin.geckoview.jinja2", filters=( @@ -197,7 +200,7 @@ def output_gecko_lookup( def output_kotlin( - objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {} + objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None ) -> None: """ Given a tree of objects, output Kotlin code to `output_dir`. @@ -213,6 +216,9 @@ def output_kotlin( This is where glean objects will be imported from in the generated code. """ + if options is None: + options = {} + template = util.get_jinja2_template( "kotlin.jinja2", filters=( diff --git a/third_party/python/glean_parser/glean_parser/lint.py b/third_party/python/glean_parser/glean_parser/lint.py index f2788780fd9b..facb632d5ec3 100644 --- a/third_party/python/glean_parser/glean_parser/lint.py +++ b/third_party/python/glean_parser/glean_parser/lint.py @@ -7,7 +7,16 @@ import enum from pathlib import Path import re import sys -from typing import Any, Callable, Dict, Generator, List, Iterable, Tuple, Union # noqa +from typing import ( + Any, + Callable, + Dict, + Generator, + List, + Iterable, + Optional, + Tuple, +) # noqa from . import metrics @@ -96,7 +105,7 @@ def check_common_prefix( def check_unit_in_name( - metric: metrics.Metric, parser_config: Dict[str, Any] = {} + metric: metrics.Metric, parser_config: Dict[str, Any] ) -> LintGenerator: """ The metric name ends in a unit. @@ -189,7 +198,7 @@ def check_category_generic( def check_bug_number( - metric: metrics.Metric, parser_config: Dict[str, Any] = {} + metric: metrics.Metric, parser_config: Dict[str, Any] ) -> LintGenerator: number_bugs = [str(bug) for bug in metric.bugs if isinstance(bug, int)] @@ -202,7 +211,7 @@ def check_bug_number( def check_valid_in_baseline( - metric: metrics.Metric, parser_config: Dict[str, Any] = {} + metric: metrics.Metric, parser_config: Dict[str, Any] ) -> LintGenerator: allow_reserved = parser_config.get("allow_reserved", False) @@ -214,7 +223,7 @@ def check_valid_in_baseline( def check_misspelled_pings( - metric: metrics.Metric, parser_config: Dict[str, Any] = {} + metric: metrics.Metric, parser_config: Dict[str, Any] ) -> LintGenerator: for ping in metric.send_in_pings: for builtin in pings.RESERVED_PING_NAMES: @@ -224,7 +233,7 @@ def check_misspelled_pings( def check_user_lifetime_expiration( - metric: metrics.Metric, parser_config: Dict[str, Any] = {} + metric: metrics.Metric, parser_config: Dict[str, Any] ) -> LintGenerator: if metric.lifetime == metrics.Lifetime.user and metric.expires != "never": @@ -236,7 +245,7 @@ def check_user_lifetime_expiration( def check_expired_date( - metric: metrics.Metric, parser_config: Dict[str, Any] = {} + metric: metrics.Metric, parser_config: Dict[str, Any] ) -> LintGenerator: try: metric.validate_expires() @@ -245,7 +254,7 @@ def check_expired_date( def check_expired_metric( - metric: metrics.Metric, parser_config: Dict[str, Any] = {} + metric: metrics.Metric, parser_config: Dict[str, Any] ) -> LintGenerator: if metric.is_expired(): yield ("Metric has expired. Please consider removing it.") @@ -291,7 +300,9 @@ class GlinterNit: def lint_metrics( - objs: metrics.ObjectTree, parser_config: Dict[str, Any] = {}, file=sys.stderr + objs: metrics.ObjectTree, + parser_config: Optional[Dict[str, Any]] = None, + file=sys.stderr, ) -> List[GlinterNit]: """ Performs glinter checks on a set of metrics objects. @@ -300,6 +311,9 @@ def lint_metrics( :param file: The stream to write errors to. :returns: List of nits. """ + if parser_config is None: + parser_config = {} + nits: List[GlinterNit] = [] for (category_name, category) in sorted(list(objs.items())): if category_name == "pings": @@ -322,7 +336,7 @@ def lint_metrics( for msg in cat_check_func(category_name, category_metrics.values()) ) - for (metric_name, metric) in sorted(list(category_metrics.items())): + for (_metric_name, metric) in sorted(list(category_metrics.items())): for (check_name, (check_func, check_type)) in INDIVIDUAL_CHECKS.items(): new_nits = list(check_func(metric, parser_config)) if len(new_nits): @@ -354,7 +368,11 @@ def lint_metrics( return nits -def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List: +def lint_yaml_files( + input_filepaths: Iterable[Path], + file=sys.stderr, + parser_config: Dict[str, Any] = None, +) -> List: """ Performs glinter YAML lint on a set of files. @@ -363,10 +381,16 @@ def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List: :returns: List of nits. """ + if parser_config is None: + parser_config = {} + # Generic type since the actual type comes from yamllint, which we don't # control. nits: List = [] for path in input_filepaths: + if not path.is_file() and parser_config.get("allow_missing_files", False): + continue + # yamllint needs both the file content and the path. file_content = None with path.open("r", encoding="utf-8") as fd: @@ -386,7 +410,9 @@ def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List: def glinter( - input_filepaths: Iterable[Path], parser_config: Dict[str, Any] = {}, file=sys.stderr + input_filepaths: Iterable[Path], + parser_config: Optional[Dict[str, Any]] = None, + file=sys.stderr, ) -> int: """ Commandline helper for glinter. @@ -397,7 +423,10 @@ def glinter( :param file: The stream to write the errors to. :return: Non-zero if there were any glinter errors. """ - if lint_yaml_files(input_filepaths, file=file): + if parser_config is None: + parser_config = {} + + if lint_yaml_files(input_filepaths, file=file, parser_config=parser_config): return 1 objs = parser.parse_objects(input_filepaths, parser_config) diff --git a/third_party/python/glean_parser/glean_parser/markdown.py b/third_party/python/glean_parser/glean_parser/markdown.py index 3e9056514375..49d95800416a 100644 --- a/third_party/python/glean_parser/glean_parser/markdown.py +++ b/third_party/python/glean_parser/glean_parser/markdown.py @@ -42,7 +42,9 @@ def extra_info(obj: Union[metrics.Metric, pings.Ping]) -> List[Tuple[str, str]]: return extra_info -def ping_desc(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) -> str: +def ping_desc( + ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None +) -> str: """ Return a text description of the ping. If a custom_pings_cache is available, look in there for non-reserved ping names description. @@ -56,7 +58,7 @@ def ping_desc(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) -> ) elif ping_name == "all-pings": desc = "These metrics are sent in every ping." - elif ping_name in custom_pings_cache: + elif custom_pings_cache is not None and ping_name in custom_pings_cache: desc = custom_pings_cache[ping_name].description return desc @@ -87,8 +89,10 @@ def ping_docs(ping_name: str) -> str: return f"https://mozilla.github.io/glean/book/user/pings/{ping_name}.html" -def if_empty(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) -> bool: - if ping_name in custom_pings_cache: +def if_empty( + ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None +) -> bool: + if custom_pings_cache is not None and ping_name in custom_pings_cache: return custom_pings_cache[ping_name].send_if_empty else: return False @@ -109,27 +113,27 @@ def ping_reasons( def ping_data_reviews( - ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {} + ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None ) -> Optional[List[str]]: - if ping_name in custom_pings_cache: + if custom_pings_cache is not None and ping_name in custom_pings_cache: return custom_pings_cache[ping_name].data_reviews else: return None def ping_bugs( - ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {} + ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None ) -> Optional[List[str]]: - if ping_name in custom_pings_cache: + if custom_pings_cache is not None and ping_name in custom_pings_cache: return custom_pings_cache[ping_name].bugs else: return None def ping_include_client_id( - ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {} + ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None ) -> bool: - if ping_name in custom_pings_cache: + if custom_pings_cache is not None and ping_name in custom_pings_cache: return custom_pings_cache[ping_name].include_client_id else: return False @@ -145,7 +149,7 @@ def data_sensitivity_numbers( def output_markdown( - objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {} + objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None ) -> None: """ Given a tree of objects, output Markdown docs to `output_dir`. @@ -159,6 +163,8 @@ def output_markdown( :param options: options dictionary, with the following optional key: - `project_title`: The projects title. """ + if options is None: + options = {} # Build a dictionary that associates pings with their metrics. # @@ -177,7 +183,7 @@ def output_markdown( # This also builds a dictionary of custom pings, if available. custom_pings_cache: Dict[str, pings.Ping] = defaultdict() metrics_by_pings: Dict[str, List[metrics.Metric]] = defaultdict(list) - for category_key, category_val in objs.items(): + for _category_key, category_val in objs.items(): for obj in category_val.values(): # Filter out custom pings. We will need them for extracting # the description diff --git a/third_party/python/glean_parser/glean_parser/metrics.py b/third_party/python/glean_parser/glean_parser/metrics.py index 1221322754fe..8fcb96e3fb67 100644 --- a/third_party/python/glean_parser/glean_parser/metrics.py +++ b/third_party/python/glean_parser/glean_parser/metrics.py @@ -52,7 +52,7 @@ class Metric: disabled: bool = False, lifetime: str = "ping", send_in_pings: Optional[List[str]] = None, - unit: str = "", + unit: Optional[str] = None, gecko_datapoint: str = "", no_lint: Optional[List[str]] = None, data_sensitivity: Optional[List[str]] = None, @@ -78,7 +78,8 @@ class Metric: if send_in_pings is None: send_in_pings = ["default"] self.send_in_pings = send_in_pings - self.unit = unit + if unit is not None: + self.unit = unit self.gecko_datapoint = gecko_datapoint if no_lint is None: no_lint = [] @@ -120,7 +121,7 @@ class Metric: category: str, name: str, metric_info: Dict[str, util.JSONType], - config: Dict[str, Any] = {}, + config: Optional[Dict[str, Any]] = None, validated: bool = False, ): """ @@ -136,6 +137,9 @@ class Metric: jsonschema validation :return: A new Metric instance. """ + if config is None: + config = {} + metric_type = metric_info["type"] if not isinstance(metric_type, str): raise TypeError(f"Unknown metric type {metric_type}") diff --git a/third_party/python/glean_parser/glean_parser/parser.py b/third_party/python/glean_parser/glean_parser/parser.py index 4bb908463f65..dc6caff9079a 100644 --- a/third_party/python/glean_parser/glean_parser/parser.py +++ b/third_party/python/glean_parser/glean_parser/parser.py @@ -53,13 +53,21 @@ def _update_validator(validator): def _load_file( - filepath: Path, + filepath: Path, parser_config: Dict[str, Any] ) -> Generator[str, None, Tuple[Dict[str, util.JSONType], Optional[str]]]: """ Load a metrics.yaml or pings.yaml format file. + + If the `filepath` does not exist, raises `FileNotFoundError`, unless + `parser_config["allow_missing_files"]` is `True`. """ try: content = util.load_yaml_or_json(filepath, ordered_dict=True) + except FileNotFoundError: + if not parser_config.get("allow_missing_files", False): + raise + else: + return {}, None except Exception as e: yield util.format_error(filepath, "", textwrap.fill(str(e))) return {}, None @@ -313,7 +321,7 @@ def _preprocess_objects(objs: ObjectTree, config: Dict[str, Any]) -> ObjectTree: @util.keep_value def parse_objects( - filepaths: Iterable[Path], config: Dict[str, Any] = {} + filepaths: Iterable[Path], config: Optional[Dict[str, Any]] = None ) -> Generator[str, None, ObjectTree]: """ Parse one or more metrics.yaml and/or pings.yaml files, returning a tree of @@ -342,12 +350,17 @@ def parse_objects( This is useful when you want to retain the original "disabled" value from the `metrics.yaml`, rather than having it overridden when the metric expires. + - `allow_missing_files`: Do not raise a `FileNotFoundError` if any of + the input `filepaths` do not exist. """ + if config is None: + config = {} + all_objects: ObjectTree = OrderedDict() sources: Dict[Any, Path] = {} filepaths = util.ensure_list(filepaths) for filepath in filepaths: - content, filetype = yield from _load_file(filepath) + content, filetype = yield from _load_file(filepath, config) if filetype == "metrics": yield from _instantiate_metrics( all_objects, sources, content, filepath, config diff --git a/third_party/python/glean_parser/glean_parser/pings.py b/third_party/python/glean_parser/glean_parser/pings.py index dcf82c401688..cff2f48edb9e 100644 --- a/third_party/python/glean_parser/glean_parser/pings.py +++ b/third_party/python/glean_parser/glean_parser/pings.py @@ -73,3 +73,9 @@ class Ping: d = self.__dict__.copy() del d["name"] return d + + def identifier(self) -> str: + """ + Used for the "generated from ..." comment in the output. + """ + return self.name diff --git a/third_party/python/glean_parser/glean_parser/swift.py b/third_party/python/glean_parser/glean_parser/swift.py index d7b25e6fd4dd..22a48d908610 100644 --- a/third_party/python/glean_parser/glean_parser/swift.py +++ b/third_party/python/glean_parser/glean_parser/swift.py @@ -11,7 +11,7 @@ Outputter to generate Swift code for metrics. import enum import json from pathlib import Path -from typing import Any, Dict, Union +from typing import Any, Dict, Optional, Union from . import metrics from . import pings @@ -117,7 +117,7 @@ class Category: def output_swift( - objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {} + objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None ) -> None: """ Given a tree of objects, output Swift code to `output_dir`. @@ -130,6 +130,9 @@ def output_swift( - glean_namespace: The namespace to import Glean from - allow_reserved: When True, this is a Glean-internal build """ + if options is None: + options = {} + template = util.get_jinja2_template( "swift.jinja2", filters=( diff --git a/third_party/python/glean_parser/glean_parser/templates/csharp.jinja2 b/third_party/python/glean_parser/glean_parser/templates/csharp.jinja2 index 9e3ac0ae7cc6..9f5e67aeb93a 100644 --- a/third_party/python/glean_parser/glean_parser/templates/csharp.jinja2 +++ b/third_party/python/glean_parser/glean_parser/templates/csharp.jinja2 @@ -11,7 +11,7 @@ Jinja2 template is not. Please file bugs! #} // file, You can obtain one at http://mozilla.org/MPL/2.0/. {% macro obj_declaration(obj, suffix='', access='', lazy=False) %} {{ access }} {% if lazy %} Lazy<{{ obj|type_name }}>{%- else %} {{ obj|type_name }}{% endif %} {{ obj.name|camelize }}{{ suffix }} -{%- if lazy %} = new Lazy<{{ obj|type_name }}>(() => {%- else %} ={% endif %} +{%- if lazy %} = new Lazy<{{ obj|type_name }}>(() => {%- else %} = // generated from {{ obj.identifier() }}{% endif %} new {{ obj|type_name }}( {% for arg_name in extra_args if obj[arg_name] is defined %} @@ -57,7 +57,7 @@ namespace {{ namespace }} {% for obj in objs.values() %} {% if obj.labeled %} {{ obj_declaration(obj, 'Label', 'private ') }} - private readonly Lazy> {{ obj.name|camelize }}Lazy = new Lazy>(() => new LabeledMetricType<{{ obj|type_name }}>( + private readonly Lazy> {{ obj.name|camelize }}Lazy = new Lazy>(() => new LabeledMetricType<{{ obj|type_name }}>( // generated from {{ obj.identifier() }} category: {{ obj.category|csharp }}, name: {{ obj.name|csharp }}, submetric: {{ category_name|Camelize }}.{{ obj.name|camelize }}Label, @@ -82,7 +82,7 @@ namespace {{ namespace }} /// /// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }} /// - internal {{ obj|type_name }} {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value; + internal {{ obj|type_name }} {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value; // generated from {{ obj.identifier() }} {% else %} {# Finally handle pings. #} diff --git a/third_party/python/glean_parser/glean_parser/templates/kotlin.jinja2 b/third_party/python/glean_parser/glean_parser/templates/kotlin.jinja2 index 99eca980298f..7fd1bc6cdf5b 100644 --- a/third_party/python/glean_parser/glean_parser/templates/kotlin.jinja2 +++ b/third_party/python/glean_parser/glean_parser/templates/kotlin.jinja2 @@ -13,7 +13,7 @@ Jinja2 template is not. Please file bugs! #} {% if (access != "private ") -%} @get:JvmName("{{ obj.name|camelize }}{{ suffix }}") {% endif -%} -{{ access }}val {{ obj.name|camelize }}{{ suffix }}: {{ obj|type_name }}{% if lazy %} by lazy { {%- else %} ={% endif %} +{{ access }}val {{ obj.name|camelize }}{{ suffix }}: {{ obj|type_name }}{% if lazy %} by lazy { {%- else %} ={% endif %} // generated from {{ obj.identifier() }} {{ obj|type_name }}( {% for arg_name in extra_args if obj[arg_name] is defined %} @@ -60,7 +60,7 @@ internal object {{ category_name|Camelize }} { /** * {{ obj.description|wordwrap() | replace('\n', '\n * ') }} */ - val {{ obj.name|camelize }}: LabeledMetricType<{{ obj|type_name }}> by lazy { + val {{ obj.name|camelize }}: LabeledMetricType<{{ obj|type_name }}> by lazy { // generated from {{ obj.identifier() }} LabeledMetricType( category = {{ obj.category|kotlin }}, name = {{ obj.name|kotlin }}, diff --git a/third_party/python/glean_parser/glean_parser/templates/swift.jinja2 b/third_party/python/glean_parser/glean_parser/templates/swift.jinja2 index 295d3261fb73..2f6fa51fb5f8 100644 --- a/third_party/python/glean_parser/glean_parser/templates/swift.jinja2 +++ b/third_party/python/glean_parser/glean_parser/templates/swift.jinja2 @@ -8,7 +8,7 @@ Jinja2 template is not. Please file bugs! #} * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ {% macro obj_declaration(obj, suffix='', access='') %} -{{ access }}static let {{ obj.name|camelize|variable_name }}{{ suffix }} = {{ obj|type_name }}( +{{ access }}static let {{ obj.name|camelize|variable_name }}{{ suffix }} = {{ obj|type_name }}( // generated from {{ obj.identifier() }} {% for arg_name in extra_args if obj[arg_name] is defined %} {{ arg_name|camelize }}: {{ obj[arg_name]|swift }}{{ "," if not loop.last }} {% endfor %} @@ -86,7 +86,7 @@ extension {{ namespace }} { {% if obj.labeled %} {{ obj_declaration(obj, 'Label', 'private ') }} /// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }} - static let {{ obj.name|camelize|variable_name }} = try! LabeledMetricType<{{ obj|type_name }}>( + static let {{ obj.name|camelize|variable_name }} = try! LabeledMetricType<{{ obj|type_name }}>( // generated from {{ obj.identifier() }} category: {{ obj.category|swift }}, name: {{ obj.name|swift }}, sendInPings: {{ obj.send_in_pings|swift }}, diff --git a/third_party/python/glean_parser/glean_parser/translate.py b/third_party/python/glean_parser/glean_parser/translate.py index 9b0c2e770a0c..224da1eb5e34 100644 --- a/third_party/python/glean_parser/glean_parser/translate.py +++ b/third_party/python/glean_parser/glean_parser/translate.py @@ -12,7 +12,7 @@ from pathlib import Path import os import shutil import tempfile -from typing import Any, Callable, Dict, Iterable, List +from typing import Any, Callable, Dict, Iterable, List, Optional from . import lint from . import parser @@ -40,8 +40,11 @@ class Outputter: def __init__( self, output_func: Callable[[metrics.ObjectTree, Path, Dict[str, Any]], None], - clear_patterns: List[str] = [], + clear_patterns: Optional[List[str]] = None, ): + if clear_patterns is None: + clear_patterns = [] + self.output_func = output_func self.clear_patterns = clear_patterns @@ -49,7 +52,7 @@ class Outputter: OUTPUTTERS = { "csharp": Outputter(csharp.output_csharp, ["*.cs"]), "kotlin": Outputter(kotlin.output_kotlin, ["*.kt"]), - "markdown": Outputter(markdown.output_markdown), + "markdown": Outputter(markdown.output_markdown, []), "swift": Outputter(swift.output_swift, ["*.swift"]), } @@ -58,9 +61,9 @@ def translate_metrics( input_filepaths: Iterable[Path], output_dir: Path, translation_func: Callable[[metrics.ObjectTree, Path, Dict[str, Any]], None], - clear_patterns: List[str] = [], - options: Dict[str, Any] = {}, - parser_config: Dict[str, Any] = {}, + clear_patterns: Optional[List[str]] = None, + options: Optional[Dict[str, Any]] = None, + parser_config: Optional[Dict[str, Any]] = None, ): """ Translate the files in `input_filepaths` by running the metrics through a @@ -85,6 +88,15 @@ def translate_metrics( :param parser_config: A dictionary of options that change parsing behavior. See `parser.parse_metrics` for more info. """ + if clear_patterns is None: + clear_patterns = [] + + if options is None: + options = {} + + if parser_config is None: + parser_config = {} + input_filepaths = util.ensure_list(input_filepaths) if lint.glinter(input_filepaths, parser_config): @@ -128,8 +140,8 @@ def translate( input_filepaths: Iterable[Path], output_format: str, output_dir: Path, - options: Dict[str, Any] = {}, - parser_config: Dict[str, Any] = {}, + options: Optional[Dict[str, Any]] = None, + parser_config: Optional[Dict[str, Any]] = None, ): """ Translate the files in `input_filepaths` to the given `output_format` and @@ -143,6 +155,12 @@ def translate( :param parser_config: A dictionary of options that change parsing behavior. See `parser.parse_metrics` for more info. """ + if options is None: + options = {} + + if parser_config is None: + parser_config = {} + format_desc = OUTPUTTERS.get(output_format, None) if format_desc is None: diff --git a/third_party/python/glean_parser/glean_parser/util.py b/third_party/python/glean_parser/glean_parser/util.py index a35b4f1548ff..f3d478ed8276 100644 --- a/third_party/python/glean_parser/glean_parser/util.py +++ b/third_party/python/glean_parser/glean_parser/util.py @@ -114,14 +114,12 @@ def load_yaml_or_json(path: Path, ordered_dict: bool = False): :param path: `pathlib.Path` object :rtype object: The tree of objects as a result of parsing the file. :raises ValueError: The file is neither a .json, .yml or .yaml file. + :raises FileNotFoundError: The file does not exist. """ # If in py.test, support bits of literal JSON/YAML content if TESTING_MODE and isinstance(path, dict): return path - if not path.is_file(): - return {} - if path.suffix == ".json": with path.open("r", encoding="utf-8") as fd: return json.load(fd) @@ -434,5 +432,7 @@ extra_ping_args = [ ] -# Names of parameters to pass to both metric and ping constructors. -extra_args = list(set(extra_metric_args) | set(extra_ping_args)) +# Names of parameters to pass to both metric and ping constructors (no duplicates). +extra_args = extra_metric_args + [ + v for v in extra_ping_args if v not in extra_metric_args +] diff --git a/third_party/python/glean_parser/requirements_dev.txt b/third_party/python/glean_parser/requirements_dev.txt index defbc149e7e0..2bd318c0454c 100644 --- a/third_party/python/glean_parser/requirements_dev.txt +++ b/third_party/python/glean_parser/requirements_dev.txt @@ -1,13 +1,14 @@ black==20.8b1 -coverage==5.2.1 -flake8==3.8.3 +coverage==5.3 +flake8==3.8.4 +flake8-bugbear==20.1.4 m2r==0.2.1 mypy==0.782 pip pytest-runner==5.2 -pytest==6.0.1 +pytest==6.1.1 Sphinx==3.2.1 twine==3.2.0 watchdog==0.10.3 wheel -yamllint==1.24.2 +yamllint==1.25.0 diff --git a/third_party/python/glean_parser/setup.py b/third_party/python/glean_parser/setup.py index 268131bb3ece..a44af27297aa 100755 --- a/third_party/python/glean_parser/setup.py +++ b/third_party/python/glean_parser/setup.py @@ -51,11 +51,12 @@ setup( "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], description="Parser tools for Mozilla's Glean telemetry", entry_points={ "console_scripts": [ - "glean_parser=glean_parser.__main__:main", + "glean_parser=glean_parser.__main__:main_wrapper", ], }, install_requires=requirements, diff --git a/third_party/python/requirements.in b/third_party/python/requirements.in index e33c867b5268..36b16ed10c98 100644 --- a/third_party/python/requirements.in +++ b/third_party/python/requirements.in @@ -26,7 +26,7 @@ ecdsa==0.15 esprima==4.0.1 fluent.migrate==0.10 fluent.syntax==0.18.1 -glean_parser==1.28.5 +glean_parser==1.29.0 jsmin==2.1.0 json-e==2.7.0 mozilla-version==0.3.4 diff --git a/third_party/python/requirements.txt b/third_party/python/requirements.txt index f4539805f836..9ed7a79dd090 100644 --- a/third_party/python/requirements.txt +++ b/third_party/python/requirements.txt @@ -87,9 +87,9 @@ fluent.syntax==0.18.1 \ --hash=sha256:0e63679fa4f1b3042565220a5127b4bab842424f07d6a13c12299e3b3835486a \ --hash=sha256:3a55f5e605d1b029a65cc8b6492c86ec4608e15447e73db1495de11fd46c104f \ # via -r requirements-mach-vendor-python.in, compare-locales, fluent.migrate -glean_parser==1.28.5 \ - --hash=sha256:29ac33298898e0fd607163b704d68f598c1d118c5056852246d621ec26f973bb \ - --hash=sha256:330e045fd8410f661e8e4a67edc8ab4a125996381e5519c40ca98b34b9dc5ec8 \ +glean_parser==1.29.0 \ + --hash=sha256:7cf1b02ef87fad57bf0f6b9711a98c1fd8f89c9df702245d16c09bf1b042a255 \ + --hash=sha256:df7436e164148594176ec55f7d7c3c5c944daca67c3cc30428514628625b214b \ # via -r requirements-mach-vendor-python.in jinja2==2.11.2 \ --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \