Bug 1675534 - Update glean-parser to v1.29.0. r=janerik

This version bump is required by the Rust glean-core version
bump to v33.1.2.

Differential Revision: https://phabricator.services.mozilla.com/D96305

third_party/python/glean_parser/.circleci/config.yml

@@ -62,14 +62,14 @@ commands:
 jobs:
     build-36:
         docker:
-            - image: circleci/python:3.6.9
+            - image: circleci/python:3.6.12
         steps:
             - test-start
             - test-python-version
 
     build-36-min:
         docker:
-            - image: circleci/python:3.6.9
+            - image: circleci/python:3.6.12
         steps:
             - test-start
             - test-min-requirements
@@ -77,7 +77,7 @@ jobs:
 
     build-37:
         docker:
-            - image: circleci/python:3.7.5
+            - image: circleci/python:3.7.9
         steps:
             - test-start
             - test-python-version
@@ -92,19 +92,26 @@ jobs:
 
     build-38:
         docker:
-            - image: circleci/python:3.8.0
+            - image: circleci/python:3.8.5
         steps:
             - test-start
             - test-python-version
 
     build-38-min:
         docker:
-            - image: circleci/python:3.8.0
+            - image: circleci/python:3.8.5
         steps:
             - test-start
             - test-min-requirements
             - test-python-version
 
+    build-39:
+        docker:
+            - image: circleci/python:3.9.0rc1
+        steps:
+            - test-start
+            - test-python-version
+
     docs-deploy:
         docker:
             - image: node:8.10.0
@@ -174,6 +181,10 @@ workflows:
                     filters:
                         tags:
                             only: /.*/
+            - build-39:
+                    filters:
+                        tags:
+                            only: /.*/
             - docs-deploy:
                     requires:
                         - build-37

third_party/python/glean_parser/CONTRIBUTING.rst

@@ -141,7 +141,7 @@ Get a clean main branch with all of the changes from `upstream`::
 
 - Make sure all your changes are committed.
 
-- Push the changes upstream::
+- Push the changes upstream. (Normally pushing directly without review is frowned upon, but the `main` branch is protected from force pushes and release tagging requires the same permissions as pushing to `main`)::
 
   $ git push upstream main
 
@@ -149,8 +149,12 @@ Get a clean main branch with all of the changes from `upstream`::
 
 - Make the release on GitHub using [this link](https://github.com/mozilla/glean_parser/releases/new)
 
-- Enter the new version in the form `vX.Y.Z`.
+- Both the tag and the release title should be in the form `vX.Y.Z`.
 
 - Copy and paste the relevant part of the `HISTORY.rst` file into the description.
 
+- Tagging the release will trigger a CI workflow which will build the distribution of `glean_parser` and publish it to PyPI.
+
 The continuous integration system will then automatically deploy to PyPI.
+
+See also the [instructions for updating the version of `glean_parser` used by the Glean SDK](https://mozilla.github.io/glean/book/dev/upgrading-glean-parser.html).

third_party/python/glean_parser/HISTORY.rst

@@ -5,6 +5,18 @@ History
 Unreleased
 ----------
 
+1.29.0 (2020-10-07)
+-------------------
+
+* **Breaking change:** `glean_parser` will now return an error code when any of the input files do not exist (unless the `--allow-missing-files` flag is passed).
+* Generated code now includes a comment next to each metric containing the name of the metric in its original `snake_case` form.
+* When metrics don't provide a `unit` parameter, it is not included in the output (as provided by probe-scraper).
+
+1.28.6 (2020-09-24)
+-------------------
+
+* BUGFIX: Ensure Kotlin arguments are deterministically ordered
+
 1.28.5 (2020-09-14)
 -------------------
 

third_party/python/glean_parser/Makefile

@@ -36,9 +36,7 @@ clean-test: ## remove test and coverage artifacts
 
 lint: ## check style with flake8
 	python3 -m flake8 glean_parser tests
-	if python3 --version | grep 'Python 3\.[678]\..*'; then \
-		python3 -m black --check glean_parser tests setup.py; \
-	fi
+	python3 -m black --check glean_parser tests setup.py
 	python3 -m yamllint glean_parser tests
 	python3 -m mypy glean_parser
 

third_party/python/glean_parser/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: glean_parser
-Version: 1.28.5
+Version: 1.29.0
 Summary: Parser tools for Mozilla's Glean telemetry
 Home-page: https://github.com/mozilla/glean_parser
 Author: Michael Droettboom
@@ -69,6 +69,18 @@ Description: ============
         Unreleased
         ----------
         
+        1.29.0 (2020-10-07)
+        -------------------
+        
+        * **Breaking change:** `glean_parser` will now return an error code when any of the input files do not exist (unless the `--allow-missing-files` flag is passed).
+        * Generated code now includes a comment next to each metric containing the name of the metric in its original `snake_case` form.
+        * When metrics don't provide a `unit` parameter, it is not included in the output (as provided by probe-scraper).
+        
+        1.28.6 (2020-09-24)
+        -------------------
+        
+        * BUGFIX: Ensure Kotlin arguments are deterministically ordered
+        
         1.28.5 (2020-09-14)
         -------------------
         
@@ -475,3 +487,4 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9

third_party/python/glean_parser/glean_parser/__main__.py

@@ -53,7 +53,12 @@ from . import validate_ping
         "Should only be set when building the Glean library itself."
     ),
 )
-def translate(input, format, output, option, allow_reserved):
+@click.option(
+    "--allow-missing-files",
+    is_flag=True,
+    help=("Do not treat missing input files as an error."),
+)
+def translate(input, format, output, option, allow_reserved, allow_missing_files):
     """
     Translate metrics.yaml and pings.yaml files to other formats.
     """
@@ -68,7 +73,10 @@ def translate(input, format, output, option, allow_reserved):
             format,
             Path(output),
             option_dict,
-            {"allow_reserved": allow_reserved},
+            {
+                "allow_reserved": allow_reserved,
+                "allow_missing_files": allow_missing_files,
+            },
         )
     )
 
@@ -112,11 +120,24 @@ def check(schema):
         "Should only be set when building the Glean library itself."
     ),
 )
-def glinter(input, allow_reserved):
+@click.option(
+    "--allow-missing-files",
+    is_flag=True,
+    help=("Do not treat missing input files as an error."),
+)
+def glinter(input, allow_reserved, allow_missing_files):
     """
     Runs a linter over the metrics.
     """
-    sys.exit(lint.glinter([Path(x) for x in input], {"allow_reserved": allow_reserved}))
+    sys.exit(
+        lint.glinter(
+            [Path(x) for x in input],
+            {
+                "allow_reserved": allow_reserved,
+                "allow_missing_files": allow_missing_files,
+            },
+        )
+    )
 
 
 @click.group()
@@ -131,5 +152,18 @@ main.add_command(check)
 main.add_command(glinter)
 
 
+def main_wrapper(args=None):
+    """
+    A simple wrapper around click's `main` to display the glean_parser version
+    when there is an error.
+    """
+    try:
+        main(args=args)
+    except SystemExit as e:
+        if e.code != 0:
+            print(f"ERROR running glean_parser v{glean_parser.__version__}")
+        raise
+
+
 if __name__ == "__main__":
-    sys.exit(main())  # pragma: no cover
+    main_wrapper()  # pragma: no cover

third_party/python/glean_parser/glean_parser/csharp.py

@@ -11,7 +11,7 @@ Outputter to generate C# code for metrics.
 import enum
 import json
 from pathlib import Path
-from typing import Any, Dict, List, Union  # noqa
+from typing import Any, Dict, List, Optional, Union  # noqa
 
 from . import metrics
 from . import pings
@@ -104,7 +104,7 @@ def class_name(obj_type: str) -> str:
 
 
 def output_csharp(
-    objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
+    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
 ) -> None:
     """
     Given a tree of objects, output C# code to `output_dir`.
@@ -120,6 +120,9 @@ def output_csharp(
           This is where glean objects will be imported from in the generated
           code.
     """
+    if options is None:
+        options = {}
+
     template = util.get_jinja2_template(
         "csharp.jinja2",
         filters=(

third_party/python/glean_parser/glean_parser/kotlin.py

@@ -12,7 +12,7 @@ from collections import OrderedDict
 import enum
 import json
 from pathlib import Path
-from typing import Any, Dict, List, Union  # noqa
+from typing import Any, Dict, List, Optional, Union  # noqa
 
 from . import metrics
 from . import pings
@@ -102,7 +102,7 @@ def class_name(obj_type: str) -> str:
 
 
 def output_gecko_lookup(
-    objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
+    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
 ) -> None:
     """
     Given a tree of objects, generate a Kotlin map between Gecko histograms and
@@ -119,6 +119,9 @@ def output_gecko_lookup(
           This is where glean objects will be imported from in the generated
           code.
     """
+    if options is None:
+        options = {}
+
     template = util.get_jinja2_template(
         "kotlin.geckoview.jinja2",
         filters=(
@@ -197,7 +200,7 @@ def output_gecko_lookup(
 
 
 def output_kotlin(
-    objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
+    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
 ) -> None:
     """
     Given a tree of objects, output Kotlin code to `output_dir`.
@@ -213,6 +216,9 @@ def output_kotlin(
           This is where glean objects will be imported from in the generated
           code.
     """
+    if options is None:
+        options = {}
+
     template = util.get_jinja2_template(
         "kotlin.jinja2",
         filters=(

third_party/python/glean_parser/glean_parser/lint.py

@@ -7,7 +7,16 @@ import enum
 from pathlib import Path
 import re
 import sys
-from typing import Any, Callable, Dict, Generator, List, Iterable, Tuple, Union  # noqa
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Iterable,
+    Optional,
+    Tuple,
+)  # noqa
 
 
 from . import metrics
@@ -96,7 +105,7 @@ def check_common_prefix(
 
 
 def check_unit_in_name(
-    metric: metrics.Metric, parser_config: Dict[str, Any] = {}
+    metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
     """
     The metric name ends in a unit.
@@ -189,7 +198,7 @@ def check_category_generic(
 
 
 def check_bug_number(
-    metric: metrics.Metric, parser_config: Dict[str, Any] = {}
+    metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
     number_bugs = [str(bug) for bug in metric.bugs if isinstance(bug, int)]
 
@@ -202,7 +211,7 @@ def check_bug_number(
 
 
 def check_valid_in_baseline(
-    metric: metrics.Metric, parser_config: Dict[str, Any] = {}
+    metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
     allow_reserved = parser_config.get("allow_reserved", False)
 
@@ -214,7 +223,7 @@ def check_valid_in_baseline(
 
 
 def check_misspelled_pings(
-    metric: metrics.Metric, parser_config: Dict[str, Any] = {}
+    metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
     for ping in metric.send_in_pings:
         for builtin in pings.RESERVED_PING_NAMES:
@@ -224,7 +233,7 @@ def check_misspelled_pings(
 
 
 def check_user_lifetime_expiration(
-    metric: metrics.Metric, parser_config: Dict[str, Any] = {}
+    metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
 
     if metric.lifetime == metrics.Lifetime.user and metric.expires != "never":
@@ -236,7 +245,7 @@ def check_user_lifetime_expiration(
 
 
 def check_expired_date(
-    metric: metrics.Metric, parser_config: Dict[str, Any] = {}
+    metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
     try:
         metric.validate_expires()
@@ -245,7 +254,7 @@ def check_expired_date(
 
 
 def check_expired_metric(
-    metric: metrics.Metric, parser_config: Dict[str, Any] = {}
+    metric: metrics.Metric, parser_config: Dict[str, Any]
 ) -> LintGenerator:
     if metric.is_expired():
         yield ("Metric has expired. Please consider removing it.")
@@ -291,7 +300,9 @@ class GlinterNit:
 
 
 def lint_metrics(
-    objs: metrics.ObjectTree, parser_config: Dict[str, Any] = {}, file=sys.stderr
+    objs: metrics.ObjectTree,
+    parser_config: Optional[Dict[str, Any]] = None,
+    file=sys.stderr,
 ) -> List[GlinterNit]:
     """
     Performs glinter checks on a set of metrics objects.
@@ -300,6 +311,9 @@ def lint_metrics(
     :param file: The stream to write errors to.
     :returns: List of nits.
     """
+    if parser_config is None:
+        parser_config = {}
+
     nits: List[GlinterNit] = []
     for (category_name, category) in sorted(list(objs.items())):
         if category_name == "pings":
@@ -322,7 +336,7 @@ def lint_metrics(
                 for msg in cat_check_func(category_name, category_metrics.values())
             )
 
-        for (metric_name, metric) in sorted(list(category_metrics.items())):
+        for (_metric_name, metric) in sorted(list(category_metrics.items())):
             for (check_name, (check_func, check_type)) in INDIVIDUAL_CHECKS.items():
                 new_nits = list(check_func(metric, parser_config))
                 if len(new_nits):
@@ -354,7 +368,11 @@ def lint_metrics(
     return nits
 
 
-def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List:
+def lint_yaml_files(
+    input_filepaths: Iterable[Path],
+    file=sys.stderr,
+    parser_config: Dict[str, Any] = None,
+) -> List:
     """
     Performs glinter YAML lint on a set of files.
 
@@ -363,10 +381,16 @@ def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List:
     :returns: List of nits.
     """
 
+    if parser_config is None:
+        parser_config = {}
+
     # Generic type since the actual type comes from yamllint, which we don't
     # control.
     nits: List = []
     for path in input_filepaths:
+        if not path.is_file() and parser_config.get("allow_missing_files", False):
+            continue
+
         # yamllint needs both the file content and the path.
         file_content = None
         with path.open("r", encoding="utf-8") as fd:
@@ -386,7 +410,9 @@ def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List:
 
 
 def glinter(
-    input_filepaths: Iterable[Path], parser_config: Dict[str, Any] = {}, file=sys.stderr
+    input_filepaths: Iterable[Path],
+    parser_config: Optional[Dict[str, Any]] = None,
+    file=sys.stderr,
 ) -> int:
     """
     Commandline helper for glinter.
@@ -397,7 +423,10 @@ def glinter(
     :param file: The stream to write the errors to.
     :return: Non-zero if there were any glinter errors.
     """
-    if lint_yaml_files(input_filepaths, file=file):
+    if parser_config is None:
+        parser_config = {}
+
+    if lint_yaml_files(input_filepaths, file=file, parser_config=parser_config):
         return 1
 
     objs = parser.parse_objects(input_filepaths, parser_config)

third_party/python/glean_parser/glean_parser/markdown.py

@@ -42,7 +42,9 @@ def extra_info(obj: Union[metrics.Metric, pings.Ping]) -> List[Tuple[str, str]]:
     return extra_info
 
 
-def ping_desc(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) -> str:
+def ping_desc(
+    ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
+) -> str:
     """
     Return a text description of the ping. If a custom_pings_cache
     is available, look in there for non-reserved ping names description.
@@ -56,7 +58,7 @@ def ping_desc(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) ->
         )
     elif ping_name == "all-pings":
         desc = "These metrics are sent in every ping."
-    elif ping_name in custom_pings_cache:
+    elif custom_pings_cache is not None and ping_name in custom_pings_cache:
         desc = custom_pings_cache[ping_name].description
 
     return desc
@@ -87,8 +89,10 @@ def ping_docs(ping_name: str) -> str:
     return f"https://mozilla.github.io/glean/book/user/pings/{ping_name}.html"
 
 
-def if_empty(ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}) -> bool:
-    if ping_name in custom_pings_cache:
+def if_empty(
+    ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
+) -> bool:
+    if custom_pings_cache is not None and ping_name in custom_pings_cache:
         return custom_pings_cache[ping_name].send_if_empty
     else:
         return False
@@ -109,27 +113,27 @@ def ping_reasons(
 
 
 def ping_data_reviews(
-    ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}
+    ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
 ) -> Optional[List[str]]:
-    if ping_name in custom_pings_cache:
+    if custom_pings_cache is not None and ping_name in custom_pings_cache:
         return custom_pings_cache[ping_name].data_reviews
     else:
         return None
 
 
 def ping_bugs(
-    ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}
+    ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
 ) -> Optional[List[str]]:
-    if ping_name in custom_pings_cache:
+    if custom_pings_cache is not None and ping_name in custom_pings_cache:
         return custom_pings_cache[ping_name].bugs
     else:
         return None
 
 
 def ping_include_client_id(
-    ping_name: str, custom_pings_cache: Dict[str, pings.Ping] = {}
+    ping_name: str, custom_pings_cache: Optional[Dict[str, pings.Ping]] = None
 ) -> bool:
-    if ping_name in custom_pings_cache:
+    if custom_pings_cache is not None and ping_name in custom_pings_cache:
         return custom_pings_cache[ping_name].include_client_id
     else:
         return False
@@ -145,7 +149,7 @@ def data_sensitivity_numbers(
 
 
 def output_markdown(
-    objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
+    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
 ) -> None:
     """
     Given a tree of objects, output Markdown docs to `output_dir`.
@@ -159,6 +163,8 @@ def output_markdown(
     :param options: options dictionary, with the following optional key:
         - `project_title`: The projects title.
     """
+    if options is None:
+        options = {}
 
     # Build a dictionary that associates pings with their metrics.
     #
@@ -177,7 +183,7 @@ def output_markdown(
     # This also builds a dictionary of custom pings, if available.
     custom_pings_cache: Dict[str, pings.Ping] = defaultdict()
     metrics_by_pings: Dict[str, List[metrics.Metric]] = defaultdict(list)
-    for category_key, category_val in objs.items():
+    for _category_key, category_val in objs.items():
         for obj in category_val.values():
             # Filter out custom pings. We will need them for extracting
             # the description

third_party/python/glean_parser/glean_parser/metrics.py

@@ -52,7 +52,7 @@ class Metric:
         disabled: bool = False,
         lifetime: str = "ping",
         send_in_pings: Optional[List[str]] = None,
-        unit: str = "",
+        unit: Optional[str] = None,
         gecko_datapoint: str = "",
         no_lint: Optional[List[str]] = None,
         data_sensitivity: Optional[List[str]] = None,
@@ -78,7 +78,8 @@ class Metric:
         if send_in_pings is None:
             send_in_pings = ["default"]
         self.send_in_pings = send_in_pings
-        self.unit = unit
+        if unit is not None:
+            self.unit = unit
         self.gecko_datapoint = gecko_datapoint
         if no_lint is None:
             no_lint = []
@@ -120,7 +121,7 @@ class Metric:
         category: str,
         name: str,
         metric_info: Dict[str, util.JSONType],
-        config: Dict[str, Any] = {},
+        config: Optional[Dict[str, Any]] = None,
         validated: bool = False,
     ):
         """
@@ -136,6 +137,9 @@ class Metric:
             jsonschema validation
         :return: A new Metric instance.
         """
+        if config is None:
+            config = {}
+
         metric_type = metric_info["type"]
         if not isinstance(metric_type, str):
             raise TypeError(f"Unknown metric type {metric_type}")

third_party/python/glean_parser/glean_parser/parser.py

@@ -53,13 +53,21 @@ def _update_validator(validator):
 
 
 def _load_file(
-    filepath: Path,
+    filepath: Path, parser_config: Dict[str, Any]
 ) -> Generator[str, None, Tuple[Dict[str, util.JSONType], Optional[str]]]:
     """
     Load a metrics.yaml or pings.yaml format file.
+
+    If the `filepath` does not exist, raises `FileNotFoundError`, unless
+    `parser_config["allow_missing_files"]` is `True`.
     """
     try:
         content = util.load_yaml_or_json(filepath, ordered_dict=True)
+    except FileNotFoundError:
+        if not parser_config.get("allow_missing_files", False):
+            raise
+        else:
+            return {}, None
     except Exception as e:
         yield util.format_error(filepath, "", textwrap.fill(str(e)))
         return {}, None
@@ -313,7 +321,7 @@ def _preprocess_objects(objs: ObjectTree, config: Dict[str, Any]) -> ObjectTree:
 
 @util.keep_value
 def parse_objects(
-    filepaths: Iterable[Path], config: Dict[str, Any] = {}
+    filepaths: Iterable[Path], config: Optional[Dict[str, Any]] = None
 ) -> Generator[str, None, ObjectTree]:
     """
     Parse one or more metrics.yaml and/or pings.yaml files, returning a tree of
@@ -342,12 +350,17 @@ def parse_objects(
           This is useful when you want to retain the original "disabled"
           value from the `metrics.yaml`, rather than having it overridden when
           the metric expires.
+        - `allow_missing_files`: Do not raise a `FileNotFoundError` if any of
+          the input `filepaths` do not exist.
     """
+    if config is None:
+        config = {}
+
     all_objects: ObjectTree = OrderedDict()
     sources: Dict[Any, Path] = {}
     filepaths = util.ensure_list(filepaths)
     for filepath in filepaths:
-        content, filetype = yield from _load_file(filepath)
+        content, filetype = yield from _load_file(filepath, config)
         if filetype == "metrics":
             yield from _instantiate_metrics(
                 all_objects, sources, content, filepath, config

third_party/python/glean_parser/glean_parser/pings.py

@@ -73,3 +73,9 @@ class Ping:
         d = self.__dict__.copy()
         del d["name"]
         return d
+
+    def identifier(self) -> str:
+        """
+        Used for the "generated from ..." comment in the output.
+        """
+        return self.name

third_party/python/glean_parser/glean_parser/swift.py

@@ -11,7 +11,7 @@ Outputter to generate Swift code for metrics.
 import enum
 import json
 from pathlib import Path
-from typing import Any, Dict, Union
+from typing import Any, Dict, Optional, Union
 
 from . import metrics
 from . import pings
@@ -117,7 +117,7 @@ class Category:
 
 
 def output_swift(
-    objs: metrics.ObjectTree, output_dir: Path, options: Dict[str, Any] = {}
+    objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
 ) -> None:
     """
     Given a tree of objects, output Swift code to `output_dir`.
@@ -130,6 +130,9 @@ def output_swift(
         - glean_namespace: The namespace to import Glean from
         - allow_reserved: When True, this is a Glean-internal build
     """
+    if options is None:
+        options = {}
+
     template = util.get_jinja2_template(
         "swift.jinja2",
         filters=(

third_party/python/glean_parser/glean_parser/templates/csharp.jinja2

@@ -11,7 +11,7 @@ Jinja2 template is not. Please file bugs! #}
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 {% macro obj_declaration(obj, suffix='', access='', lazy=False) %}
 {{ access }} {% if lazy %} Lazy<{{ obj|type_name }}>{%- else %} {{ obj|type_name }}{% endif %} {{ obj.name|camelize }}{{ suffix }}
-{%- if lazy %} = new Lazy<{{ obj|type_name }}>(() => {%- else %} ={% endif %}
+{%- if lazy %} = new Lazy<{{ obj|type_name }}>(() => {%- else %} = // generated from {{ obj.identifier() }}{% endif %}
 
             new {{ obj|type_name }}(
                 {% for arg_name in extra_args if obj[arg_name] is defined %}
@@ -57,7 +57,7 @@ namespace {{ namespace }}
     {% for obj in objs.values() %}
         {% if obj.labeled %}
         {{ obj_declaration(obj, 'Label', 'private ') }}
-        private readonly Lazy<LabeledMetricType<{{ obj|type_name }}>> {{ obj.name|camelize }}Lazy = new Lazy<LabeledMetricType<{{ obj|type_name }}>>(() => new LabeledMetricType<{{ obj|type_name }}>(
+        private readonly Lazy<LabeledMetricType<{{ obj|type_name }}>> {{ obj.name|camelize }}Lazy = new Lazy<LabeledMetricType<{{ obj|type_name }}>>(() => new LabeledMetricType<{{ obj|type_name }}>( // generated from {{ obj.identifier() }}
                 category: {{ obj.category|csharp }},
                 name: {{ obj.name|csharp }},
                 submetric: {{ category_name|Camelize }}.{{ obj.name|camelize }}Label,
@@ -82,7 +82,7 @@ namespace {{ namespace }}
         /// <summary>
         /// {{ obj.description|wordwrap() | replace('\n', '\n        /// ') }}
         /// </summary>
-        internal {{ obj|type_name }} {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value;
+        internal {{ obj|type_name }} {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value; // generated from {{ obj.identifier() }}
 
         {% else %}
         {# Finally handle pings. #}

third_party/python/glean_parser/glean_parser/templates/kotlin.jinja2

@@ -13,7 +13,7 @@ Jinja2 template is not. Please file bugs! #}
 {% if (access != "private ") -%}
 @get:JvmName("{{ obj.name|camelize }}{{ suffix }}")
 {% endif -%}
-{{ access }}val {{ obj.name|camelize }}{{ suffix }}: {{ obj|type_name }}{% if lazy %} by lazy { {%- else %} ={% endif %}
+{{ access }}val {{ obj.name|camelize }}{{ suffix }}: {{ obj|type_name }}{% if lazy %} by lazy { {%- else %} ={% endif %} // generated from {{ obj.identifier() }}
 
         {{ obj|type_name }}(
             {% for arg_name in extra_args if obj[arg_name] is defined %}
@@ -60,7 +60,7 @@ internal object {{ category_name|Camelize }} {
     /**
      * {{ obj.description|wordwrap() | replace('\n', '\n        * ') }}
      */
-    val {{ obj.name|camelize }}: LabeledMetricType<{{ obj|type_name }}> by lazy {
+    val {{ obj.name|camelize }}: LabeledMetricType<{{ obj|type_name }}> by lazy { // generated from {{ obj.identifier() }}
         LabeledMetricType(
             category = {{ obj.category|kotlin }},
             name = {{ obj.name|kotlin }},

third_party/python/glean_parser/glean_parser/templates/swift.jinja2

@@ -8,7 +8,7 @@ Jinja2 template is not. Please file bugs! #}
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 {% macro obj_declaration(obj, suffix='', access='') %}
-{{ access }}static let {{ obj.name|camelize|variable_name }}{{ suffix }} = {{ obj|type_name }}(
+{{ access }}static let {{ obj.name|camelize|variable_name }}{{ suffix }} = {{ obj|type_name }}( // generated from {{ obj.identifier() }}
             {% for arg_name in extra_args if obj[arg_name] is defined %}
             {{ arg_name|camelize }}: {{ obj[arg_name]|swift }}{{ "," if not loop.last }}
             {% endfor %}
@@ -86,7 +86,7 @@ extension {{ namespace }} {
         {% if obj.labeled %}
         {{ obj_declaration(obj, 'Label', 'private ') }}
         /// {{ obj.description|wordwrap() | replace('\n', '\n        /// ') }}
-        static let {{ obj.name|camelize|variable_name }} = try! LabeledMetricType<{{ obj|type_name }}>(
+        static let {{ obj.name|camelize|variable_name }} = try! LabeledMetricType<{{ obj|type_name }}>( // generated from {{ obj.identifier() }}
             category: {{ obj.category|swift }},
             name: {{ obj.name|swift }},
             sendInPings: {{ obj.send_in_pings|swift }},

third_party/python/glean_parser/glean_parser/translate.py

@@ -12,7 +12,7 @@ from pathlib import Path
 import os
 import shutil
 import tempfile
-from typing import Any, Callable, Dict, Iterable, List
+from typing import Any, Callable, Dict, Iterable, List, Optional
 
 from . import lint
 from . import parser
@@ -40,8 +40,11 @@ class Outputter:
     def __init__(
         self,
         output_func: Callable[[metrics.ObjectTree, Path, Dict[str, Any]], None],
-        clear_patterns: List[str] = [],
+        clear_patterns: Optional[List[str]] = None,
     ):
+        if clear_patterns is None:
+            clear_patterns = []
+
         self.output_func = output_func
         self.clear_patterns = clear_patterns
 
@@ -49,7 +52,7 @@ class Outputter:
 OUTPUTTERS = {
     "csharp": Outputter(csharp.output_csharp, ["*.cs"]),
     "kotlin": Outputter(kotlin.output_kotlin, ["*.kt"]),
-    "markdown": Outputter(markdown.output_markdown),
+    "markdown": Outputter(markdown.output_markdown, []),
     "swift": Outputter(swift.output_swift, ["*.swift"]),
 }
 
@@ -58,9 +61,9 @@ def translate_metrics(
     input_filepaths: Iterable[Path],
     output_dir: Path,
     translation_func: Callable[[metrics.ObjectTree, Path, Dict[str, Any]], None],
-    clear_patterns: List[str] = [],
-    options: Dict[str, Any] = {},
-    parser_config: Dict[str, Any] = {},
+    clear_patterns: Optional[List[str]] = None,
+    options: Optional[Dict[str, Any]] = None,
+    parser_config: Optional[Dict[str, Any]] = None,
 ):
     """
     Translate the files in `input_filepaths` by running the metrics through a
@@ -85,6 +88,15 @@ def translate_metrics(
     :param parser_config: A dictionary of options that change parsing behavior.
         See `parser.parse_metrics` for more info.
     """
+    if clear_patterns is None:
+        clear_patterns = []
+
+    if options is None:
+        options = {}
+
+    if parser_config is None:
+        parser_config = {}
+
     input_filepaths = util.ensure_list(input_filepaths)
 
     if lint.glinter(input_filepaths, parser_config):
@@ -128,8 +140,8 @@ def translate(
     input_filepaths: Iterable[Path],
     output_format: str,
     output_dir: Path,
-    options: Dict[str, Any] = {},
-    parser_config: Dict[str, Any] = {},
+    options: Optional[Dict[str, Any]] = None,
+    parser_config: Optional[Dict[str, Any]] = None,
 ):
     """
     Translate the files in `input_filepaths` to the given `output_format` and
@@ -143,6 +155,12 @@ def translate(
     :param parser_config: A dictionary of options that change parsing behavior.
         See `parser.parse_metrics` for more info.
     """
+    if options is None:
+        options = {}
+
+    if parser_config is None:
+        parser_config = {}
+
     format_desc = OUTPUTTERS.get(output_format, None)
 
     if format_desc is None:

third_party/python/glean_parser/glean_parser/util.py

@@ -114,14 +114,12 @@ def load_yaml_or_json(path: Path, ordered_dict: bool = False):
     :param path: `pathlib.Path` object
     :rtype object: The tree of objects as a result of parsing the file.
     :raises ValueError: The file is neither a .json, .yml or .yaml file.
+    :raises FileNotFoundError: The file does not exist.
     """
     # If in py.test, support bits of literal JSON/YAML content
     if TESTING_MODE and isinstance(path, dict):
         return path
 
-    if not path.is_file():
-        return {}
-
     if path.suffix == ".json":
         with path.open("r", encoding="utf-8") as fd:
             return json.load(fd)
@@ -434,5 +432,7 @@ extra_ping_args = [
 ]
 
 
-# Names of parameters to pass to both metric and ping constructors.
-extra_args = list(set(extra_metric_args) | set(extra_ping_args))
+# Names of parameters to pass to both metric and ping constructors (no duplicates).
+extra_args = extra_metric_args + [
+    v for v in extra_ping_args if v not in extra_metric_args
+]

third_party/python/glean_parser/requirements_dev.txt

@@ -1,13 +1,14 @@
 black==20.8b1
-coverage==5.2.1
-flake8==3.8.3
+coverage==5.3
+flake8==3.8.4
+flake8-bugbear==20.1.4
 m2r==0.2.1
 mypy==0.782
 pip
 pytest-runner==5.2
-pytest==6.0.1
+pytest==6.1.1
 Sphinx==3.2.1
 twine==3.2.0
 watchdog==0.10.3
 wheel
-yamllint==1.24.2
+yamllint==1.25.0

third_party/python/glean_parser/setup.py

@@ -51,11 +51,12 @@ setup(
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
     description="Parser tools for Mozilla's Glean telemetry",
     entry_points={
         "console_scripts": [
-            "glean_parser=glean_parser.__main__:main",
+            "glean_parser=glean_parser.__main__:main_wrapper",
         ],
     },
     install_requires=requirements,

third_party/python/requirements.in

@@ -26,7 +26,7 @@ ecdsa==0.15
 esprima==4.0.1
 fluent.migrate==0.10
 fluent.syntax==0.18.1
-glean_parser==1.28.5
+glean_parser==1.29.0
 jsmin==2.1.0
 json-e==2.7.0
 mozilla-version==0.3.4

third_party/python/requirements.txt

@@ -87,9 +87,9 @@ fluent.syntax==0.18.1 \
     --hash=sha256:0e63679fa4f1b3042565220a5127b4bab842424f07d6a13c12299e3b3835486a \
     --hash=sha256:3a55f5e605d1b029a65cc8b6492c86ec4608e15447e73db1495de11fd46c104f \
     # via -r requirements-mach-vendor-python.in, compare-locales, fluent.migrate
-glean_parser==1.28.5 \
-    --hash=sha256:29ac33298898e0fd607163b704d68f598c1d118c5056852246d621ec26f973bb \
-    --hash=sha256:330e045fd8410f661e8e4a67edc8ab4a125996381e5519c40ca98b34b9dc5ec8 \
+glean_parser==1.29.0 \
+    --hash=sha256:7cf1b02ef87fad57bf0f6b9711a98c1fd8f89c9df702245d16c09bf1b042a255 \
+    --hash=sha256:df7436e164148594176ec55f7d7c3c5c944daca67c3cc30428514628625b214b \
     # via -r requirements-mach-vendor-python.in
 jinja2==2.11.2 \
     --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \