Bug 1664306 - Bump vendored glean_parser version to 1.28.5 r=janerik

Differential Revision: https://phabricator.services.mozilla.com/D90392
This commit is contained in:
Chris H-C 2020-09-22 15:21:55 +00:00
Родитель a3ecb2e052
Коммит 3ce880dd41
18 изменённых файлов: 213 добавлений и 98 удалений

Просмотреть файл

@ -8,6 +8,10 @@ commands:
name: environment
command: |
echo 'export PATH=.:$HOME/.local/bin:$PATH' >> $BASH_ENV
- run:
name: Upgrade pip
command: |
pip install --upgrade --user pip
test-min-requirements:
steps:
@ -23,7 +27,7 @@ commands:
.rb/bin/pip install requirements-builder
.rb/bin/requirements-builder --level=min setup.py > min_requirements.txt
pip install --progress-bar off --user -U -r min_requirements.txt
pip install --use-feature=2020-resolver --progress-bar off --user -U -r min_requirements.txt
test-python-version:
parameters:
@ -34,7 +38,7 @@ commands:
- run:
name: install
command: |
pip install --progress-bar off --user -U -r <<parameters.requirements-file>>
pip install --use-feature=2020-resolver --progress-bar off --user -U -r <<parameters.requirements-file>>
sudo apt update -q
sudo apt upgrade -q
sudo apt install openjdk-11-jdk-headless
@ -131,10 +135,14 @@ jobs:
name: environment
command: |
echo 'export PATH=.:$HOME/.local/bin:$PATH' >> $BASH_ENV
- run:
name: Upgrade pip
command: |
pip install --upgrade --user pip
- run:
name: install
command: |
pip install --user -U -r requirements_dev.txt
pip install --use-feature=2020-resolver --user -U -r requirements_dev.txt
- run:
name: deploy
# Requires that the TWINE_USERNAME and TWINE_PASSWORD environment

6
third_party/python/glean_parser/.github/dependabot.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,6 @@
version: 2
updates:
- package-ecosystem: "pip"
directory: "/"
schedule:
interval: "daily"

35
third_party/python/glean_parser/HISTORY.rst поставляемый
Просмотреть файл

@ -5,6 +5,41 @@ History
Unreleased
----------
1.28.5 (2020-09-14)
-------------------
* Fix deploy step to update pip before deploying to pypi.
1.28.4 (2020-09-14)
-------------------
* The `SUPERFLUOUS_NO_LINT` warning has been removed from the glinter.
It likely did more harm than good, and makes it hard to make
`metrics.yaml` files that pass across different versions of `glean_parser`.
* Expired metrics will now produce a linter warning, `EXPIRED_METRIC`.
* Expiry dates that are more than 730 days (~2 years) in the future will produce a linter warning,
`EXPIRATION_DATE_TOO_FAR`.
* Allow using the Quantity metric type outside of Gecko.
* New parser configs `custom_is_expired` and `custom_validate_expires` added.
These are both functions that take the `expires` value of the metric and return a bool.
(See `Metric.is_expired` and `Metric.validate_expires`).
These will allow FOG to provide custom validation for its version-based `expires` values.
1.28.3 (2020-07-28)
-------------------
* BUGFIX: Support HashSet and Dictionary in the C# generated code.
1.28.2 (2020-07-28)
-------------------
* BUGFIX: Generate valid C# code when using Labeled metric types.
1.28.1 (2020-07-24)
-------------------
* BUGFIX: Add missing column to correctly render markdown tables in generated documentation.
1.28.0 (2020-07-23)
-------------------

37
third_party/python/glean_parser/PKG-INFO поставляемый
Просмотреть файл

@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: glean_parser
Version: 1.28.0
Version: 1.28.5
Summary: Parser tools for Mozilla's Glean telemetry
Home-page: https://github.com/mozilla/glean_parser
Author: Michael Droettboom
@ -69,6 +69,41 @@ Description: ============
Unreleased
----------
1.28.5 (2020-09-14)
-------------------
* Fix deploy step to update pip before deploying to pypi.
1.28.4 (2020-09-14)
-------------------
* The `SUPERFLUOUS_NO_LINT` warning has been removed from the glinter.
It likely did more harm than good, and makes it hard to make
`metrics.yaml` files that pass across different versions of `glean_parser`.
* Expired metrics will now produce a linter warning, `EXPIRED_METRIC`.
* Expiry dates that are more than 730 days (~2 years) in the future will produce a linter warning,
`EXPIRATION_DATE_TOO_FAR`.
* Allow using the Quantity metric type outside of Gecko.
* New parser configs `custom_is_expired` and `custom_validate_expires` added.
These are both functions that take the `expires` value of the metric and return a bool.
(See `Metric.is_expired` and `Metric.validate_expires`).
These will allow FOG to provide custom validation for its version-based `expires` values.
1.28.3 (2020-07-28)
-------------------
* BUGFIX: Support HashSet and Dictionary in the C# generated code.
1.28.2 (2020-07-28)
-------------------
* BUGFIX: Generate valid C# code when using Labeled metric types.
1.28.1 (2020-07-24)
-------------------
* BUGFIX: Add missing column to correctly render markdown tables in generated documentation.
1.28.0 (2020-07-23)
-------------------

Просмотреть файл

@ -12,6 +12,10 @@ import sys
import click
import glean_parser
from . import lint
from . import translate as mod_translate
from . import validate_ping
@ -116,7 +120,7 @@ def glinter(input, allow_reserved):
@click.group()
@click.version_option()
@click.version_option(glean_parser.__version__, prog_name="glean_parser")
def main(args=None):
"""Command line utility for glean_parser."""
pass

Просмотреть файл

@ -24,8 +24,8 @@ def csharp_datatypes_filter(value: util.JSONType) -> str:
Based on Python's JSONEncoder, but overrides:
- lists to use `new string[] {}` (only strings)
- dicts to use mapOf (not currently supported)
- sets to use setOf (not currently supported)
- dicts to use `new Dictionary<string, string> { ...}` (string, string)
- sets to use `new HashSet<string>() {}` (only strings)
- enums to use the like-named C# enum
"""
@ -42,27 +42,29 @@ def csharp_datatypes_filter(value: util.JSONType) -> str:
first = False
yield "}"
elif isinstance(value, dict):
yield "mapOf("
yield "new Dictionary<string, string> {"
first = True
for key, subvalue in value.items():
if not first:
yield ", "
yield "{"
yield from self.iterencode(key)
yield " to "
yield ", "
yield from self.iterencode(subvalue)
yield "}"
first = False
yield ")"
yield "}"
elif isinstance(value, enum.Enum):
yield (value.__class__.__name__ + "." + util.Camelize(value.name))
elif isinstance(value, set):
yield "setOf("
yield "new HashSet<string>() {"
first = True
for subvalue in sorted(list(value)):
if not first:
yield ", "
yield from self.iterencode(subvalue)
first = False
yield ")"
yield "}"
else:
yield from super().iterencode(value)

Просмотреть файл

@ -235,6 +235,22 @@ def check_user_lifetime_expiration(
)
def check_expired_date(
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
) -> LintGenerator:
try:
metric.validate_expires()
except ValueError as e:
yield (str(e))
def check_expired_metric(
metric: metrics.Metric, parser_config: Dict[str, Any] = {}
) -> LintGenerator:
if metric.is_expired():
yield ("Metric has expired. Please consider removing it.")
# The checks that operate on an entire category of metrics:
# {NAME: (function, is_error)}
CATEGORY_CHECKS: Dict[
@ -254,7 +270,9 @@ INDIVIDUAL_CHECKS: Dict[
"BUG_NUMBER": (check_bug_number, CheckType.error),
"BASELINE_PING": (check_valid_in_baseline, CheckType.error),
"MISSPELLED_PING": (check_misspelled_pings, CheckType.error),
"EXPIRATION_DATE_TOO_FAR": (check_expired_date, CheckType.warning),
"USER_LIFETIME_EXPIRATION": (check_user_lifetime_expiration, CheckType.warning),
"EXPIRED": (check_expired_metric, CheckType.warning),
}
@ -318,22 +336,6 @@ def lint_metrics(
)
for msg in new_nits
)
else:
if (
check_name not in CATEGORY_CHECKS
and check_name in metric.no_lint
):
nits.append(
GlinterNit(
"SUPERFLUOUS_NO_LINT",
".".join([metric.category, metric.name]),
(
f"Superfluous no_lint entry '{check_name}'. "
"Please remove it."
),
CheckType.warning,
)
)
if len(nits):
print("Sorry, Glean found some glinter nits:", file=file)

Просмотреть файл

@ -36,6 +36,9 @@ def extra_info(obj: Union[metrics.Metric, pings.Ping]) -> List[Tuple[str, str]]:
if isinstance(obj, metrics.Jwe):
extra_info.append(("decrypted_name", obj.decrypted_name))
if isinstance(obj, metrics.Quantity):
extra_info.append(("unit", obj.unit))
return extra_info

Просмотреть файл

@ -46,7 +46,7 @@ class Metric:
bugs: List[str],
description: str,
notification_emails: List[str],
expires: str,
expires: Any,
data_reviews: Optional[List[str]] = None,
version: int = 0,
disabled: bool = False,
@ -56,7 +56,7 @@ class Metric:
gecko_datapoint: str = "",
no_lint: Optional[List[str]] = None,
data_sensitivity: Optional[List[str]] = None,
_config: Optional[Dict[str, Any]] = None,
_config: Dict[str, Any] = None,
_validated: bool = False,
):
# Avoid cyclical import
@ -98,6 +98,11 @@ class Metric:
for error in parser.validate(data):
raise ValueError(error)
# Store the config, but only after validation.
if _config is None:
_config = {}
self._config = _config
# Metrics in the special category "glean.internal.metrics" need to have
# an empty category string when identifying the metrics in the ping.
if self.category == Metric.glean_internal_metric_cat:
@ -173,11 +178,12 @@ class Metric:
return self.disabled or self.is_expired()
def is_expired(self) -> bool:
return util.is_expired(self.expires)
return self._config.get("custom_is_expired", util.is_expired)(self.expires)
@staticmethod
def validate_expires(expires) -> None:
return util.validate_expires(expires)
def validate_expires(self):
return self._config.get("custom_validate_expires", util.validate_expires)(
self.expires
)
def is_internal_metric(self) -> bool:
return self.category in (Metric.glean_internal_metric_cat, "")

Просмотреть файл

@ -115,7 +115,9 @@ def _get_schema(
if schema_id not in schemas:
raise ValueError(
util.format_error(
filepath, "", f"$schema key must be one of {', '.join(schemas.keys())}",
filepath,
"",
f"$schema key must be one of {', '.join(schemas.keys())}",
)
)
return schemas[schema_id]
@ -204,7 +206,9 @@ def _instantiate_metrics(
)
except Exception as e:
yield util.format_error(
filepath, f"On instance {category_key}.{metric_key}", str(e),
filepath,
f"On instance {category_key}.{metric_key}",
str(e),
)
metric_obj = None
else:

Просмотреть файл

@ -83,8 +83,7 @@ definitions:
- `counter`: A numeric value that can only be incremented.
- `quantity`: A numeric value that is set directly. Only allowed for
metrics coming from GeckoView.
- `quantity`: A numeric value that is set directly.
- `timespan`: Represents a time interval. Additional properties:
`time_unit`.
@ -240,15 +239,17 @@ definitions:
description: |
**Required.**
May be one of the following values:
By default it may be one of the following values:
- `<build date>`: An ISO date `yyyy-mm-dd` in UTC on which the
metric expires. For example, `2019-03-13`. This date is checked at
build time. Except in special cases, this form should be used so
that the metric automatically "sunsets" after a period of time.
- `never`: This metric never expires.
- `expired`: This metric is manually expired.
The default may be overriden in certain applications by the
`custom_validate_expires` and `custom_is_expired` configs.
type: string
pattern: "(never)|(expired)|([0-9]{4}-[0-9]{2}-[0-9]{2})"
version:
title: Metric version
@ -548,12 +549,11 @@ additionalProperties:
type:
enum:
- custom_distribution
- quantity
then:
required:
- gecko_datapoint
description: |
`custom_distribution` and `quantity` is only allowed for Gecko
`custom_distribution` is only allowed for Gecko
metrics.
-
if:

Просмотреть файл

@ -20,6 +20,7 @@ Jinja2 template is not. Please file bugs! #}
){% if lazy %});{% else %};{% endif %}{% endmacro %}
using System;
using System.Collections.Generic;
using {{ glean_namespace }}.Private;
{# The C# metrics design require the class name to have a 'Definition'
@ -56,21 +57,21 @@ namespace {{ namespace }}
{% for obj in objs.values() %}
{% if obj.labeled %}
{{ obj_declaration(obj, 'Label', 'private ') }}
private readonly Lazy<LabeledMetricType<{{ obj|type_name }}>> {{ obj.name|camelize }}Lazy = new Lazy<LabeledMetricType<{{ obj|type_name }}>>(() => new LabeledMetricType(
category = {{ obj.category|csharp }},
name = {{ obj.name|csharp }},
subMetric = {{ obj.name|camelize }}Label,
disabled = {{ obj.is_disabled()|csharp }},
lifetime = {{ obj.lifetime|csharp }},
sendInPings = {{ obj.send_in_pings|csharp }},
labels = {{ obj.labels|csharp }}
private readonly Lazy<LabeledMetricType<{{ obj|type_name }}>> {{ obj.name|camelize }}Lazy = new Lazy<LabeledMetricType<{{ obj|type_name }}>>(() => new LabeledMetricType<{{ obj|type_name }}>(
category: {{ obj.category|csharp }},
name: {{ obj.name|csharp }},
submetric: {{ category_name|Camelize }}.{{ obj.name|camelize }}Label,
disabled: {{ obj.is_disabled()|csharp }},
lifetime: {{ obj.lifetime|csharp }},
sendInPings: {{ obj.send_in_pings|csharp }},
labels: {{ obj.labels|csharp }}
)
);
/// <summary>
/// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }}
/// </summary>
public {{ obj|type_name }} {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value;
public LabeledMetricType<{{ obj|type_name }}> {{ obj.name|camelize }} => {{ obj.name|camelize }}Lazy.Value;
{% else %}
{# Deal with non-ping objects first. We need them to be lazy and we

Просмотреть файл

@ -3,7 +3,7 @@
Jinja2 template is not. Please file bugs! #}
# Metrics
This document enumerates the metrics collected by {{ project_title }}.
This document enumerates the metrics collected by {{ project_title }} using the [Glean SDK](https://mozilla.github.io/glean/book/index.html).
This project may depend on other projects which also collect metrics.
This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project.
@ -61,8 +61,8 @@ This ping includes the [client id](https://mozilla.github.io/glean/book/user/pin
{% if metrics_by_pings[ping_name] %}
The following metrics are added to the ping:
| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefix/Data_Collection) |
| --- | --- | --- | --- | --- | --- |
| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
| --- | --- | --- | --- | --- | --- | --- |
{% for metric in metrics_by_pings[ping_name] %}
| {{ metric.identifier() }} |
{{- '['}}{{ metric.type }}]({{ metric.type|metrics_docs }}) |

Просмотреть файл

@ -336,6 +336,26 @@ def format_error(filepath: Union[str, Path], header: str, content: str) -> str:
return f"{filepath}:\n{_utils.indent(content)}"
def parse_expires(expires: str) -> datetime.date:
"""
Parses the expired field date (yyyy-mm-dd) as a date.
Raises a ValueError in case the string is not properly formatted.
"""
try:
if sys.version_info < (3, 7):
try:
return iso8601.parse_date(expires).date()
except iso8601.ParseError:
raise ValueError()
else:
return datetime.date.fromisoformat(expires)
except ValueError:
raise ValueError(
f"Invalid expiration date '{expires}'. "
"Must be of the form yyyy-mm-dd in UTC."
)
def is_expired(expires: str) -> bool:
"""
Parses the `expires` field in a metric or ping and returns whether
@ -346,29 +366,27 @@ def is_expired(expires: str) -> bool:
elif expires == "expired":
return True
else:
try:
if sys.version_info < (3, 7):
date = iso8601.parse_date(expires).date()
else:
date = datetime.date.fromisoformat(expires)
except ValueError:
raise ValueError(
f"Invalid expiration date '{expires}'. "
"Must be of the form yyyy-mm-dd in UTC."
)
date = parse_expires(expires)
return date <= datetime.datetime.utcnow().date()
def validate_expires(expires: str) -> None:
"""
Raises ValueError if `expires` is not valid.
Raises a ValueError in case the `expires` is not ISO8601 parseable,
or in case the date is more than 730 days (~2 years) in the future.
"""
if expires in ("never", "expired"):
return
if sys.version_info < (3, 7):
iso8601.parse_date(expires)
else:
datetime.date.fromisoformat(expires)
date = parse_expires(expires)
max_date = datetime.datetime.now() + datetime.timedelta(days=730)
if date > max_date.date():
raise ValueError(
f"'{expires}' is more than 730 days (~2 years) in the future.",
"Please make sure this is intentional.",
"You can supress this warning by adding EXPIRATION_DATE_TOO_FAR to no_lint",
"See: https://mozilla.github.io/glean_parser/metrics-yaml.html#no_lint",
)
def report_validation_errors(all_objects):

Просмотреть файл

@ -1,13 +1,13 @@
black==19.10b0
coverage==4.5.2
flake8==3.7.8
black==20.8b1
coverage==5.2.1
flake8==3.8.3
m2r==0.2.1
mypy==0.761
mypy==0.782
pip
pytest-runner==4.4
pytest==4.3.0
Sphinx==1.8.4
twine==1.13.0
watchdog==0.9.0
pytest-runner==5.2
pytest==6.0.1
Sphinx==3.2.1
twine==3.2.0
watchdog==0.10.3
wheel
yamllint==1.18.0
yamllint==1.24.2

6
third_party/python/glean_parser/setup.py поставляемый
Просмотреть файл

@ -53,7 +53,11 @@ setup(
"Programming Language :: Python :: 3.8",
],
description="Parser tools for Mozilla's Glean telemetry",
entry_points={"console_scripts": ["glean_parser=glean_parser.__main__:main",],},
entry_points={
"console_scripts": [
"glean_parser=glean_parser.__main__:main",
],
},
install_requires=requirements,
long_description=readme + "\n\n" + history,
include_package_data=True,

2
third_party/python/requirements.in поставляемый
Просмотреть файл

@ -27,7 +27,7 @@ ecdsa==0.15
esprima==4.0.1
fluent.migrate==0.10
fluent.syntax==0.18.1
glean_parser==1.28.0
glean_parser==1.28.5
jsmin==2.1.0
json-e==2.7.0
mozilla-version==0.3.4

21
third_party/python/requirements.txt поставляемый
Просмотреть файл

@ -90,19 +90,10 @@ fluent.syntax==0.18.1 \
--hash=sha256:0e63679fa4f1b3042565220a5127b4bab842424f07d6a13c12299e3b3835486a \
--hash=sha256:3a55f5e605d1b029a65cc8b6492c86ec4608e15447e73db1495de11fd46c104f \
# via -r requirements-mach-vendor-python.in, compare-locales, fluent.migrate
glean_parser==1.28.0 \
--hash=sha256:994c1d12b686ffe49321496497990e24609e11a8edb27d3afbe0b1f86b01fed3 \
--hash=sha256:a8402d210fd5a50b05cc64d96a8506be972b35a47c78195f04e745756f75db61 \
glean_parser==1.28.5 \
--hash=sha256:29ac33298898e0fd607163b704d68f598c1d118c5056852246d621ec26f973bb \
--hash=sha256:330e045fd8410f661e8e4a67edc8ab4a125996381e5519c40ca98b34b9dc5ec8 \
# via -r requirements-mach-vendor-python.in
importlib-metadata==1.7.0 \
--hash=sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83 \
--hash=sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070 \
# via jsonschema
iso8601==0.1.12 \
--hash=sha256:210e0134677cc0d02f6028087fee1df1e1d76d372ee1db0bf30bf66c5c1c89a3 \
--hash=sha256:49c4b20e1f38aa5cf109ddcd39647ac419f928512c869dc01d5c7098eddede82 \
--hash=sha256:bbbae5fb4a7abfe71d4688fd64bff70b91bbd74ef6a99d964bab18f7fdf286dd \
# via glean-parser
jinja2==2.11.2 \
--hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
--hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035 \
@ -251,7 +242,7 @@ sentry-sdk==0.14.3 \
six==1.13.0 \
--hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \
--hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66 \
# via -r requirements-mach-vendor-python.in, blessings, compare-locales, ecdsa, fluent.migrate, jsonschema, more-itertools, pathlib2, pip-tools, pyrsistent, pytest, responses
# via -r requirements-mach-vendor-python.in, blessings, compare-locales, ecdsa, fluent.migrate, jsonschema, more-itertools, pathlib2, pip-tools, pytest, responses
urllib3==1.25.9 \
--hash=sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527 \
--hash=sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115 \
@ -268,10 +259,6 @@ yamllint==1.23 \
--hash=sha256:0fa69bf8a86182b7fe14918bdd3a30354c869966bbc7cbfff176af71bda9c806 \
--hash=sha256:59f3ff77f44e7f46be6aecdb985830f73a1c51e290b7082a7d38c2ae1940f4a9 \
# via -r requirements-mach-vendor-python.in, glean-parser
zipp==3.1.0 \
--hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \
--hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \
# via importlib-metadata
# WARNING: The following packages were not pinned, but pip requires them to be
# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.