diff --git a/opmon/monitoring.py b/opmon/monitoring.py index 5fd86f4..df43215 100644 --- a/opmon/monitoring.py +++ b/opmon/monitoring.py @@ -2,6 +2,7 @@ import itertools import os +import re from asyncio.log import logger from datetime import datetime from pathlib import Path @@ -118,6 +119,11 @@ class Monitoring: sql = template.render(**render_kwargs) return sql + def _app_id_to_bigquery_dataset(self, app_id: Optional[str]) -> Optional[str]: + if app_id is None: + return None + return re.sub(r"[^a-zA-Z0-9]", "_", app_id).lower() + def _get_metrics_sql(self, submission_date: datetime, first_run: Optional[bool] = None) -> str: """Return SQL for data_type ETL.""" metrics = self.config.metrics @@ -170,6 +176,18 @@ class Monitoring: if self.config.project else "firefox_desktop" ].is_glean_app, + "app_id": self._app_id_to_bigquery_dataset( + PLATFORM_CONFIGS[ + self.config.project.app_name or "firefox_desktop" + if self.config.project + else "firefox_desktop" + ].app_id.get( + self.config.project.population.channel.value + if self.config.project.population.channel + else None, + None, + ) + ), } sql_filename = METRIC_QUERY_FILENAME diff --git a/opmon/platform.py b/opmon/platform.py index 3b1473f..f30ffaa 100644 --- a/opmon/platform.py +++ b/opmon/platform.py @@ -37,6 +37,7 @@ class Platform: app_name: str = attr.ib(validator=_check_value_not_null) is_glean_app: bool = True + app_id: Dict[str, str] = {} def _generate_platform_config(config: MutableMapping[str, Any]) -> Dict[str, Platform]: @@ -47,6 +48,7 @@ def _generate_platform_config(config: MutableMapping[str, Any]) -> Dict[str, Pla processed_config[platform] = { "is_glean_app": platform_config.get("is_glean_app", True), "app_name": platform, + "app_id": platform_config.get("app_id", {}), } return { diff --git a/opmon/templates/metric_query.sql b/opmon/templates/metric_query.sql index cd2bd0c..bbb0f91 100644 --- a/opmon/templates/metric_query.sql +++ b/opmon/templates/metric_query.sql @@ -14,7 +14,7 @@ merged_metrics_{{ data_source }} AS ( {{ metric.select_expression }} AS {{ metric.name }}, {% endfor -%} FROM - {{ metrics[0].data_source.from_expression }} + {{ metrics[0].data_source.from_expr_for(app_id) }} RIGHT JOIN ( SELECT diff --git a/opmon/templates/population.sql b/opmon/templates/population.sql index a15d47e..49916ad 100644 --- a/opmon/templates/population.sql +++ b/opmon/templates/population.sql @@ -46,7 +46,11 @@ WITH population AS ( END AS branch, {% endif %} FROM - {{ config.population.data_source.from_expression }} + {%- if config.population.data_source %} + {{ config.population.data_source.from_expr_for(app_id) }} + {%- else %} + None + {%- endif %} WHERE {% if config.xaxis.value == "submission_date" %} DATE({{ config.population.data_source.submission_date_column }}) = DATE('{{ submission_date }}') diff --git a/opmon/tests/test_monitoring.py b/opmon/tests/test_monitoring.py index eda261b..ef867ad 100644 --- a/opmon/tests/test_monitoring.py +++ b/opmon/tests/test_monitoring.py @@ -151,7 +151,7 @@ class TestMonitoring: [data_sources] [data_sources.foo] - from_expression = "test" + from_expression = "test_data_source" """ ) spec = MonitoringSpec.from_dict(toml.loads(config_str)) @@ -162,6 +162,46 @@ class TestMonitoring: config=spec.resolve(experiment=None, configs=ConfigLoader.configs), ) - assert "SELECT 1" in monitoring._get_metrics_sql( + sql = monitoring._get_metrics_sql(submission_date=datetime(2022, 1, 2, tzinfo=pytz.utc)) + assert "SELECT 1" in sql + assert "test_data_source" in sql + + def test_get_metric_fenix_sql(self): + config_str = dedent( + """ + [project] + metrics = ["test"] + start_date = "2022-01-01" + end_date = "2022-02-01" + platform = "fenix" + + [project.population] + channel = "nightly" + data_source = "foo" + + [metrics] + [metrics.test] + select_expression = "SELECT 1" + data_source = "foo" + type = "scalar" + + [metrics.test.statistics] + sum = {} + + [data_sources] + [data_sources.foo] + from_expression = "{dataset}.test" + default_dataset = "test" + """ + ) + spec = MonitoringSpec.from_dict(toml.loads(config_str)) + monitoring = Monitoring( + project="test", + dataset="test", + slug="test-foo", + config=spec.resolve(experiment=None, configs=ConfigLoader.configs), + ) + + assert "org_mozilla_fenix." in monitoring._get_metrics_sql( submission_date=datetime(2022, 1, 2, tzinfo=pytz.utc) ) diff --git a/platform_config.toml b/platform_config.toml index adb901e..3ad67ee 100644 --- a/platform_config.toml +++ b/platform_config.toml @@ -4,26 +4,31 @@ [platform.firefox_desktop] is_glean_app = false - [platform.fenix] is_glean_app = true +app_id = { release = "org.mozilla.firefox", beta = "org.mozilla.firefox_beta", nightly = "org.mozilla.fenix" } [platform.firefox_ios] is_glean_app = true +app_id = { release = "org.mozilla.ios.Firefox", beta = "org.mozilla.ios.FirefoxBeta", nightly = "org.mozilla.ios.Fennec" } [platform.klar_ios] is_glean_app = true +app_id = { release = "org.mozilla.ios.Klar" } [platform.focus_ios] is_glean_app = true +app_id = { release = "org.mozilla.ios.Focus" } [platform.klar_android] is_glean_app = true +app_id = { release = "org.mozilla.klar" } [platform.focus_android] is_glean_app = true +app_id = { release = "org.mozilla.focus", beta = "org.mozilla.focus.beta", nightly = "org.mozilla.focus.nightly" } diff --git a/requirements.in b/requirements.in index c989d06..e6d3d3f 100644 --- a/requirements.in +++ b/requirements.in @@ -70,7 +70,7 @@ iniconfig==1.1.1 # via pytest isort==5.10.1 # via mozilla-opmon -mozilla-metric-config-parser==2022.11.1 +mozilla-metric-config-parser==2022.11.2 # via mozilla-opmon jinja2==3.1.0 # via mozilla-opmon diff --git a/requirements.txt b/requirements.txt index 5bb70ac..e5f8c0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -366,9 +366,9 @@ mccabe==0.6.1 \ # via # -r requirements.in # flake8 -mozilla-metric-config-parser==2022.11.1 \ - --hash=sha256:66285ff2bdb3b71b6d11b9b03b12328ffd0b92a0853eeee184d30a835d2e5922 \ - --hash=sha256:dd3acf0a6eddd93d0cd859f4815f6871834b620c05d6dc7ce1eb72a5acd0c612 +mozilla-metric-config-parser==2022.11.2 \ + --hash=sha256:1ced451f4d3958549ecaf75b07d00a153dd93d105997a644fdc48883296122c7 \ + --hash=sha256:da6de34235a9a2ba1286149266614c51f7f24c7d37f1ae6127cd0aa38b39b5e2 # via -r requirements.in mypy==0.942 \ --hash=sha256:0e2dd88410937423fba18e57147dd07cd8381291b93d5b1984626f173a26543e \