Support multiple Glean channels

opmon/monitoring.py

@@ -2,6 +2,7 @@
 
 import itertools
 import os
+import re
 from asyncio.log import logger
 from datetime import datetime
 from pathlib import Path
@@ -118,6 +119,11 @@ class Monitoring:
         sql = template.render(**render_kwargs)
         return sql
 
+    def _app_id_to_bigquery_dataset(self, app_id: Optional[str]) -> Optional[str]:
+        if app_id is None:
+            return None
+        return re.sub(r"[^a-zA-Z0-9]", "_", app_id).lower()
+
     def _get_metrics_sql(self, submission_date: datetime, first_run: Optional[bool] = None) -> str:
         """Return SQL for data_type ETL."""
         metrics = self.config.metrics
@@ -170,6 +176,18 @@ class Monitoring:
                 if self.config.project
                 else "firefox_desktop"
             ].is_glean_app,
+            "app_id": self._app_id_to_bigquery_dataset(
+                PLATFORM_CONFIGS[
+                    self.config.project.app_name or "firefox_desktop"
+                    if self.config.project
+                    else "firefox_desktop"
+                ].app_id.get(
+                    self.config.project.population.channel.value
+                    if self.config.project.population.channel
+                    else None,
+                    None,
+                )
+            ),
         }
 
         sql_filename = METRIC_QUERY_FILENAME

opmon/platform.py

@@ -37,6 +37,7 @@ class Platform:
 
     app_name: str = attr.ib(validator=_check_value_not_null)
     is_glean_app: bool = True
+    app_id: Dict[str, str] = {}
 
 
 def _generate_platform_config(config: MutableMapping[str, Any]) -> Dict[str, Platform]:
@@ -47,6 +48,7 @@ def _generate_platform_config(config: MutableMapping[str, Any]) -> Dict[str, Pla
         processed_config[platform] = {
             "is_glean_app": platform_config.get("is_glean_app", True),
             "app_name": platform,
+            "app_id": platform_config.get("app_id", {}),
         }
 
     return {

opmon/templates/metric_query.sql

@@ -14,7 +14,7 @@ merged_metrics_{{ data_source }} AS (
         {{ metric.select_expression }} AS {{ metric.name }},
         {% endfor -%}
     FROM
-        {{ metrics[0].data_source.from_expression }}
+        {{ metrics[0].data_source.from_expr_for(app_id) }}
     RIGHT JOIN
         ( 
             SELECT

opmon/templates/population.sql

@@ -46,7 +46,11 @@ WITH population AS (
           END AS branch,
         {% endif %}
     FROM
-        {{ config.population.data_source.from_expression }}
+        {%- if config.population.data_source %}
+        {{ config.population.data_source.from_expr_for(app_id) }}
+        {%- else %}
+        None
+        {%- endif %}
     WHERE
         {% if config.xaxis.value == "submission_date" %}
         DATE({{ config.population.data_source.submission_date_column }}) = DATE('{{ submission_date }}')

opmon/tests/test_monitoring.py

@@ -151,7 +151,7 @@ class TestMonitoring:
 
             [data_sources]
             [data_sources.foo]
-            from_expression = "test"
+            from_expression = "test_data_source"
             """
         )
         spec = MonitoringSpec.from_dict(toml.loads(config_str))
@@ -162,6 +162,46 @@ class TestMonitoring:
             config=spec.resolve(experiment=None, configs=ConfigLoader.configs),
         )
 
-        assert "SELECT 1" in monitoring._get_metrics_sql(
+        sql = monitoring._get_metrics_sql(submission_date=datetime(2022, 1, 2, tzinfo=pytz.utc))
+        assert "SELECT 1" in sql
+        assert "test_data_source" in sql
+
+    def test_get_metric_fenix_sql(self):
+        config_str = dedent(
+            """
+            [project]
+            metrics = ["test"]
+            start_date = "2022-01-01"
+            end_date = "2022-02-01"
+            platform = "fenix"
+
+            [project.population]
+            channel = "nightly"
+            data_source = "foo"
+
+            [metrics]
+            [metrics.test]
+            select_expression = "SELECT 1"
+            data_source = "foo"
+            type = "scalar"
+
+            [metrics.test.statistics]
+            sum = {}
+
+            [data_sources]
+            [data_sources.foo]
+            from_expression = "{dataset}.test"
+            default_dataset = "test"
+            """
+        )
+        spec = MonitoringSpec.from_dict(toml.loads(config_str))
+        monitoring = Monitoring(
+            project="test",
+            dataset="test",
+            slug="test-foo",
+            config=spec.resolve(experiment=None, configs=ConfigLoader.configs),
+        )
+
+        assert "org_mozilla_fenix." in monitoring._get_metrics_sql(
             submission_date=datetime(2022, 1, 2, tzinfo=pytz.utc)
         )

platform_config.toml

@@ -4,26 +4,31 @@
 [platform.firefox_desktop]
 is_glean_app = false
 
-
 [platform.fenix]
 is_glean_app = true
+app_id = { release = "org.mozilla.firefox", beta = "org.mozilla.firefox_beta", nightly = "org.mozilla.fenix" }
 
 
 [platform.firefox_ios]
 is_glean_app = true
+app_id = { release = "org.mozilla.ios.Firefox", beta = "org.mozilla.ios.FirefoxBeta", nightly = "org.mozilla.ios.Fennec" }
 
 
 [platform.klar_ios]
 is_glean_app = true
+app_id = { release = "org.mozilla.ios.Klar" }
 
 
 [platform.focus_ios]
 is_glean_app = true
+app_id = { release = "org.mozilla.ios.Focus" }
 
 
 [platform.klar_android]
 is_glean_app = true
+app_id = { release = "org.mozilla.klar" }
 
 
 [platform.focus_android]
 is_glean_app = true
+app_id = { release = "org.mozilla.focus", beta = "org.mozilla.focus.beta", nightly = "org.mozilla.focus.nightly" }

requirements.in

@@ -70,7 +70,7 @@ iniconfig==1.1.1
     # via pytest
 isort==5.10.1
     # via mozilla-opmon
-mozilla-metric-config-parser==2022.11.1
+mozilla-metric-config-parser==2022.11.2
     # via mozilla-opmon
 jinja2==3.1.0
     # via mozilla-opmon

requirements.txt

@@ -366,9 +366,9 @@ mccabe==0.6.1 \
     # via
     #   -r requirements.in
     #   flake8
-mozilla-metric-config-parser==2022.11.1 \
-    --hash=sha256:66285ff2bdb3b71b6d11b9b03b12328ffd0b92a0853eeee184d30a835d2e5922 \
-    --hash=sha256:dd3acf0a6eddd93d0cd859f4815f6871834b620c05d6dc7ce1eb72a5acd0c612
+mozilla-metric-config-parser==2022.11.2 \
+    --hash=sha256:1ced451f4d3958549ecaf75b07d00a153dd93d105997a644fdc48883296122c7 \
+    --hash=sha256:da6de34235a9a2ba1286149266614c51f7f24c7d37f1ae6127cd0aa38b39b5e2
     # via -r requirements.in
 mypy==0.942 \
     --hash=sha256:0e2dd88410937423fba18e57147dd07cd8381291b93d5b1984626f173a26543e \