Add event monitoring for VPN [DENG-2620] (#5007)

* Add event monitoring for VPN [DENG-2620]

* VPN support for event error monitoring

bqetl_project.yaml

@@ -436,3 +436,9 @@ generate:
       - org_mozilla_fenix
       - org_mozilla_fenix_beta
       - org_mozilla_firefox
+    events_monitoring:
+      default_event_table: events_v1
+      event_table:
+        mozillavpn: main_v1
+        accounts_frontend: accounts_events_v1
+        accounts_backend: accounts_events_v1

sql_generators/glean_usage/event_error_monitoring.py

@@ -4,6 +4,7 @@ import os
 from collections import namedtuple
 from pathlib import Path
 
+from bigquery_etl.config import ConfigLoader
 from bigquery_etl.schema.stable_table_schema import get_stable_table_schemas
 from sql_generators.glean_usage.common import (
     GleanTable,
@@ -45,11 +46,24 @@ class EventErrorMonitoring(GleanTable):
             and s.bq_table == "events_v1"
         ]
 
+        default_events_table = ConfigLoader.get(
+            "generate",
+            "glean_usage",
+            "events_monitoring",
+            "default_event_table",
+            fallback="events_v1",
+        )
+        events_table_overwrites = ConfigLoader.get(
+            "generate", "glean_usage", "events_monitoring", "event_table", fallback={}
+        )
+
         render_kwargs = dict(
             project_id=project_id,
             target_table=f"{TARGET_DATASET_CROSS_APP}_derived.{AGGREGATE_TABLE_NAME}",
             apps=apps,
             prod_datasets=prod_datasets_with_event,
+            default_events_table=default_events_table,
+            events_table_overwrites=events_table_overwrites,
         )
         render_kwargs.update(self.custom_render_kwargs)
 

sql_generators/glean_usage/event_monitoring_live.py

@@ -5,6 +5,7 @@ from collections import namedtuple
 from datetime import datetime
 from pathlib import Path
 
+from bigquery_etl.config import ConfigLoader
 from bigquery_etl.schema.stable_table_schema import get_stable_table_schemas
 from sql_generators.glean_usage.common import (
     GleanTable,
@@ -52,6 +53,17 @@ class EventMonitoringLive(GleanTable):
         table = tables[f"{self.prefix}"]
         dataset = tables[self.prefix].split(".")[-2].replace("_derived", "")
 
+        default_events_table = ConfigLoader.get(
+            "generate",
+            "glean_usage",
+            "events_monitoring",
+            "default_event_table",
+            fallback="events_v1",
+        )
+        events_table_overwrites = ConfigLoader.get(
+            "generate", "glean_usage", "events_monitoring", "event_table", fallback={}
+        )
+
         render_kwargs = dict(
             header="-- Generated via bigquery_etl.glean_usage\n",
             header_yaml="---\n# Generated via bigquery_etl.glean_usage\n",
@@ -65,6 +77,9 @@ class EventMonitoringLive(GleanTable):
                 for app_dataset in app
                 if dataset == app_dataset["bq_dataset_family"]
             ][0],
+            events_table=default_events_table
+            if dataset not in events_table_overwrites
+            else events_table_overwrites[dataset],
         )
 
         render_kwargs.update(self.custom_render_kwargs)
@@ -123,6 +138,17 @@ class EventMonitoringLive(GleanTable):
         aggregate_table = "event_monitoring_aggregates_v1"
         target_view_name = "_".join(self.target_table_id.split("_")[:-1])
 
+        default_events_table = ConfigLoader.get(
+            "generate",
+            "glean_usage",
+            "events_monitoring",
+            "default_event_table",
+            fallback="events_v1",
+        )
+        events_table_overwrites = ConfigLoader.get(
+            "generate", "glean_usage", "events_monitoring", "event_table", fallback={}
+        )
+
         render_kwargs = dict(
             header="-- Generated via bigquery_etl.glean_usage\n",
             header_yaml="---\n# Generated via bigquery_etl.glean_usage\n",
@@ -132,6 +158,8 @@ class EventMonitoringLive(GleanTable):
             target_table=f"{TARGET_DATASET_CROSS_APP}_derived.{aggregate_table}",
             apps=apps,
             prod_datasets=prod_datasets_with_event,
+            default_events_table=default_events_table,
+            events_table_overwrites=events_table_overwrites,
         )
         render_kwargs.update(self.custom_render_kwargs)
 

sql_generators/glean_usage/templates/event_error_monitoring_aggregates_v1.query.sql

@@ -17,7 +17,11 @@
       client_info.app_channel AS channel,
       metrics.labeled_counter
     FROM
-      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.events_v1`
+      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ 
+        default_events_table
+        if dataset['bq_dataset_family'] not in events_table_overwrites
+        else events_table_overwrites[dataset['bq_dataset_family']] 
+      }}`
     WHERE
         DATE(submission_timestamp) = @submission_date
   )
@@ -70,7 +74,11 @@
       client_info.app_channel AS channel,
       metrics.labeled_counter
     FROM
-      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.accounts_events_v1`
+      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ 
+        default_events_table
+        if dataset['bq_dataset_family'] not in events_table_overwrites
+        else events_table_overwrites[dataset['bq_dataset_family']] 
+      }}`
     WHERE
         DATE(submission_timestamp) = @submission_date
   )

sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql

@@ -61,7 +61,11 @@
     END AS experiment_branch,
     COUNT(*) AS total_events
   FROM
-    `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.events_v1`
+    `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ 
+        default_events_table
+        if dataset['bq_dataset_family'] not in events_table_overwrites
+        else events_table_overwrites[dataset['bq_dataset_family']] 
+    }}`
   CROSS JOIN
     UNNEST(events) AS event,
     -- Iterator for accessing experiments.
@@ -144,7 +148,11 @@
     END AS experiment_branch,
     COUNT(*) AS total_events
   FROM
-    `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.accounts_events_v1`
+    `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ 
+        default_events_table
+        if dataset['bq_dataset_family'] not in events_table_overwrites
+        else events_table_overwrites[dataset['bq_dataset_family']] 
+    }}`
   CROSS JOIN
     -- Iterator for accessing experiments.
     -- Add one more for aggregating events across all experiments

sql_generators/glean_usage/templates/event_monitoring_live_v1.init.sql

@@ -59,7 +59,7 @@ IF
       END AS experiment_branch,
       COUNT(*) AS total_events
     FROM
-      `{{ project_id }}.{{ dataset }}_live.events_v1`
+      `{{ project_id }}.{{ dataset }}_live.{{ events_table }}`
     CROSS JOIN
       UNNEST(events) AS event,
       -- Iterator for accessing experiments.
@@ -124,7 +124,7 @@ IF
       END AS experiment_branch,
       COUNT(*) AS total_events
     FROM
-      `{{ project_id }}.{{ dataset }}_live.accounts_events_v1`
+      `{{ project_id }}.{{ dataset }}_live.{{ events_table }}`
     CROSS JOIN
       -- Iterator for accessing experiments.
       -- Add one more for aggregating events across all experiments