OpMon alerting (#447)

generator/dashboards/operational_monitoring_dashboard.py

@@ -68,47 +68,42 @@ class OperationalMonitoringDashboard(Dashboard):
             "elements": [],
             "dimensions": [],
             "group_by_dimension": self.group_by_dimension,
+            "alerts": None,
         }
 
         includes = []
         graph_index = 0
         for table_defn in self.tables:
-            if len(kwargs["dimensions"]) == 0:
-                kwargs["dimensions"] = [
-                    {
-                        "name": name,
-                        "title": lookml_utils.slug_to_title(name),
-                        "default": info["default"],
-                        "options": info["options"],
-                    }
-                    for name, info in self.dimensions.items()
-                ]
-
             explore = table_defn["explore"]
             includes.append(
                 f"/looker-hub/{self.namespace}/explores/{explore}.explore.lkml"
             )
 
-            series_colors = self._map_series_to_colours(table_defn["branches"], explore)
-            for metric in table_defn.get("probes", []):
-                title = lookml_utils.slug_to_title(metric)
-                kwargs["elements"].append(
-                    {
-                        "title": title,
-                        "metric": metric,
-                        "explore": explore,
-                        "series_colors": series_colors,
-                        "xaxis": self.xaxis,
-                        "row": int(graph_index / 2) * 10,
-                        "col": 0 if graph_index % 2 == 0 else 12,
-                    }
-                )
-                graph_index += 1
+            if table_defn["table"].endswith("alerts"):
+                kwargs["alerts"] = {
+                    "explore": explore,
+                    "col": 0,
+                }
+            else:
+                if len(kwargs["dimensions"]) == 0:
+                    kwargs["dimensions"] = [
+                        {
+                            "name": name,
+                            "title": lookml_utils.slug_to_title(name),
+                            "default": info["default"],
+                            "options": info["options"],
+                        }
+                        for name, info in self.dimensions.items()
+                    ]
 
-                if self.group_by_dimension:
+                series_colors = self._map_series_to_colours(
+                    table_defn["branches"], explore
+                )
+                for metric in table_defn.get("probes", []):
+                    title = lookml_utils.slug_to_title(metric)
                     kwargs["elements"].append(
                         {
-                            "title": f"{title} - By {self.group_by_dimension}",
+                            "title": title,
                             "metric": metric,
                             "explore": explore,
                             "series_colors": series_colors,
@@ -119,6 +114,23 @@ class OperationalMonitoringDashboard(Dashboard):
                     )
                     graph_index += 1
 
+                    if self.group_by_dimension:
+                        kwargs["elements"].append(
+                            {
+                                "title": f"{title} - By {self.group_by_dimension}",
+                                "metric": metric,
+                                "explore": explore,
+                                "series_colors": series_colors,
+                                "xaxis": self.xaxis,
+                                "row": int(graph_index / 2) * 10,
+                                "col": 0 if graph_index % 2 == 0 else 12,
+                            }
+                        )
+                        graph_index += 1
+
+        if "alerts" in kwargs and kwargs["alerts"] is not None:
+            kwargs["alerts"]["row"] = (int(graph_index / 2) * 10,)
+
         dash_lookml = lookml_utils.render_template(
             "dashboard.lkml", "dashboards", **kwargs
         )

generator/dashboards/templates/dashboard.lkml

@@ -41,6 +41,63 @@
     {{ branch }}: "{{ color }}"
     {%- endfor %}
   {% endfor -%}
+  {% if alerts is not none %}
+  - title: Alerts
+    name: Alerts
+    model: operational_monitoring
+    explore: {{alerts.explore}}
+    type: looker_grid
+    fields: [{{alerts.explore}}.submission_date,
+      {{alerts.explore}}.probe, {{alerts.explore}}.percentile,
+      {{alerts.explore}}.message, {{alerts.explore}}.branch, {{alerts.explore}}.errors]
+    sorts: [{{alerts.explore}}.submission_date
+        desc]
+    limit: 500
+    show_view_names: false
+    show_row_numbers: true
+    transpose: false
+    truncate_text: true
+    hide_totals: false
+    hide_row_totals: false
+    size_to_fit: true
+    table_theme: white
+    limit_displayed_rows: false
+    enable_conditional_formatting: false
+    header_text_alignment: left
+    header_font_size: 12
+    rows_font_size: 12
+    conditional_formatting_include_totals: false
+    conditional_formatting_include_nulls: false
+    x_axis_gridlines: false
+    y_axis_gridlines: true
+    show_y_axis_labels: true
+    show_y_axis_ticks: true
+    y_axis_tick_density: default
+    y_axis_tick_density_custom: 5
+    show_x_axis_label: true
+    show_x_axis_ticks: true
+    y_axis_scale_mode: linear
+    x_axis_reversed: false
+    y_axis_reversed: false
+    plot_size_by_field: false
+    trellis: ''
+    stacking: ''
+    legend_position: center
+    point_style: none
+    show_value_labels: false
+    label_density: 25
+    x_axis_scale: auto
+    y_axis_combined: true
+    show_null_points: true
+    interpolation: linear
+    defaults_version: 1
+    series_types: {}
+    listen: {}
+    row: {{ alerts.row }}
+    col: {{ alerts.col }}
+    width: 24
+    height: 6
+  {% endif %}
   filters:
   - name: Percentile
     title: Percentile

generator/explores/__init__.py

@@ -5,7 +5,10 @@ from .events_explore import EventsExplore
 from .funnel_analysis_explore import FunnelAnalysisExplore
 from .glean_ping_explore import GleanPingExplore
 from .growth_accounting_explore import GrowthAccountingExplore
-from .operational_monitoring_explore import OperationalMonitoringExplore
+from .operational_monitoring_explore import (
+    OperationalMonitoringAlertingExplore,
+    OperationalMonitoringExplore,
+)
 from .ping_explore import PingExplore
 
 EXPLORE_TYPES = {
@@ -16,4 +19,5 @@ EXPLORE_TYPES = {
     PingExplore.type: PingExplore,
     GrowthAccountingExplore.type: GrowthAccountingExplore,
     OperationalMonitoringExplore.type: OperationalMonitoringExplore,
+    OperationalMonitoringAlertingExplore.type: OperationalMonitoringAlertingExplore,
 }

generator/explores/operational_monitoring_explore.py

@@ -98,3 +98,75 @@ class OperationalMonitoringExplore(Explore):
         ]
 
         return defn
+
+
+class OperationalMonitoringAlertingExplore(Explore):
+    """An Operational Monitoring Alerting Explore."""
+
+    type: str = "operational_monitoring_alerting_explore"
+
+    def __init__(
+        self,
+        name: str,
+        views: Dict[str, str],
+        views_path: Path = None,
+        defn: Dict[str, Any] = None,
+    ):
+        """Initialize OperationalMonitoringExplore."""
+        super().__init__(name, views, views_path)
+
+    @staticmethod
+    def from_views(views: List[View]) -> Iterator[Explore]:
+        """Generate an Operational Monitoring explore for this namespace."""
+        for view in views:
+            if view.view_type in {
+                "operational_monitoring_alerting_view",
+            }:
+                yield OperationalMonitoringAlertingExplore(
+                    "operational_monitoring",
+                    {"base_view": view.name},
+                )
+
+    @staticmethod
+    def from_dict(
+        name: str, defn: dict, views_path: Path
+    ) -> OperationalMonitoringAlertingExplore:
+        """Get an instance of this explore from a dictionary definition."""
+        return OperationalMonitoringAlertingExplore(
+            name, defn["views"], views_path, defn
+        )
+
+    def _to_lookml(
+        self,
+        bq_client: bigquery.Client,
+        v1_name: Optional[str],
+    ) -> List[Dict[str, Any]]:
+        aggregate_tables = []
+        aggregate_tables.append(
+            {
+                "name": "rollup_alerts",
+                "query": {
+                    "dimensions": [
+                        "submission_date",
+                        "branch",
+                        "percentile",
+                        "probe",
+                        "message",
+                    ],
+                    "measures": ["errors"],
+                },
+                "materialization": {
+                    # Reload the table at 9am when ETL should have been completed
+                    "sql_trigger_value": "SELECT CAST(TIMESTAMP_SUB(CURRENT_TIMESTAMP, INTERVAL 9 HOUR) AS DATE)"
+                },
+            }
+        )
+
+        defn: List[Dict[str, Any]] = [
+            {
+                "name": self.views["base_view"],
+                "aggregate_table": aggregate_tables,
+            },
+        ]
+
+        return defn

generator/namespaces.py

@@ -128,6 +128,21 @@ def _get_opmon(bq_client: bigquery.Client, namespaces: Dict[str, Any]):
                 ],
             }
 
+        if "alerting" in project and project["alerting"]:
+            # create an alerting view if available
+            om_content["views"][f"{table_prefix}_alerts"] = {
+                "type": "operational_monitoring_alerting_view",
+                "tables": [
+                    {
+                        "table": f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}_alerts",
+                    }
+                ],
+            }
+            om_content["explores"][f"{table_prefix}_alerts"] = {
+                "type": "operational_monitoring_alerting_explore",
+                "views": {"base_view": f"{table_prefix}_alerts"},
+            }
+
         om_content["dashboards"][table_prefix] = {
             "type": "operational_monitoring_dashboard",
             "title": project_name,
@@ -149,6 +164,14 @@ def _get_opmon(bq_client: bigquery.Client, namespaces: Dict[str, Any]):
             ],
         }
 
+        if "alerting" in project and project["alerting"]:
+            om_content["dashboards"][table_prefix]["tables"].append(
+                {
+                    "explore": f"{table_prefix}_alerts",
+                    "table": f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}_alerts",
+                }
+            )
+
     return om_content
 
 

generator/views/__init__.py

@@ -4,6 +4,7 @@ from .events_view import EventsView
 from .funnel_analysis_view import FunnelAnalysisView
 from .glean_ping_view import GleanPingView
 from .growth_accounting_view import GrowthAccountingView
+from .operational_monitoring_alerting_view import OperationalMonitoringAlertingView
 from .operational_monitoring_histogram_view import OperationalMonitoringHistogramView
 from .operational_monitoring_scalar_view import OperationalMonitoringScalarView
 from .ping_view import PingView
@@ -19,5 +20,6 @@ VIEW_TYPES = {
     GrowthAccountingView.type: GrowthAccountingView,
     OperationalMonitoringScalarView.type: OperationalMonitoringScalarView,
     OperationalMonitoringHistogramView.type: OperationalMonitoringHistogramView,
+    OperationalMonitoringAlertingView.type: OperationalMonitoringAlertingView,
     TableView.type: TableView,
 }

generator/views/operational_monitoring_alerting_view.py

@@ -0,0 +1,45 @@
+"""Class to describe an Operational Monitoring Alert View."""
+
+from typing import Any, Dict, Optional
+
+from . import lookml_utils
+from .operational_monitoring_view import OperationalMonitoringView
+
+
+class OperationalMonitoringAlertingView(OperationalMonitoringView):
+    """A view on a alert operational monitoring table."""
+
+    type: str = "operational_monitoring_alerting_view"
+
+    def to_lookml(self, bq_client, v1_name: Optional[str]) -> Dict[str, Any]:
+        """Get this view as LookML."""
+        if len(self.tables) == 0:
+            raise Exception((f"Operational Monitoring view {self.name} has no tables"))
+
+        reference_table = self.tables[0]["table"]
+        dimensions = [
+            d
+            for d in lookml_utils._generate_dimensions(bq_client, reference_table)
+            if d["name"] != "submission"
+        ]
+
+        dimensions.append(
+            {
+                "name": "submission_date",
+                "type": "date",
+                "sql": "${TABLE}.submission_date",
+            }
+        )
+
+        return {
+            "views": [
+                {
+                    "name": self.name,
+                    "sql_table_name": f"`{reference_table}`",
+                    "dimensions": dimensions,
+                    "measures": [
+                        {"name": "errors", "type": "number", "sql": "COUNT(*)"}
+                    ],
+                }
+            ]
+        }

tests/test_operational_monitoring.py

@@ -444,6 +444,7 @@ def test_dashboard_lookml(operational_monitoring_dashboard):
             Os: fission_histogram.os
           enabled: "#3FE1B0"
           disabled: "#0060E0"
+
         filters:
         - name: Percentile
           title: Percentile