mozilla
/
lookml-generator
зеркало из https://github.com/mozilla/lookml-generator.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #391 from mozilla/opmon-dashboards 

Opmon dashboards
This commit is contained in:
Anna Scholtz 2022-03-25 11:40:45 -07:00 коммит произвёл GitHub
Родитель ba08fc3164
Коммит db7ccdaf4b
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
21 изменённых файлов: 375 добавлений и 337 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
custom-namespaces.yaml
Просмотреть файл

@ -4,7 +4,11 @@ operational_monitoring:
owners:
- msamuel@mozilla.com
pretty_name: Operational Monitoring
data_functions: ["compute_opmon_dimensions"]
views:
projects:
type: table_view
tables:
- table: moz-fx-data-shared-prod.operational_monitoring_derived.projects_v1
duet:
glean_app: false
owners:

24
generator/constants.py
Просмотреть файл

@ -1,24 +0,0 @@
"""Constants."""
from typing import List, Set
# These are fields we don't need for opmon views/explores/dashboards
OPMON_EXCLUDED_FIELDS: Set[str] = {
"submission",
"client_id",
"build_id",
"agg_type",
"value",
"histogram__VALUES",
"histogram__bucket_count",
"histogram__histogram_type",
"histogram__range",
"histogram__sum",
}
# These are fields we don't need for opmon dashboards
OPMON_DASH_EXCLUDED_FIELDS: List[str] = [
"branch",
"probe",
"histogram__VALUES__key",
"histogram__VALUES__value",
]

37
generator/dashboards/operational_monitoring_dashboard.py
Просмотреть файл

@ -1,9 +1,8 @@
"""Class to describe Operational Monitoring Dashboard."""
from __future__ import annotations
from typing import Dict, List
from typing import Any, Dict, List
from .. import operational_monitoring_utils
from ..views import lookml_utils
from .dashboard import Dashboard
@ -26,10 +25,13 @@ class OperationalMonitoringDashboard(Dashboard):
name: str,
layout: str,
namespace: str,
tables: List[Dict[str, str]],
defn: List[Dict[str, Any]],
):
"""Get an instance of a FunnelAnalysisView."""
super().__init__(title, name, layout, namespace, tables)
"""Get an instance of a Operational Monitoring Dashboard."""
self.dimensions = defn[0].get("dimensions", {})
self.xaxis = defn[0]["xaxis"]
super().__init__(title, name, layout, namespace, defn)
@classmethod
def from_dict(
@ -50,7 +52,7 @@ class OperationalMonitoringDashboard(Dashboard):
]
return dict((label, colour) for (label, colour) in zip(series_labels, colours))
def to_lookml(self, bq_client, data):
def to_lookml(self, bq_client):
"""Get this dashboard as LookML."""
kwargs = {
"name": self.name,
@ -60,27 +62,27 @@ class OperationalMonitoringDashboard(Dashboard):
"dimensions": [],
}
table_data = {}
for view_data in data.get("compute_opmon_dimensions", {}).values():
table_data.update(view_data)
includes = []
graph_index = 0
for table_defn in self.tables:
table_name = table_defn["table"]
if len(kwargs["dimensions"]) == 0:
kwargs["dimensions"] = table_data[table_name]
kwargs["dimensions"] = [
{
"name": name,
"title": lookml_utils.slug_to_title(name),
"default": info["default"],
"options": info["options"],
}
for name, info in self.dimensions.items()
]
xaxis = operational_monitoring_utils.get_xaxis_val(bq_client, table_name)
metrics = lookml_utils.get_distinct_vals(bq_client, table_name, "probe")
explore = table_defn["explore"]
includes.append(
f"/looker-hub/{self.namespace}/explores/{explore}.explore.lkml"
)
series_colors = self._map_series_to_colours(table_defn["branches"], explore)
for metric in metrics:
for metric in table_defn.get("probes", []):
title = lookml_utils.slug_to_title(metric)
kwargs["elements"].append(
{
@ -88,13 +90,12 @@ class OperationalMonitoringDashboard(Dashboard):
"metric": metric,
"explore": explore,
"series_colors": series_colors,
"xaxis": xaxis,
"xaxis": self.xaxis,
"row": int(graph_index / 2) * 10,
"col": 0 if graph_index % 2 == 0 else 12,
}
)
graph_index += 1
dash_lookml = lookml_utils.render_template(
"dashboard.lkml", "dashboards", **kwargs
)

2
generator/explores/client_counts_explore.py
Просмотреть файл

@ -40,7 +40,7 @@ class ClientCountsExplore(Explore):
]
def _to_lookml(
self, client: bigquery.Client, v1_name: Optional[str], data: Dict = {}
self, client: bigquery.Client, v1_name: Optional[str]
) -> List[Dict[str, Any]]:
"""Generate LookML to represent this explore."""
return [

2
generator/explores/events_explore.py
Просмотреть файл

@ -47,7 +47,7 @@ class EventsExplore(Explore):
return EventsExplore(name, defn["views"], views_path)
def _to_lookml(
self, client: bigquery.Client, v1_name: Optional[str], data: Dict = {}
self, client: bigquery.Client, v1_name: Optional[str]
) -> List[Dict[str, Any]]:
name = self.name
if not name.endswith("_counts"):

5
generator/explores/explore.py
Просмотреть файл

@ -26,7 +26,7 @@ class Explore:
return {self.name: {"type": self.type, "views": self.views}}
def to_lookml(
self, client: bigquery.Client, v1_name: Optional[str], data: Dict = {}
self, client: bigquery.Client, v1_name: Optional[str]
) -> List[Dict[str, Any]]:
"""
Generate LookML for this explore.
@ -62,7 +62,7 @@ class Explore:
] = f"${{{base_view_name}.submission_date}} >= '2010-01-01'"
# We only update the first returned explore
new_lookml = self._to_lookml(client, v1_name, data)
new_lookml = self._to_lookml(client, v1_name)
base_lookml.update(new_lookml[0])
new_lookml[0] = base_lookml
@ -72,7 +72,6 @@ class Explore:
self,
client: bigquery.Client,
v1_name: Optional[str],
data: Dict = {},
) -> List[Dict[str, Any]]:
raise NotImplementedError("Only implemented in subclasses")

2
generator/explores/funnel_analysis_explore.py
Просмотреть файл

@ -36,7 +36,7 @@ class FunnelAnalysisExplore(Explore):
return FunnelAnalysisExplore(name, defn["views"], views_path)
def _to_lookml(
self, client: bigquery.Client, v1_name: Optional[str], data: Dict = {}
self, client: bigquery.Client, v1_name: Optional[str]
) -> List[Dict[str, Any]]:
view_lookml = self.get_view_lookml("funnel_analysis")
views = view_lookml["views"]

2
generator/explores/glean_ping_explore.py
Просмотреть файл

@ -17,7 +17,7 @@ class GleanPingExplore(PingExplore):
type: str = "glean_ping_explore"
def _to_lookml(
self, client: bigquery.Client, v1_name: Optional[str], data: Dict = {}
self, client: bigquery.Client, v1_name: Optional[str]
) -> List[Dict[str, Any]]:
"""Generate LookML to represent this explore."""
repo = next((r for r in GleanPing.get_repos() if r["name"] == v1_name))

2
generator/explores/growth_accounting_explore.py
Просмотреть файл

@ -16,7 +16,7 @@ class GrowthAccountingExplore(Explore):
type: str = "growth_accounting_explore"
def _to_lookml(
self, client: bigquery.Client, v1_name: Optional[str], data: Dict = {}
self, client: bigquery.Client, v1_name: Optional[str]
) -> List[Dict[str, Any]]:
"""Generate LookML to represent this explore."""
return [

32
generator/explores/operational_monitoring_explore.py
Просмотреть файл

@ -7,8 +7,7 @@ from typing import Any, Dict, Iterator, List, Optional
from google.cloud import bigquery
from .. import operational_monitoring_utils
from ..views import View, lookml_utils
from ..views import View
from . import Explore
@ -22,12 +21,15 @@ class OperationalMonitoringExplore(Explore):
name: str,
views: Dict[str, str],
views_path: Path = None,
defn: Dict[str, str] = None,
defn: Dict[str, Any] = None,
):
"""Initialize OperationalMonitoringExplore."""
super().__init__(name, views, views_path)
if defn is not None:
self.branches = ", ".join(defn["branches"])
self.xaxis = defn.get("xaxis")
self.dimensions = defn.get("dimensions", {})
self.probes = defn.get("probes", [])
@staticmethod
def from_views(views: List[View]) -> Iterator[Explore]:
@ -53,40 +55,26 @@ class OperationalMonitoringExplore(Explore):
self,
bq_client: bigquery.Client,
v1_name: Optional[str],
data: Dict = {},
) -> List[Dict[str, Any]]:
base_view_name = self.views["base_view"]
namespace_data = data.get("compute_opmon_dimensions", {}).get(
base_view_name, {}
)
table_name = (
"" if len(namespace_data.keys()) == 0 else list(namespace_data.keys())[0]
)
dimension_data = namespace_data[table_name]
xaxis = operational_monitoring_utils.get_xaxis_val(bq_client, table_name)
filters = [
{f"{base_view_name}.branch": self.branches},
{f"{base_view_name}.percentile_conf": "50"},
]
for dimension in dimension_data:
if "default" in dimension:
filters.append(
{f"{base_view_name}.{dimension['name']}": dimension["default"]}
)
for dimension, info in self.dimensions.items():
if "default" in info:
filters.append({f"{base_view_name}.{dimension}": info["default"]})
aggregate_tables = []
probes = lookml_utils.get_distinct_vals(bq_client, table_name, "probe")
for probe in probes:
for probe in self.probes:
filters_copy = deepcopy(filters)
filters_copy.append({f"{base_view_name}.probe": probe})
aggregate_tables.append(
{
"name": f"rollup_{probe}",
"query": {
"dimensions": [xaxis, "branch"],
"dimensions": [self.xaxis, "branch"],
"measures": ["low", "high", "percentile"],
"filters": filters_copy,
},

2
generator/explores/ping_explore.py
Просмотреть файл

@ -16,7 +16,7 @@ class PingExplore(Explore):
type: str = "ping_explore"
def _to_lookml(
self, client: bigquery.Client, v1_name: Optional[str], data: Dict = {}
self, client: bigquery.Client, v1_name: Optional[str]
) -> List[Dict[str, Any]]:
"""Generate LookML to represent this explore."""
return [

35
generator/lookml.py
Просмотреть файл

@ -8,7 +8,6 @@ import lkml
import yaml
from google.cloud import bigquery
from . import operational_monitoring_utils
from .dashboards import DASHBOARD_TYPES
from .explores import EXPLORE_TYPES
from .namespaces import _get_glean_apps
@ -45,7 +44,6 @@ def _generate_explores(
v1_name: Optional[
str
], # v1_name for Glean explores: see: https://mozilla.github.io/probe-scraper/#tag/library
namespace_data: dict,
) -> Iterable[Path]:
for explore_name, defn in explores.items():
logging.info(f"Generating lookml for explore {explore_name} in {namespace}")
@ -58,7 +56,7 @@ def _generate_explores(
f"/looker-hub/{namespace}/views/{view}.view.lkml"
for view in explore.get_dependent_views()
],
"explores": explore.to_lookml(client, v1_name, namespace_data),
"explores": explore.to_lookml(client, v1_name),
}
path = out_dir / (explore_name + ".explore.lkml")
path.write_text(FILE_HEADER + lkml.dump(file_lookml))
@ -70,7 +68,6 @@ def _generate_dashboards(
dash_dir: Path,
namespace: str,
dashboards: dict,
namespace_data: dict,
):
for dashboard_name, dashboard_info in dashboards.items():
logging.info(f"Generating lookml for dashboard {dashboard_name} in {namespace}")
@ -78,7 +75,7 @@ def _generate_dashboards(
namespace, dashboard_name, dashboard_info
)
dashboard_lookml = dashboard.to_lookml(client, namespace_data)
dashboard_lookml = dashboard.to_lookml(client)
dash_path = dash_dir / f"{dashboard_name}.dashboard.lookml"
dash_path.write_text(FILE_HEADER + dashboard_lookml)
yield dash_path
@ -95,24 +92,6 @@ def _glean_apps_to_v1_map(glean_apps):
return {d["name"]: d["v1_name"] for d in glean_apps}
def _append_data(bq_client, data_functions, view, namespace_data):
# For now, assume that data functions are applied
# to each table. Later it could be a mapping so only
# certain functions are applied to certain tables.
for function_name in data_functions:
if function_name not in namespace_data:
namespace_data[function_name] = {}
namespace_data[function_name][view.name] = {}
for table in view.tables:
table_name = table["table"]
data_function = getattr(operational_monitoring_utils, function_name)
namespace_data[function_name][view.name][table_name] = data_function(
bq_client, table_name
)
def _lookml(namespaces, glean_apps, target_dir):
client = bigquery.Client()
@ -134,12 +113,6 @@ def _lookml(namespaces, glean_apps, target_dir):
view_dir.mkdir(parents=True, exist_ok=True)
views = list(_get_views_from_dict(lookml_objects.get("views", {}), namespace))
namespace_data = {}
for view in views:
_append_data(
client, lookml_objects.get("data_functions", []), view, namespace_data
)
logging.info(" Generating views")
v1_name: Optional[str] = v1_mapping.get(namespace)
for view_path in _generate_views(client, view_dir, views, v1_name):
@ -150,7 +123,7 @@ def _lookml(namespaces, glean_apps, target_dir):
explores = lookml_objects.get("explores", {})
logging.info(" Generating explores")
for explore_path in _generate_explores(
client, explore_dir, namespace, explores, view_dir, v1_name, namespace_data
client, explore_dir, namespace, explores, view_dir, v1_name
):
logging.info(f" ...Generating {explore_path}")
@ -159,7 +132,7 @@ def _lookml(namespaces, glean_apps, target_dir):
dashboard_dir.mkdir(parents=True, exist_ok=True)
dashboards = lookml_objects.get("dashboards", {})
for dashboard_path in _generate_dashboards(
client, dashboard_dir, namespace, dashboards, namespace_data
client, dashboard_dir, namespace, dashboards
):
logging.info(f" ...Generating {dashboard_path}")

126
generator/namespaces.py
Просмотреть файл

@ -11,20 +11,21 @@ from io import BytesIO
from itertools import groupby
from operator import itemgetter
from pathlib import Path
from typing import Dict, List, Union
from typing import Any, Dict, List, Union
import click
import yaml
from google.cloud import storage
from google.cloud import bigquery
from generator import operational_monitoring_utils
from .explores import EXPLORE_TYPES
from .views import VIEW_TYPES, View
PROBE_INFO_BASE_URI = "https://probeinfo.telemetry.mozilla.org"
DEFAULT_SPOKE = "looker-spoke-default"
OPMON_BUCKET_NAME = "operational_monitoring"
OPMON_DATASET = "operational_monitoring"
PROD_PROJECT = "moz-fx-data-shared-prod"
PROJECTS_FOLDER = "projects/"
DATA_TYPES = ["histogram", "scalar"]
@ -32,15 +33,6 @@ def _normalize_slug(name):
return re.sub(r"[^a-zA-Z0-9_]", "_", name)
def _download_json_file(project, bucket, filename):
blob = bucket.get_blob(filename)
return json.loads(blob.download_as_string()), blob.updated
def _get_first(tuple_):
return tuple_[0]
def _merge_namespaces(dct, merge_dct):
"""Recursively merge namespaces."""
for k, _ in merge_dct.items():
@ -77,66 +69,82 @@ def _get_db_views(uri):
return views
def _append_view_and_explore_for_data_type(
om_content, project_name, table_prefix, data_type, branches, xaxis
):
project_data_type_id = f"{project_name}_{data_type}"
def _get_opmon(bq_client: bigquery.Client, namespaces: Dict[str, Any]):
om_content: Dict[str, Any] = {"views": {}, "explores": {}, "dashboards": {}}
# get operational monitoring namespace information
om_content["views"][project_data_type_id] = {
"type": f"operational_monitoring_{data_type}_view",
"tables": [
{
"table": f"{PROD_PROJECT}.operational_monitoring.{table_prefix}_{data_type}",
"xaxis": xaxis,
}
],
}
om_content["explores"][project_data_type_id] = {
"type": "operational_monitoring_explore",
"views": {"base_view": f"{project_name}_{data_type}"},
"branches": branches,
}
opmon_namespace = namespaces["operational_monitoring"]
views = opmon_namespace.get("views")
if views is None:
print("No views defined for operational monitoring")
return {}
def _get_opmon_views_explores_dashboards():
client = storage.Client(PROD_PROJECT)
bucket = client.get_bucket(OPMON_BUCKET_NAME)
om_content = {"views": {}, "explores": {}, "dashboards": {}}
projects_view = views.get("projects")
if projects_view is None:
print("No projects view defined for operational monitoring")
return {}
projects_table = projects_view["tables"][0]["table"]
projects = operational_monitoring_utils.get_projects(
bq_client, project_table=projects_table
)
# Iterating over all defined operational monitoring projects
for blob in bucket.list_blobs(prefix=PROJECTS_FOLDER):
# The folder itself is not a project file
if blob.name == PROJECTS_FOLDER:
continue
om_project, project_last_modified = _download_json_file(
PROD_PROJECT, bucket, blob.name
)
table_prefix = _normalize_slug(om_project["slug"])
project_name = "_".join(om_project["name"].lower().split(" "))
branches = om_project.get("branches", ["enabled", "disabled"])
for project in projects:
table_prefix = _normalize_slug(project["slug"])
project_name = "_".join(project["name"].lower().split(" "))
branches = project.get("branches", ["enabled", "disabled"])
for data_type in DATA_TYPES:
_append_view_and_explore_for_data_type(
om_content,
project_name,
table_prefix,
data_type,
branches,
om_project["xaxis"],
# append view and explore for data type
project_data_type_id = f"{project_name}_{data_type}"
table = f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}_{data_type}"
dimensions = operational_monitoring_utils.get_dimension_defaults(
bq_client, table, project["dimensions"]
)
om_content["views"][project_data_type_id] = {
"type": f"operational_monitoring_{data_type}_view",
"tables": [
{
"table": table,
"xaxis": project["xaxis"],
"dimensions": dimensions,
}
],
}
om_content["explores"][project_data_type_id] = {
"type": "operational_monitoring_explore",
"views": {"base_view": f"{project_name}_{data_type}"},
"branches": branches,
"xaxis": project["xaxis"],
"dimensions": dimensions,
"probes": [
p["name"] for p in project["probes"] if p["agg_type"] == data_type
],
}
om_content["dashboards"][project_name] = {
"type": "operational_monitoring_dashboard",
"tables": [
{
"explore": f"{project_name}_{data_type}",
"table": f"{PROD_PROJECT}.operational_monitoring.{table_prefix}_{data_type}",
"table": f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}_{data_type}",
"branches": branches,
"xaxis": project["xaxis"],
"dimensions": dimensions,
"probes": [
p["name"]
for p in project["probes"]
if p["agg_type"] == data_type
],
}
for data_type in DATA_TYPES
],
}
return om_content
@ -285,9 +293,13 @@ def namespaces(custom_namespaces, generated_sql_uri, app_listings_uri, disallowl
if custom_namespaces is not None:
custom_namespaces = yaml.safe_load(custom_namespaces.read()) or {}
views_explores_dashboards = _get_opmon_views_explores_dashboards()
custom_namespaces["operational_monitoring"].update(views_explores_dashboards)
_merge_namespaces(namespaces, custom_namespaces)
# generating operational monitoring namespace, if available
if "operational_monitoring" in custom_namespaces:
client = bigquery.Client()
opmon = _get_opmon(bq_client=client, namespaces=custom_namespaces)
custom_namespaces["operational_monitoring"].update(opmon)
_merge_namespaces(namespaces, custom_namespaces)
disallowed_namespaces = yaml.safe_load(disallowlist.read()) or {}

74
generator/operational_monitoring_utils.py
Просмотреть файл

@ -1,61 +1,43 @@
"""Utils for operational monitoring."""
from typing import Any, Dict, List
from google.api_core import exceptions
from google.cloud import bigquery
from .constants import OPMON_DASH_EXCLUDED_FIELDS, OPMON_EXCLUDED_FIELDS
from .views import lookml_utils
def compute_opmon_dimensions(
bq_client: bigquery.Client, table: str
) -> List[Dict[str, Any]]:
def get_dimension_defaults(
bq_client: bigquery.Client, table: str, dimensions: List[str]
) -> Dict[str, Any]:
"""
Compute dimensions for Operational Monitoring.
Find default values for certain dimensions.
For a given Operational Monitoring dimension, find its default (most common)
value and its top 10 most common to be used as dropdown options.
"""
all_dimensions = lookml_utils._generate_dimensions(bq_client, table)
copy_excluded = OPMON_EXCLUDED_FIELDS.copy()
copy_excluded.update(OPMON_DASH_EXCLUDED_FIELDS)
dimensions = []
dimension_defaults = {}
relevant_dimensions = [
dimension
for dimension in all_dimensions
if dimension["name"] not in copy_excluded
]
for dimension in relevant_dimensions:
dimension_name = dimension["name"]
for dimension in dimensions:
query_job = bq_client.query(
f"""
SELECT DISTINCT {dimension_name}, COUNT(*)
FROM {table}
GROUP BY 1
ORDER BY 2 DESC
"""
SELECT DISTINCT {dimension} AS option, COUNT(*)
FROM {table}
WHERE {dimension} IS NOT NULL
GROUP BY 1
ORDER BY 2 DESC
"""
)
title = lookml_utils.slug_to_title(dimension_name)
dimension_options = query_job.result().to_dataframe()[dimension_name].tolist()
dimension_kwarg = {
"title": title,
"name": dimension_name,
}
dimension_options = [dict(row) for row in query_job.result()]
if len(dimension_options) > 0:
dimension_kwarg.update(
{
"default": dimension_options[0],
"options": dimension_options[:10],
}
)
dimension_defaults[dimension] = {
"default": dimension_options[0]["option"],
"options": [d["option"] for d in dimension_options[:10]],
}
dimensions.append(dimension_kwarg)
return dimensions
return dimension_defaults
def get_xaxis_val(bq_client: bigquery.Client, table: str) -> str:
@ -70,3 +52,21 @@ def get_xaxis_val(bq_client: bigquery.Client, table: str) -> str:
if "build_id" in {dimension["name"] for dimension in all_dimensions}
else "submission_date"
)
def get_projects(
bq_client: bigquery.Client, project_table: str
) -> List[Dict[str, Any]]:
"""Select all operational monitoring projects."""
try:
query_job = bq_client.query(
f"""
SELECT *
FROM `{project_table}`
"""
)
projects = [dict(row) for row in query_job.result()]
except exceptions.Forbidden:
projects = []
return projects

20
generator/views/operational_monitoring_histogram_view.py
Просмотреть файл

@ -3,10 +3,16 @@
from textwrap import dedent
from typing import Any, Dict, Optional
from ..constants import OPMON_EXCLUDED_FIELDS
from . import lookml_utils
from .operational_monitoring_view import OperationalMonitoringView
ALLOWED_DIMENSIONS = {
"branch",
"probe",
"histogram__VALUES__key",
"histogram__VALUES__value",
}
class OperationalMonitoringHistogramView(OperationalMonitoringView):
"""A view on a scalar operational monitoring table."""
@ -40,12 +46,14 @@ class OperationalMonitoringHistogramView(OperationalMonitoringView):
reference_table = self.tables[0]["table"]
all_dimensions = lookml_utils._generate_dimensions(bq_client, reference_table)
additional_dimensions = [
dimension
for dimension in all_dimensions
if dimension["name"] not in OPMON_EXCLUDED_FIELDS
filtered_dimensions = [
d
for d in all_dimensions
if d["name"] in ALLOWED_DIMENSIONS
or d["name"] in self.tables[0].get("dimensions", {}).keys()
]
self.dimensions.extend(additional_dimensions)
self.dimensions.extend(filtered_dimensions)
return {
"views": [

17
generator/views/operational_monitoring_scalar_view.py
Просмотреть файл

@ -3,10 +3,14 @@
from textwrap import dedent
from typing import Any, Dict, Optional
from ..constants import OPMON_EXCLUDED_FIELDS
from . import lookml_utils
from .operational_monitoring_view import OperationalMonitoringView
ALLOWED_DIMENSIONS = {
"branch",
"probe",
}
class OperationalMonitoringScalarView(OperationalMonitoringView):
"""A view on a scalar operational monitoring table."""
@ -40,12 +44,13 @@ class OperationalMonitoringScalarView(OperationalMonitoringView):
reference_table = self.tables[0]["table"]
all_dimensions = lookml_utils._generate_dimensions(bq_client, reference_table)
additional_dimensions = [
dimension
for dimension in all_dimensions
if dimension["name"] not in OPMON_EXCLUDED_FIELDS
filtered_dimensions = [
d
for d in all_dimensions
if d["name"] in ALLOWED_DIMENSIONS
or d["name"] in self.tables[0].get("dimensions", {}).keys()
]
self.dimensions.extend(additional_dimensions)
self.dimensions.extend(filtered_dimensions)
return {
"views": [

2
generator/views/operational_monitoring_view.py
Просмотреть файл

@ -13,7 +13,7 @@ class OperationalMonitoringView(PingView):
type: str = "operational_monitoring_view"
percentile_ci_labels = ["percentile", "low", "high"]
def __init__(self, namespace: str, name: str, tables: List[Dict[str, str]]):
def __init__(self, namespace: str, name: str, tables: List[Dict[str, Any]]):
"""Create instance of a OperationalMonitoringView."""
super().__init__(namespace, name, tables)
xaxis = "build_id"

2
generator/views/ping_view.py
Просмотреть файл

@ -14,7 +14,7 @@ class PingView(View):
type: str = "ping_view"
allow_glean: bool = False
def __init__(self, namespace: str, name: str, tables: List[Dict[str, str]]):
def __init__(self, namespace: str, name: str, tables: List[Dict[str, Any]]):
"""Create instance of a PingView."""
super().__init__(namespace, name, self.__class__.type, tables)

6
generator/views/view.py
Просмотреть файл

@ -20,7 +20,7 @@ class View(object):
name: str
view_type: str
tables: List[Dict[str, str]]
tables: List[Dict[str, Any]]
namespace: str
def __init__(
@ -28,7 +28,7 @@ class View(object):
namespace: str,
name: str,
view_type: str,
tables: List[Dict[str, str]],
tables: List[Dict[str, Any]],
**kwargs,
):
"""Create an instance of a view."""
@ -72,7 +72,7 @@ class View(object):
"""Check for equality with other View."""
def comparable_dict(d):
return {tuple(sorted(t.items())) for t in d}
return {tuple(sorted([(k, str(v)) for k, v in t.items()])) for t in d}
if isinstance(other, View):
return (

152
tests/test_namespaces.py
Просмотреть файл

@ -43,6 +43,10 @@ def custom_namespaces(tmp_path):
owners:
- opmon-owner@allizom.com
pretty_name: Operational Monitoring
views:
projects:
tables:
- table: mozdata.operational_monitoring.projects
custom:
connection: bigquery-oauth
glean_app: false
@ -99,35 +103,53 @@ def namespace_disallowlist(tmp_path):
return dest.absolute()
class MockBlob:
"""Mock Blob."""
class MockClient:
"""Mock bigquery.Client."""
def __init__(self, name):
self.name = name
self.updated = "2021-05-01"
def query(self, query):
class QueryJob:
def result(self):
print(query)
if "os AS option" in query:
return [
{
"option": "Windows",
"count": "10",
},
{
"option": "Linux",
"count": "1",
},
]
elif "cores_count AS option" in query:
return [
{
"option": "4",
"count": "10",
},
{
"option": "1",
"count": "1",
},
]
else:
return [
{
"slug": "op-mon",
"name": "OpMon",
"branches": ["enabled", "disabled"],
"xaxis": "submission_date",
"probes": [
{"name": "GC_MS", "agg_type": "histogram"},
{"name": "GC_MS_CONTENT", "agg_type": "histogram"},
],
"dimensions": {
"cores_count": {"default": "4", "options": ["4", "1"]}
},
}
]
def download_as_string(self):
return '{"slug": "test", "name": "op_mon", "xaxis": "build_id"}'
class MockBucket:
"""Mock Bucket."""
def list_blobs(self, prefix):
return [MockBlob("test")]
def get_blob(self, filename):
return MockBlob("test")
class MockStorageClient:
"""Mock storage.Client."""
def __init__(self, project_name):
pass
def get_bucket(self, bucket_name):
return MockBucket()
return QueryJob()
def add_to_tar(tar, path, content):
@ -193,7 +215,7 @@ def test_namespaces_full(
app_listings_uri,
namespace_disallowlist,
):
with patch("google.cloud.storage.Client", MockStorageClient):
with patch("google.cloud.bigquery.Client", MockClient):
with runner.isolated_filesystem():
result = runner.invoke(
namespaces,
@ -385,32 +407,64 @@ def test_namespaces_full(
},
"operational_monitoring": {
"dashboards": {
"op_mon": {
"opmon": {
"tables": [
{
"explore": "op_mon_histogram",
"table": "moz-fx-data-shared-prod.operational_monitoring.test_histogram",
"explore": "opmon_histogram",
"table": "moz-fx-data-shared-prod.operational_monitoring.op_mon_histogram",
"branches": ["enabled", "disabled"],
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
}
},
"xaxis": "submission_date",
"probes": ["GC_MS", "GC_MS_CONTENT"],
},
{
"explore": "op_mon_scalar",
"table": "moz-fx-data-shared-prod.operational_monitoring.test_scalar",
"explore": "opmon_scalar",
"table": "moz-fx-data-shared-prod.operational_monitoring.op_mon_scalar",
"branches": ["enabled", "disabled"],
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
}
},
"xaxis": "submission_date",
"probes": [],
},
],
"type": "operational_monitoring_dashboard",
}
},
"explores": {
"op_mon_histogram": {
"opmon_histogram": {
"branches": ["enabled", "disabled"],
"type": "operational_monitoring_explore",
"views": {"base_view": "op_mon_histogram"},
"views": {"base_view": "opmon_histogram"},
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
}
},
"xaxis": "submission_date",
"probes": ["GC_MS", "GC_MS_CONTENT"],
},
"op_mon_scalar": {
"opmon_scalar": {
"branches": ["enabled", "disabled"],
"type": "operational_monitoring_explore",
"views": {"base_view": "op_mon_scalar"},
"views": {"base_view": "opmon_scalar"},
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
}
},
"xaxis": "submission_date",
"probes": [],
},
},
"glean_app": False,
@ -418,20 +472,32 @@ def test_namespaces_full(
"pretty_name": "Operational Monitoring",
"spoke": "looker-spoke-default",
"views": {
"op_mon_histogram": {
"opmon_histogram": {
"tables": [
{
"table": "moz-fx-data-shared-prod.operational_monitoring.test_histogram",
"xaxis": "build_id",
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
}
},
"table": "moz-fx-data-shared-prod.operational_monitoring.op_mon_histogram",
"xaxis": "submission_date",
}
],
"type": "operational_monitoring_histogram_view",
},
"op_mon_scalar": {
"opmon_scalar": {
"tables": [
{
"table": "moz-fx-data-shared-prod.operational_monitoring.test_scalar",
"xaxis": "build_id",
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
}
},
"table": "moz-fx-data-shared-prod.operational_monitoring.op_mon_scalar",
"xaxis": "submission_date",
}
],
"type": "operational_monitoring_scalar_view",

162
tests/test_operational_monitoring.py
Просмотреть файл

@ -16,61 +16,6 @@ from generator.views import (
from .utils import print_and_test
TABLE_HISTOGRAM = (
"moz-fx-data-shared-prod."
"operational_monitoring."
"bug_1660366_pref_ongoing_fission_nightly_experiment_nightly_83_100_histogram"
)
TABLE_SCALAR = (
"moz-fx-data-shared-prod."
"operational_monitoring."
"bug_1660366_pref_ongoing_fission_nightly_experiment_nightly_83_100_scalar"
)
DATA = {
"compute_opmon_dimensions": {
"fission_histogram": {
(
"moz-fx-data-shared-prod.operational_monitoring."
"bug_1660366_pref_ongoing_fission_nightly_experiment_nightly_83_100_histogram"
): [
{
"title": "Cores Count",
"name": "cores_count",
"default": "4",
"options": ["1", "2", "3", "4", "6", "8", "10", "12", "16", "32"],
},
{
"title": "Os",
"name": "os",
"default": "Windows",
"options": ["Windows", "Mac", "Linux"],
},
]
},
"fission_scalar": {
(
"moz-fx-data-shared-prod.operational_monitoring."
"bug_1660366_pref_ongoing_fission_nightly_experiment_nightly_83_100_scalar"
): [
{
"title": "Cores Count",
"name": "cores_count",
"default": "4",
"options": ["1", "2", "3", "4", "6", "8", "10", "12", "16", "32"],
},
{
"title": "Os",
"name": "os",
"default": "Windows",
"options": ["Windows", "Mac", "Linux"],
},
]
},
}
}
class MockClient:
"""Mock bigquery.Client."""
@ -100,7 +45,7 @@ class MockClient:
def get_table(self, table_ref):
"""Mock bigquery.Client.get_table."""
if table_ref == TABLE_HISTOGRAM:
if "histogram" in table_ref:
return bigquery.Table(
table_ref,
schema=[
@ -131,7 +76,7 @@ class MockClient:
],
)
if table_ref == TABLE_SCALAR:
if "scalar" in table_ref:
return bigquery.Table(
table_ref,
schema=[
@ -154,7 +99,22 @@ def operational_monitoring_histogram_view():
return OperationalMonitoringHistogramView(
"operational_monitoring",
"fission_histogram",
[{"table": TABLE_HISTOGRAM, "xaxis": "build_id"}],
[
{
"table": "moz-fx-data-shared-prod.operational_monitoring.bug_123_test_histogram",
"xaxis": "build_id",
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
},
"os": {
"default": "Windows",
"options": ["Windows", "Linux"],
},
},
}
],
)
@ -163,7 +123,22 @@ def operational_monitoring_scalar_view():
return OperationalMonitoringScalarView(
"operational_monitoring",
"fission_scalar",
[{"table": TABLE_SCALAR, "xaxis": "submission_date"}],
[
{
"table": "moz-fx-data-shared-prod.operational_monitoring.bug_123_test_scalar",
"xaxis": "submission_date",
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
},
"os": {
"default": "Windows",
"options": ["Windows", "Linux"],
},
},
}
],
)
@ -176,7 +151,21 @@ def operational_monitoring_explore(tmp_path, operational_monitoring_histogram_vi
"fission_histogram",
{"base_view": "fission_histogram"},
tmp_path,
{"branches": ["enabled", "disabled"]},
{
"branches": ["enabled", "disabled"],
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
},
"os": {
"default": "Windows",
"options": ["Windows", "Linux"],
},
},
"probes": ["GC_MS", "GC_MS_CONTENT"],
"xaxis": "build_id",
},
)
@ -189,10 +178,22 @@ def operational_monitoring_dashboard():
"operational_monitoring",
[
{
"table": TABLE_HISTOGRAM,
"table": "moz-fx-data-shared-prod.operational_monitoring.bug_123_test_histogram",
"explore": "fission_histogram",
"branches": ["enabled", "disabled"],
}
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
},
"os": {
"default": "Windows",
"options": ["Windows", "Linux"],
},
},
"xaxis": "build_id",
"probes": ["GC_MS", "GC_MS_CONTENT"],
},
],
)
@ -203,7 +204,22 @@ def test_view_from_dict(operational_monitoring_histogram_view):
"fission_histogram",
{
"type": "operational_monitoring_histogram_view",
"tables": [{"table": TABLE_HISTOGRAM, "xaxis": "build_id"}],
"tables": [
{
"table": "moz-fx-data-shared-prod.operational_monitoring.bug_123_test_histogram",
"xaxis": "build_id",
"dimensions": {
"cores_count": {
"default": "4",
"options": ["4", "1"],
},
"os": {
"default": "Windows",
"options": ["Windows", "Linux"],
},
},
}
],
},
)
@ -217,8 +233,7 @@ def test_histogram_view_lookml(operational_monitoring_histogram_view):
{
"name": "fission_histogram",
"sql_table_name": (
"moz-fx-data-shared-prod.operational_monitoring.bug_1660366_"
"pref_ongoing_fission_nightly_experiment_nightly_83_100_histogram"
"moz-fx-data-shared-prod.operational_monitoring.bug_123_test_histogram"
),
"parameters": operational_monitoring_histogram_view.parameters,
"measures": [
@ -273,7 +288,7 @@ def test_scalar_view_lookml(operational_monitoring_scalar_view):
"sql": dedent(
f"""
SELECT *
FROM `{TABLE_SCALAR}`
FROM `{"moz-fx-data-shared-prod.operational_monitoring.bug_123_test_scalar"}`
WHERE agg_type = "SUM"
"""
)
@ -351,7 +366,7 @@ def test_explore_lookml(operational_monitoring_explore):
}
]
actual = operational_monitoring_explore.to_lookml(mock_bq_client, None, DATA)
actual = operational_monitoring_explore.to_lookml(mock_bq_client, None)
print_and_test(expected=expected, actual=actual)
@ -460,16 +475,8 @@ def test_dashboard_lookml(operational_monitoring_dashboard):
type: dropdown_menu
display: inline
options:
- '1'
- '2'
- '3'
- '4'
- '6'
- '8'
- '10'
- '12'
- '16'
- '32'
- '1'
- title: Os
name: Os
@ -482,11 +489,10 @@ def test_dashboard_lookml(operational_monitoring_dashboard):
display: inline
options:
- 'Windows'
- 'Mac'
- 'Linux'
"""
)
actual = operational_monitoring_dashboard.to_lookml(mock_bq_client, DATA)
actual = operational_monitoring_dashboard.to_lookml(mock_bq_client)
print_and_test(expected=expected, actual=dedent(actual))