"""Generate lookml from namespaces.""" import logging from pathlib import Path from typing import Dict, Iterable, Optional import click import lkml import yaml from google.cloud import bigquery from .dashboards import DASHBOARD_TYPES from .explores import EXPLORE_TYPES from .metrics_utils import LOOKER_METRIC_HUB_REPO, METRIC_HUB_REPO, MetricsConfigLoader from .namespaces import _get_glean_apps from .views import VIEW_TYPES, View, ViewDict from .views.datagroups import generate_datagroups FILE_HEADER = """ # *Do not manually modify this file* # # This file has been generated via https://github.com/mozilla/lookml-generator # You can extend this view in the looker-spoke-default project (https://github.com/mozilla/looker-spoke-default) """ def _generate_views( client, out_dir: Path, views: Iterable[View], v1_name: Optional[str] ) -> Iterable[Path]: for view in views: logging.info( f"Generating lookml for view {view.name} in {view.namespace} of type {view.view_type}" ) path = out_dir / f"{view.name}.view.lkml" lookml = view.to_lookml(client, v1_name) if lookml == {}: continue # lkml.dump may return None, in which case write an empty file path.write_text(FILE_HEADER + (lkml.dump(lookml) or "")) yield path def _generate_explores( client, out_dir: Path, namespace: str, explores: dict, views_dir: Path, v1_name: Optional[ str ], # v1_name for Glean explores: see: https://mozilla.github.io/probe-scraper/#tag/library ) -> Iterable[Path]: for explore_name, defn in explores.items(): logging.info(f"Generating lookml for explore {explore_name} in {namespace}") explore = EXPLORE_TYPES[defn["type"]].from_dict(explore_name, defn, views_dir) file_lookml = { # Looker validates all included files, # so if we're not explicit about files here, validation takes # forever as looker re-validates all views for every explore (if we used *). "includes": [ f"/looker-hub/{namespace}/views/{view}.view.lkml" for view in explore.get_dependent_views() ], "explores": explore.to_lookml(client, v1_name), } path = out_dir / (explore_name + ".explore.lkml") # lkml.dump may return None, in which case write an empty file path.write_text(FILE_HEADER + (lkml.dump(file_lookml) or "")) yield path def _generate_dashboards( client, dash_dir: Path, namespace: str, dashboards: dict, ): for dashboard_name, dashboard_info in dashboards.items(): logging.info(f"Generating lookml for dashboard {dashboard_name} in {namespace}") dashboard = DASHBOARD_TYPES[dashboard_info["type"]].from_dict( namespace, dashboard_name, dashboard_info ) dashboard_lookml = dashboard.to_lookml(client) dash_path = dash_dir / f"{dashboard_name}.dashboard.lookml" dash_path.write_text(FILE_HEADER + dashboard_lookml) yield dash_path def _get_views_from_dict(views: Dict[str, ViewDict], namespace: str) -> Iterable[View]: for view_name, view_info in views.items(): yield VIEW_TYPES[view_info["type"]].from_dict( # type: ignore namespace, view_name, view_info ) def _glean_apps_to_v1_map(glean_apps): return {d["name"]: d["v1_name"] for d in glean_apps} def _lookml(namespaces, glean_apps, target_dir): client = bigquery.Client() namespaces_content = namespaces.read() _namespaces = yaml.safe_load(namespaces_content) target = Path(target_dir) target.mkdir(parents=True, exist_ok=True) # Write namespaces file to target directory, for use # by the Glean Dictionary and other tools with open(target / "namespaces.yaml", "w") as target_namespaces_file: target_namespaces_file.write(namespaces_content) v1_mapping = _glean_apps_to_v1_map(glean_apps) for namespace, lookml_objects in _namespaces.items(): logging.info(f"\nGenerating namespace {namespace}") view_dir = target / namespace / "views" view_dir.mkdir(parents=True, exist_ok=True) views = list(_get_views_from_dict(lookml_objects.get("views", {}), namespace)) logging.info(" Generating views") v1_name: Optional[str] = v1_mapping.get(namespace) for view_path in _generate_views(client, view_dir, views, v1_name): logging.info(f" ...Generating {view_path}") logging.info(" Generating datagroups") generate_datagroups(views, target, namespace, client) explore_dir = target / namespace / "explores" explore_dir.mkdir(parents=True, exist_ok=True) explores = lookml_objects.get("explores", {}) logging.info(" Generating explores") for explore_path in _generate_explores( client, explore_dir, namespace, explores, view_dir, v1_name ): logging.info(f" ...Generating {explore_path}") logging.info(" Generating dashboards") dashboard_dir = target / namespace / "dashboards" dashboard_dir.mkdir(parents=True, exist_ok=True) dashboards = lookml_objects.get("dashboards", {}) for dashboard_path in _generate_dashboards( client, dashboard_dir, namespace, dashboards ): logging.info(f" ...Generating {dashboard_path}") @click.command(help=__doc__) @click.option( "--namespaces", default="namespaces.yaml", type=click.File(), help="Path to a yaml namespaces file", ) @click.option( "--app-listings-uri", default="https://probeinfo.telemetry.mozilla.org/v2/glean/app-listings", help="URI for probeinfo service v2 glean app listings", ) @click.option( "--target-dir", default="looker-hub/", type=click.Path(), help="Path to a directory where lookml will be written", ) @click.option( "--metric-hub-repos", "--metric-hub-repos", multiple=True, default=[METRIC_HUB_REPO, LOOKER_METRIC_HUB_REPO], help="Repos to load metric configs from.", ) def lookml(namespaces, app_listings_uri, target_dir, metric_hub_repos): """Generate lookml from namespaces.""" if metric_hub_repos: MetricsConfigLoader.update_repos(metric_hub_repos) glean_apps = _get_glean_apps(app_listings_uri) return _lookml(namespaces, glean_apps, target_dir)