Add auto-generated opmon dashboards.

2021-10-13 11:37:51 -04:00 · 2021-10-13 11:37:51 -04:00 · 98ef40718a
--- a/generator/constants.py
+++ b/generator/constants.py
@ -0,0 +1,16 @@
+"""Constants."""
+from typing import Set
+
+# These are fields we don't need for opmon views/explores/dashboards
+OPMON_EXCLUDED_FIELDS: Set[str] = {
+    "submission",
+    "client_id",
+    "build_id",
+    "agg_type",
+    "value",
+    "histogram__VALUES",
+    "histogram__bucket_count",
+    "histogram__histogram_type",
+    "histogram__range",
+    "histogram__sum",
+}
--- a/generator/dashboards/init.py
+++ b/generator/dashboards/init.py
@ -0,0 +1,7 @@
+"""All possible dashboard types."""
+from .dashboard import Dashboard  # noqa: F401
+from .operational_monitoring_dashboard import OperationalMonitoringDashboard
+
+DASHBOARD_TYPES = {
+    OperationalMonitoringDashboard.type: OperationalMonitoringDashboard,
+}
--- a/generator/dashboards/dashboard.py
+++ b/generator/dashboards/dashboard.py
@ -0,0 +1,33 @@
+"""Generic dashboard type."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+
+@dataclass
+class Dashboard(object):
+    """A generic Looker Dashboard."""
+
+    title: str
+    name: str
+    layout: str
+    namespace: str
+    tables: List[Dict[str, str]]
+    type: str = field(init=False)
+
+    def to_dict(self) -> dict:
+        """Dashboard instance represented as a dict."""
+        return {
+            self.name: {
+                "title": self.title,
+                "type": self.type,
+                "layout": self.layout,
+                "namespace": self.namespace,
+                "tables": self.tables,
+            }
+        }
+
+    def to_lookml(self, client):
+        """Generate Lookml for this dashboard."""
+        raise NotImplementedError("Only implemented in subclass.")
--- a/generator/dashboards/operational_monitoring_dashboard.py
+++ b/generator/dashboards/operational_monitoring_dashboard.py
@ -0,0 +1,126 @@
+"""Class to describe Operational Monitoring Dashboard."""
+from __future__ import annotations
+
+from typing import Dict, List
+
+from ..constants import OPMON_EXCLUDED_FIELDS
+from ..views import lookml_utils
+from .dashboard import Dashboard
+
+
+class OperationalMonitoringDashboard(Dashboard):
+    """An Operational Monitoring dashboard."""
+
+    type: str = "operational_monitoring_dashboard"
+
+    OPMON_DASH_EXCLUDED_FIELDS: List[str] = [
+        "branch",
+        "probe",
+        "histogram__VALUES__key",
+        "histogram__VALUES__value",
+    ]
+
+    def __init__(
+        self,
+        title: str,
+        name: str,
+        layout: str,
+        namespace: str,
+        tables: List[Dict[str, str]],
+    ):
+        """Get an instance of a FunnelAnalysisView."""
+        super().__init__(title, name, layout, namespace, tables)
+
+    @classmethod
+    def _slug_to_title(self, slug):
+        return slug.replace("_", " ").title()
+
+    def _compute_dimension_data(self, bq_client, table, kwargs):
+        all_dimensions = lookml_utils._generate_dimensions(bq_client, table)
+        copy_excluded = OPMON_EXCLUDED_FIELDS.copy()
+        copy_excluded.update(self.OPMON_DASH_EXCLUDED_FIELDS)
+
+        relevant_dimensions = [
+            dimension
+            for dimension in all_dimensions
+            if dimension["name"] not in copy_excluded
+        ]
+
+        for dimension in relevant_dimensions:
+            dimension_name = dimension["name"]
+            query_job = bq_client.query(
+                f"""
+                    SELECT DISTINCT {dimension_name}, COUNT(*)
+                    FROM {table}
+                    GROUP BY 1
+                    ORDER BY 2 DESC
+                """
+            )
+            dimension_options = (
+                query_job.result().to_dataframe()[dimension_name].tolist()
+            )
+
+            title = self._slug_to_title(dimension_name)
+            kwargs["dimensions"].append(
+                {
+                    "title": title,
+                    "name": dimension_name,
+                    "default": dimension_options[0],
+                    "options": dimension_options[:10],
+                }
+            )
+
+    @classmethod
+    def from_dict(
+        klass, namespace: str, name: str, defn: dict
+    ) -> OperationalMonitoringDashboard:
+        """Get a OperationalMonitoringDashboard from a dict representation."""
+        title = klass._slug_to_title(name)
+        return klass(title, name, "newspaper", namespace, defn["tables"])
+
+    def to_lookml(self, bq_client):
+        """Get this dashboard as LookML."""
+        kwargs = {
+            "name": self.name,
+            "title": self.title,
+            "layout": self.layout,
+            "elements": [],
+            "dimensions": [],
+        }
+
+        includes = []
+        for table_defn in self.tables:
+            if len(kwargs["dimensions"]) == 0:
+                self._compute_dimension_data(bq_client, table_defn["table"], kwargs)
+
+            query_job = bq_client.query(
+                f"""
+                    SELECT DISTINCT probe
+                    FROM {table_defn["table"]}
+                """
+            )
+            metrics = query_job.result().to_dataframe()["probe"].tolist()
+            explore = table_defn["explore"]
+            includes.append(
+                f"/looker-hub/{self.namespace}/explores/{explore}.explore.lkml"
+            )
+
+            for i, metric in enumerate(metrics):
+                title = self._slug_to_title(metric)
+                kwargs["elements"].append(
+                    {
+                        "title": title,
+                        "metric": metric,
+                        "explore": explore,
+                        "row": int(i / 2),
+                        "col": 0 if i % 2 == 0 else 12,
+                    }
+                )
+
+        model_lookml = lookml_utils.render_template(
+            "model.lkml", "dashboards", **{"includes": includes}
+        )
+        dash_lookml = lookml_utils.render_template(
+            "dashboard.lkml", "dashboards", **kwargs
+        )
+        return dash_lookml, model_lookml
--- a/generator/dashboards/templates/dashboard.lkml
+++ b/generator/dashboards/templates/dashboard.lkml
@ -0,0 +1,69 @@
+- dashboard: {{name}}
+  title: {{title}}
+  layout: {{layout}}
+  preferred_viewer: dashboards-next
+
+  elements:
+  {% for element in elements -%}
+  - title: {{element.title}}
+    name: {{element.title}}
+    explore: {{element.explore}}
+    type: "looker_line"
+    fields: [
+      {{element.explore}}.build_id,
+      {{element.explore}}.branch,
+      {{element.explore}}.high,
+      {{element.explore}}.low,
+      {{element.explore}}.percentile
+    ]
+    pivots: [{{element.explore}}.branch]
+    filters:
+      {{element.explore}}.probe: {{element.metric}}
+    row: {{element.row}}
+    col: {{element.col}}
+    width: 12
+    height: 8
+    listen:
+      Percentile: {{element.explore}}.percentile_conf
+      {% for dimension in dimensions -%}
+      {{dimension.title}}: {{element.explore}}.{{dimension.name}}
+      {% endfor %}
+  {% endfor -%}
+  filters:
+  - name: Percentile
+    title: Percentile
+    type: number_filter
+    default_value: '50'
+    allow_multiple_values: false
+    required: true
+    ui_config:
+      type: dropdown_menu
+      display: inline
+      options:
+      - '10'
+      - '20'
+      - '30'
+      - '40'
+      - '50'
+      - '60'
+      - '70'
+      - '80'
+      - '90'
+      - '95'
+      - '99'
+
+  {% for dimension in dimensions -%}
+  - title: {{dimension.title}}
+    name: {{dimension.title}}
+    type: string_filter
+    default_value: {{dimension.default}}
+    allow_multiple_values: false
+    required: true
+    ui_config:
+      type: dropdown_menu
+      display: inline
+      options:
+      {% for option in dimension.options -%}
+      - {{option}}
+      {% endfor %}
+  {% endfor -%}
--- a/generator/dashboards/templates/model.lkml
+++ b/generator/dashboards/templates/model.lkml
@ -0,0 +1,7 @@
+connection: "telemetry"
+
+include: "dashboards/*.dashboard"
+
+{% for include in includes %}
+include: "{{include}}"
+{% endfor %}
--- a/generator/explores/glean_ping_explore.py
+++ b/generator/explores/glean_ping_explore.py
@ -25,8 +25,7 @@ class GleanPingExplore(PingExplore):
            k.replace("-", "_"): v for k, v in glean_app.get_ping_descriptions().items()
        }
        # collapse whitespace in the description so the lookml looks a little better
-        ping_description = " ".join(ping_descriptions[self.name].split())
-
+        ping_description = " ".join(ping_descriptions.get(self.name, "").split())
        views_lookml = self.get_view_lookml(self.views["base_view"])

        # The first view, by convention, is always the base view with the
--- a/generator/lookml.py
+++ b/generator/lookml.py
@ -8,6 +8,7 @@ import lkml
 import yaml
 from google.cloud import bigquery

+from .dashboards import DASHBOARD_TYPES
 from .explores import EXPLORE_TYPES
 from .namespaces import _get_glean_apps
 from .views import VIEW_TYPES, View, ViewDict
@ -54,6 +55,26 @@ def _generate_explores(
        yield path


+def _generate_dashboards(
+    client, dash_dir: Path, model_dir: Path, namespace: str, dashboards: dict
+):
+    for dashboard_name, dashboard_info in dashboards.items():
+        logging.info(f"Generating lookml for dashboard {dashboard_name} in {namespace}")
+        dashboard = DASHBOARD_TYPES[dashboard_info["type"]].from_dict(
+            namespace, dashboard_name, dashboard_info
+        )
+
+        # A dashboard needs an associated model to render
+        dashboard_lookml, model_lookml = dashboard.to_lookml(client)
+
+        dash_path = dash_dir / f"{dashboard_name}.dashboard.lookml"
+        model_path = model_dir / f"{dashboard_name}_data.model.lkml"
+
+        dash_path.write_text(dashboard_lookml)
+        model_path.write_text(model_lookml)
+        yield dash_path
+
+
 def _get_views_from_dict(views: Dict[str, ViewDict], namespace: str) -> Iterable[View]:
    for view_name, view_info in views.items():
        yield VIEW_TYPES[view_info["type"]].from_dict(  # type: ignore
@ -100,6 +121,16 @@ def _lookml(namespaces, glean_apps, target_dir):
        ):
            logging.info(f"    ...Generating {explore_path}")

+        logging.info("  Generating dashboards")
+        namespace_dir = target / namespace
+        dashboard_dir = namespace_dir / "dashboards"
+        dashboard_dir.mkdir(parents=True, exist_ok=True)
+        dashboards = lookml_objects.get("dashboards", {})
+        for dashboard_path in _generate_dashboards(
+            client, dashboard_dir, namespace_dir, namespace, dashboards
+        ):
+            logging.info(f"    ...Generating {dashboard_path}")
+

@click.command(help=__doc__)
@click.option(
--- a/generator/namespaces.py
+++ b/generator/namespaces.py
@ -78,11 +78,11 @@ def _get_db_views(uri):


 def _append_view_and_explore_for_data_type(
-    om_views_and_explores, project_name, table_prefix, data_type, branches
+    om_content, project_name, table_prefix, data_type, branches
 ):
    project_data_type_id = f"{project_name}_{data_type}"

-    om_views_and_explores["views"][project_data_type_id] = {
+    om_content["views"][project_data_type_id] = {
        "type": f"operational_monitoring_{data_type}_view",
        "tables": [
            {
@ -90,17 +90,17 @@ def _append_view_and_explore_for_data_type(
            }
        ],
    }
-    om_views_and_explores["explores"][project_data_type_id] = {
+    om_content["explores"][project_data_type_id] = {
        "type": "operational_monitoring_explore",
        "views": {"base_view": f"{project_name}_{data_type}"},
        "branches": branches,
    }


-def _get_opmon_views_and_explores():
+def _get_opmon_views_explores_dashboards():
    client = storage.Client(PROD_PROJECT)
    bucket = client.get_bucket(OPMON_BUCKET_NAME)
-    om_views_and_explores = {"views": {}, "explores": {}}
+    om_content = {"views": {}, "explores": {}, "dashboards": {}}

    # Iterating over all defined operational monitoring projects
    for blob in bucket.list_blobs(prefix=PROJECTS_FOLDER):
@ -117,10 +117,20 @@ def _get_opmon_views_and_explores():

        for data_type in DATA_TYPES:
            _append_view_and_explore_for_data_type(
-                om_views_and_explores, project_name, table_prefix, data_type, branches
+                om_content, project_name, table_prefix, data_type, branches
            )

-        return om_views_and_explores
+        om_content["dashboards"][project_name] = {
+            "type": "operational_monitoring_dashboard",
+            "tables": [
+                {
+                    "explore": f"{project_name}_{data_type}",
+                    "table": f"{PROD_PROJECT}.operational_monitoring.{table_prefix}_{data_type}",
+                }
+                for data_type in DATA_TYPES
+            ],
+        }
+        return om_content


 def _get_glean_apps(
@ -268,8 +278,8 @@ def namespaces(custom_namespaces, generated_sql_uri, app_listings_uri, disallowl

    if custom_namespaces is not None:
        custom_namespaces = yaml.safe_load(custom_namespaces.read()) or {}
-        views_and_explores = _get_opmon_views_and_explores()
-        custom_namespaces["operational_monitoring"].update(views_and_explores)
+        views_explores_dashboards = _get_opmon_views_explores_dashboards()
+        custom_namespaces["operational_monitoring"].update(views_explores_dashboards)
        _merge_namespaces(namespaces, custom_namespaces)

    disallowed_namespaces = yaml.safe_load(disallowlist.read()) or {}
--- a/generator/views/lookml_utils.py
+++ b/generator/views/lookml_utils.py
@ -4,6 +4,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple

 import click
 from google.cloud import bigquery
+from jinja2 import Environment, PackageLoader

 BIGQUERY_TYPE_TO_DIMENSION_TYPE = {
    "BIGNUMERIC": "string",
@ -176,3 +177,11 @@ def _is_nested_dimension(dimension: dict):
        and "nested" in dimension
        and dimension["nested"]
    )
+
+
+def render_template(filename, template_folder, **kwargs) -> str:
+    """Render a given template using Jinja."""
+    env = Environment(loader=PackageLoader("generator", f"{template_folder}/templates"))
+    template = env.get_template(filename)
+    rendered = template.render(**kwargs)
+    return rendered
--- a/generator/views/operational_monitoring_histogram_view.py
+++ b/generator/views/operational_monitoring_histogram_view.py
@ -1,23 +1,12 @@
 """Class to describe an Operational Monitoring Histogram View."""

 from textwrap import dedent
-from typing import Any, Dict, Optional, Set
+from typing import Any, Dict, Optional

+from ..constants import OPMON_EXCLUDED_FIELDS
 from . import lookml_utils
 from .operational_monitoring_view import OperationalMonitoringView

-# These are fields we don't need in our view
-EXCLUDED_FIELDS: Set[str] = {
-    "submission",
-    "client_id",
-    "build_id",
-    "histogram__VALUES",
-    "histogram__bucket_count",
-    "histogram__histogram_type",
-    "histogram__range",
-    "histogram__sum",
-}
-

 class OperationalMonitoringHistogramView(OperationalMonitoringView):
    """A view on a scalar operational monitoring table."""
@ -54,7 +43,7 @@ class OperationalMonitoringHistogramView(OperationalMonitoringView):
        additional_dimensions = [
            dimension
            for dimension in all_dimensions
-            if dimension["name"] not in EXCLUDED_FIELDS
+            if dimension["name"] not in OPMON_EXCLUDED_FIELDS
        ]
        self.dimensions.extend(additional_dimensions)

--- a/generator/views/operational_monitoring_scalar_view.py
+++ b/generator/views/operational_monitoring_scalar_view.py
@ -1,20 +1,12 @@
 """Class to describe an Operational Monitoring Scalar View."""

 from textwrap import dedent
-from typing import Any, Dict, Optional, Set
+from typing import Any, Dict, Optional

+from ..constants import OPMON_EXCLUDED_FIELDS
 from . import lookml_utils
 from .operational_monitoring_view import OperationalMonitoringView

-# These are fields we don't need in our view
-EXCLUDED_FIELDS: Set[str] = {
-    "submission",
-    "client_id",
-    "build_id",
-    "agg_type",
-    "value",
-}
-

 class OperationalMonitoringScalarView(OperationalMonitoringView):
    """A view on a scalar operational monitoring table."""
@ -51,7 +43,7 @@ class OperationalMonitoringScalarView(OperationalMonitoringView):
        additional_dimensions = [
            dimension
            for dimension in all_dimensions
-            if dimension["name"] not in EXCLUDED_FIELDS
+            if dimension["name"] not in OPMON_EXCLUDED_FIELDS
        ]
        self.dimensions.extend(additional_dimensions)

--- a/requirements.in
+++ b/requirements.in
@ -4,11 +4,14 @@ google-api-core==1.31.3  # transitive dep that needs dependabot updates
 google-cloud-bigquery==2.28.1
 google-cloud-storage==1.42.3
 googleapis-common-protos==1.53.0  # transitive dep that needs dependabot updates
+Jinja2==3.0.1
 lkml==1.1.1
 looker-sdk==21.14.0
 mozilla-schema-generator==0.5.1
 pip-tools==6.4.0
+pandas==1.3.3
 pre-commit==2.15.0
+pyarrow==5.0.0
 pytest-black==0.3.12
 pytest-flake8==1.0.7
 pytest-isort==2.0.0
--- a/requirements.txt
+++ b/requirements.txt
@ -265,6 +265,10 @@ isort==5.9.3 \
    --hash=sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899 \
    --hash=sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2
    # via pytest-isort
+jinja2==3.0.1 \
+    --hash=sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4 \
+    --hash=sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4
+    # via -r requirements.in
 jsonschema==3.2.0 \
    --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \
    --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a
@ -277,6 +281,62 @@ looker-sdk==21.14.0 \
    --hash=sha256:9dc896e48e60f69b235e670addd24f93847490d7860ce0280731986c575512ba \
    --hash=sha256:e053a0ff8d05832691512f82b455bd65314b7e380082727e8a6ab544209653ed
    # via -r requirements.in
+markupsafe==2.0.1 \
+    --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \
+    --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \
+    --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \
+    --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \
+    --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \
+    --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \
+    --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \
+    --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \
+    --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \
+    --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \
+    --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \
+    --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \
+    --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \
+    --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \
+    --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \
+    --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \
+    --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \
+    --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \
+    --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \
+    --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \
+    --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \
+    --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \
+    --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \
+    --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \
+    --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \
+    --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \
+    --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \
+    --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \
+    --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \
+    --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \
+    --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \
+    --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \
+    --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \
+    --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \
+    --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \
+    --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \
+    --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \
+    --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \
+    --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \
+    --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \
+    --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \
+    --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \
+    --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \
+    --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \
+    --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \
+    --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \
+    --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \
+    --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \
+    --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \
+    --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \
+    --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \
+    --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \
+    --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \
+    --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872
+    # via jinja2
 mccabe==0.6.1 \
    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
@ -320,6 +380,40 @@ nodeenv==1.6.0 \
    --hash=sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b \
    --hash=sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7
    # via pre-commit
+numpy==1.21.2 \
+    --hash=sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf \
+    --hash=sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2 \
+    --hash=sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6 \
+    --hash=sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad \
+    --hash=sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc \
+    --hash=sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254 \
+    --hash=sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0 \
+    --hash=sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218 \
+    --hash=sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13 \
+    --hash=sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef \
+    --hash=sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737 \
+    --hash=sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723 \
+    --hash=sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3 \
+    --hash=sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1 \
+    --hash=sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d \
+    --hash=sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52 \
+    --hash=sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3 \
+    --hash=sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5 \
+    --hash=sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f \
+    --hash=sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496 \
+    --hash=sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f \
+    --hash=sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724 \
+    --hash=sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931 \
+    --hash=sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f \
+    --hash=sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7 \
+    --hash=sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38 \
+    --hash=sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557 \
+    --hash=sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57 \
+    --hash=sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310 \
+    --hash=sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419
+    # via
+    #   pandas
+    #   pyarrow
 packaging==21.0 \
    --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \
    --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14
@ -327,6 +421,29 @@ packaging==21.0 \
    #   google-api-core
    #   google-cloud-bigquery
    #   pytest
+pandas==1.3.3 \
+    --hash=sha256:272c8cb14aa9793eada6b1ebe81994616e647b5892a370c7135efb2924b701df \
+    --hash=sha256:3334a5a9eeaca953b9db1b2b165dcdc5180b5011f3bec3a57a3580c9c22eae68 \
+    --hash=sha256:37d63e78e87eb3791da7be4100a65da0383670c2b59e493d9e73098d7a879226 \
+    --hash=sha256:3f5020613c1d8e304840c34aeb171377dc755521bf5e69804991030c2a48aec3 \
+    --hash=sha256:45649503e167d45360aa7c52f18d1591a6d5c70d2f3a26bc90a3297a30ce9a66 \
+    --hash=sha256:49fd2889d8116d7acef0709e4c82b8560a8b22b0f77471391d12c27596e90267 \
+    --hash=sha256:4def2ef2fb7fcd62f2aa51bacb817ee9029e5c8efe42fe527ba21f6a3ddf1a9f \
+    --hash=sha256:53e2fb11f86f6253bb1df26e3aeab3bf2e000aaa32a953ec394571bec5dc6fd6 \
+    --hash=sha256:629138b7cf81a2e55aa29ce7b04c1cece20485271d1f6c469c6a0c03857db6a4 \
+    --hash=sha256:68408a39a54ebadb9014ee5a4fae27b2fe524317bc80adf56c9ac59e8f8ea431 \
+    --hash=sha256:7326b37de08d42dd3fff5b7ef7691d0fd0bf2428f4ba5a2bdc3b3247e9a52e4c \
+    --hash=sha256:7557b39c8e86eb0543a17a002ac1ea0f38911c3c17095bc9350d0a65b32d801c \
+    --hash=sha256:86b16b1b920c4cb27fdd65a2c20258bcd9c794be491290660722bb0ea765054d \
+    --hash=sha256:a800df4e101b721e94d04c355e611863cc31887f24c0b019572e26518cbbcab6 \
+    --hash=sha256:a9f1b54d7efc9df05320b14a48fb18686f781aa66cc7b47bb62fabfc67a0985c \
+    --hash=sha256:c399200631db9bd9335d013ec7fce4edb98651035c249d532945c78ad453f23a \
+    --hash=sha256:e574c2637c9d27f322e911650b36e858c885702c5996eda8a5a60e35e6648cf2 \
+    --hash=sha256:e9bc59855598cb57f68fdabd4897d3ed2bc3a3b3bef7b868a0153c4cd03f3207 \
+    --hash=sha256:ebbed7312547a924df0cbe133ff1250eeb94cdff3c09a794dc991c5621c8c735 \
+    --hash=sha256:ed2f29b4da6f6ae7c68f4b3708d9d9e59fa89b2f9e87c2b64ce055cbd39f729e \
+    --hash=sha256:f7d84f321674c2f0f31887ee6d5755c54ca1ea5e144d6d54b3bbf566dd9ea0cc
+    # via -r requirements.in
 pathspec==0.9.0 \
    --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \
    --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1
@ -395,6 +512,36 @@ py==1.10.0 \
    --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \
    --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a
    # via pytest
+pyarrow==5.0.0 \
+    --hash=sha256:1832709281efefa4f199c639e9f429678286329860188e53beeda71750775923 \
+    --hash=sha256:1d9485741e497ccc516cb0a0c8f56e22be55aea815be185c3f9a681323b0e614 \
+    --hash=sha256:24e64ea33eed07441cc0e80c949e3a1b48211a1add8953268391d250f4d39922 \
+    --hash=sha256:2d26186ca9748a1fb89ae6c1fa04fb343a4279b53f118734ea8096f15d66c820 \
+    --hash=sha256:357605665fbefb573d40939b13a684c2490b6ed1ab4a5de8dd246db4ab02e5a4 \
+    --hash=sha256:4341ac0f552dc04c450751e049976940c7f4f8f2dae03685cc465ebe0a61e231 \
+    --hash=sha256:456a4488ae810a0569d1adf87dbc522bcc9a0e4a8d1809b934ca28c163d8edce \
+    --hash=sha256:4d8adda1892ef4553c4804af7f67cce484f4d6371564e2d8374b8e2bc85293e2 \
+    --hash=sha256:53e550dec60d1ab86cba3afa1719dc179a8bc9632a0e50d9fe91499cf0a7f2bc \
+    --hash=sha256:5c0d1b68e67bb334a5af0cecdf9b6a702aaa4cc259c5cbb71b25bbed40fcedaf \
+    --hash=sha256:601b0aabd6fb066429e706282934d4d8d38f53bdb8d82da9576be49f07eedf5c \
+    --hash=sha256:64f30aa6b28b666a925d11c239344741850eb97c29d3aa0f7187918cf82494f7 \
+    --hash=sha256:6e1f0e4374061116f40e541408a8a170c170d0a070b788717e18165ebfdd2a54 \
+    --hash=sha256:6e937ce4a40ea0cc7896faff96adecadd4485beb53fbf510b46858e29b2e75ae \
+    --hash=sha256:7560332e5846f0e7830b377c14c93624e24a17f91c98f0b25dafb0ca1ea6ba02 \
+    --hash=sha256:7c4edd2bacee3eea6c8c28bddb02347f9d41a55ec9692c71c6de6e47c62a7f0d \
+    --hash=sha256:99c8b0f7e2ce2541dd4c0c0101d9944bb8e592ae3295fe7a2f290ab99222666d \
+    --hash=sha256:9e04d3621b9f2f23898eed0d044203f66c156d880f02c5534a7f9947ebb1a4af \
+    --hash=sha256:b1453c2411b5062ba6bf6832dbc4df211ad625f678c623a2ee177aee158f199b \
+    --hash=sha256:b3115df938b8d7a7372911a3cb3904196194bcea8bb48911b4b3eafee3ab8d90 \
+    --hash=sha256:b6387d2058d95fa48ccfedea810a768187affb62f4a3ef6595fa30bf9d1a65cf \
+    --hash=sha256:bbe2e439bec2618c74a3bb259700c8a7353dc2ea0c5a62686b6cf04a50ab1e0d \
+    --hash=sha256:c3fc856f107ca2fb3c9391d7ea33bbb33f3a1c2b4a0e2b41f7525c626214cc03 \
+    --hash=sha256:c5493d2414d0d690a738aac8dd6d38518d1f9b870e52e24f89d8d7eb3afd4161 \
+    --hash=sha256:e9ec80f4a77057498cf4c5965389e42e7f6a618b6859e6dd615e57505c9167a6 \
+    --hash=sha256:ed135a99975380c27077f9d0e210aea8618ed9fadcec0e71f8a3190939557afe \
+    --hash=sha256:f4db312e9ba80e730cefcae0a05b63ea5befc7634c28df56682b628ad8e1c25c \
+    --hash=sha256:ff21711f6ff3b0bc90abc8ca8169e676faeb2401ddc1a0bc1c7dc181708a3406
+    # via -r requirements.in
 pyasn1==0.4.8 \
    --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
    --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba
@ -475,10 +622,16 @@ pytest-mypy==0.8.1 \
 pytest-pydocstyle==2.2.0 \
    --hash=sha256:b4e7eee47252fbb7495f4a0126e547a25f5f114f3096c504e738db49a5034333
    # via -r requirements.in
+python-dateutil==2.8.2 \
+    --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
+    --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
+    # via pandas
 pytz==2021.1 \
    --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \
    --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798
-    # via google-api-core
+    # via
+    #   google-api-core
+    #   pandas
 pyyaml==6.0 \
    --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
    --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \
@ -586,6 +739,7 @@ six==1.16.0 \
    #   grpcio
    #   jsonschema
    #   protobuf
+    #   python-dateutil
    #   virtualenv
 smmap==4.0.0 \
    --hash=sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182 \