From 98ef40718af4ed9acec2fdd8af12388e7672dffc Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Wed, 13 Oct 2021 11:37:51 -0400 Subject: [PATCH] Add auto-generated opmon dashboards. --- generator/constants.py | 16 ++ generator/dashboards/__init__.py | 7 + generator/dashboards/dashboard.py | 33 ++++ .../operational_monitoring_dashboard.py | 126 ++++++++++++++ generator/dashboards/templates/dashboard.lkml | 69 ++++++++ generator/dashboards/templates/model.lkml | 7 + generator/explores/glean_ping_explore.py | 3 +- generator/lookml.py | 31 ++++ generator/namespaces.py | 28 +++- generator/views/lookml_utils.py | 9 + .../operational_monitoring_histogram_view.py | 17 +- .../operational_monitoring_scalar_view.py | 14 +- requirements.in | 3 + requirements.txt | 156 +++++++++++++++++- 14 files changed, 482 insertions(+), 37 deletions(-) create mode 100644 generator/constants.py create mode 100644 generator/dashboards/__init__.py create mode 100644 generator/dashboards/dashboard.py create mode 100644 generator/dashboards/operational_monitoring_dashboard.py create mode 100644 generator/dashboards/templates/dashboard.lkml create mode 100644 generator/dashboards/templates/model.lkml diff --git a/generator/constants.py b/generator/constants.py new file mode 100644 index 0000000..2f9b8a7 --- /dev/null +++ b/generator/constants.py @@ -0,0 +1,16 @@ +"""Constants.""" +from typing import Set + +# These are fields we don't need for opmon views/explores/dashboards +OPMON_EXCLUDED_FIELDS: Set[str] = { + "submission", + "client_id", + "build_id", + "agg_type", + "value", + "histogram__VALUES", + "histogram__bucket_count", + "histogram__histogram_type", + "histogram__range", + "histogram__sum", +} diff --git a/generator/dashboards/__init__.py b/generator/dashboards/__init__.py new file mode 100644 index 0000000..d61a58c --- /dev/null +++ b/generator/dashboards/__init__.py @@ -0,0 +1,7 @@ +"""All possible dashboard types.""" +from .dashboard import Dashboard # noqa: F401 +from .operational_monitoring_dashboard import OperationalMonitoringDashboard + +DASHBOARD_TYPES = { + OperationalMonitoringDashboard.type: OperationalMonitoringDashboard, +} diff --git a/generator/dashboards/dashboard.py b/generator/dashboards/dashboard.py new file mode 100644 index 0000000..edde648 --- /dev/null +++ b/generator/dashboards/dashboard.py @@ -0,0 +1,33 @@ +"""Generic dashboard type.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List + + +@dataclass +class Dashboard(object): + """A generic Looker Dashboard.""" + + title: str + name: str + layout: str + namespace: str + tables: List[Dict[str, str]] + type: str = field(init=False) + + def to_dict(self) -> dict: + """Dashboard instance represented as a dict.""" + return { + self.name: { + "title": self.title, + "type": self.type, + "layout": self.layout, + "namespace": self.namespace, + "tables": self.tables, + } + } + + def to_lookml(self, client): + """Generate Lookml for this dashboard.""" + raise NotImplementedError("Only implemented in subclass.") diff --git a/generator/dashboards/operational_monitoring_dashboard.py b/generator/dashboards/operational_monitoring_dashboard.py new file mode 100644 index 0000000..c55357c --- /dev/null +++ b/generator/dashboards/operational_monitoring_dashboard.py @@ -0,0 +1,126 @@ +"""Class to describe Operational Monitoring Dashboard.""" +from __future__ import annotations + +from typing import Dict, List + +from ..constants import OPMON_EXCLUDED_FIELDS +from ..views import lookml_utils +from .dashboard import Dashboard + + +class OperationalMonitoringDashboard(Dashboard): + """An Operational Monitoring dashboard.""" + + type: str = "operational_monitoring_dashboard" + + OPMON_DASH_EXCLUDED_FIELDS: List[str] = [ + "branch", + "probe", + "histogram__VALUES__key", + "histogram__VALUES__value", + ] + + def __init__( + self, + title: str, + name: str, + layout: str, + namespace: str, + tables: List[Dict[str, str]], + ): + """Get an instance of a FunnelAnalysisView.""" + super().__init__(title, name, layout, namespace, tables) + + @classmethod + def _slug_to_title(self, slug): + return slug.replace("_", " ").title() + + def _compute_dimension_data(self, bq_client, table, kwargs): + all_dimensions = lookml_utils._generate_dimensions(bq_client, table) + copy_excluded = OPMON_EXCLUDED_FIELDS.copy() + copy_excluded.update(self.OPMON_DASH_EXCLUDED_FIELDS) + + relevant_dimensions = [ + dimension + for dimension in all_dimensions + if dimension["name"] not in copy_excluded + ] + + for dimension in relevant_dimensions: + dimension_name = dimension["name"] + query_job = bq_client.query( + f""" + SELECT DISTINCT {dimension_name}, COUNT(*) + FROM {table} + GROUP BY 1 + ORDER BY 2 DESC + """ + ) + dimension_options = ( + query_job.result().to_dataframe()[dimension_name].tolist() + ) + + title = self._slug_to_title(dimension_name) + kwargs["dimensions"].append( + { + "title": title, + "name": dimension_name, + "default": dimension_options[0], + "options": dimension_options[:10], + } + ) + + @classmethod + def from_dict( + klass, namespace: str, name: str, defn: dict + ) -> OperationalMonitoringDashboard: + """Get a OperationalMonitoringDashboard from a dict representation.""" + title = klass._slug_to_title(name) + return klass(title, name, "newspaper", namespace, defn["tables"]) + + def to_lookml(self, bq_client): + """Get this dashboard as LookML.""" + kwargs = { + "name": self.name, + "title": self.title, + "layout": self.layout, + "elements": [], + "dimensions": [], + } + + includes = [] + for table_defn in self.tables: + if len(kwargs["dimensions"]) == 0: + self._compute_dimension_data(bq_client, table_defn["table"], kwargs) + + query_job = bq_client.query( + f""" + SELECT DISTINCT probe + FROM {table_defn["table"]} + """ + ) + metrics = query_job.result().to_dataframe()["probe"].tolist() + explore = table_defn["explore"] + includes.append( + f"/looker-hub/{self.namespace}/explores/{explore}.explore.lkml" + ) + + for i, metric in enumerate(metrics): + title = self._slug_to_title(metric) + kwargs["elements"].append( + { + "title": title, + "metric": metric, + "explore": explore, + "row": int(i / 2), + "col": 0 if i % 2 == 0 else 12, + } + ) + + model_lookml = lookml_utils.render_template( + "model.lkml", "dashboards", **{"includes": includes} + ) + dash_lookml = lookml_utils.render_template( + "dashboard.lkml", "dashboards", **kwargs + ) + return dash_lookml, model_lookml diff --git a/generator/dashboards/templates/dashboard.lkml b/generator/dashboards/templates/dashboard.lkml new file mode 100644 index 0000000..7e6d9eb --- /dev/null +++ b/generator/dashboards/templates/dashboard.lkml @@ -0,0 +1,69 @@ +- dashboard: {{name}} + title: {{title}} + layout: {{layout}} + preferred_viewer: dashboards-next + + elements: + {% for element in elements -%} + - title: {{element.title}} + name: {{element.title}} + explore: {{element.explore}} + type: "looker_line" + fields: [ + {{element.explore}}.build_id, + {{element.explore}}.branch, + {{element.explore}}.high, + {{element.explore}}.low, + {{element.explore}}.percentile + ] + pivots: [{{element.explore}}.branch] + filters: + {{element.explore}}.probe: {{element.metric}} + row: {{element.row}} + col: {{element.col}} + width: 12 + height: 8 + listen: + Percentile: {{element.explore}}.percentile_conf + {% for dimension in dimensions -%} + {{dimension.title}}: {{element.explore}}.{{dimension.name}} + {% endfor %} + {% endfor -%} + filters: + - name: Percentile + title: Percentile + type: number_filter + default_value: '50' + allow_multiple_values: false + required: true + ui_config: + type: dropdown_menu + display: inline + options: + - '10' + - '20' + - '30' + - '40' + - '50' + - '60' + - '70' + - '80' + - '90' + - '95' + - '99' + + {% for dimension in dimensions -%} + - title: {{dimension.title}} + name: {{dimension.title}} + type: string_filter + default_value: {{dimension.default}} + allow_multiple_values: false + required: true + ui_config: + type: dropdown_menu + display: inline + options: + {% for option in dimension.options -%} + - {{option}} + {% endfor %} + {% endfor -%} diff --git a/generator/dashboards/templates/model.lkml b/generator/dashboards/templates/model.lkml new file mode 100644 index 0000000..fde8862 --- /dev/null +++ b/generator/dashboards/templates/model.lkml @@ -0,0 +1,7 @@ +connection: "telemetry" + +include: "dashboards/*.dashboard" + +{% for include in includes %} +include: "{{include}}" +{% endfor %} diff --git a/generator/explores/glean_ping_explore.py b/generator/explores/glean_ping_explore.py index dc97ea7..d779727 100644 --- a/generator/explores/glean_ping_explore.py +++ b/generator/explores/glean_ping_explore.py @@ -25,8 +25,7 @@ class GleanPingExplore(PingExplore): k.replace("-", "_"): v for k, v in glean_app.get_ping_descriptions().items() } # collapse whitespace in the description so the lookml looks a little better - ping_description = " ".join(ping_descriptions[self.name].split()) - + ping_description = " ".join(ping_descriptions.get(self.name, "").split()) views_lookml = self.get_view_lookml(self.views["base_view"]) # The first view, by convention, is always the base view with the diff --git a/generator/lookml.py b/generator/lookml.py index 0c199b0..01022a8 100644 --- a/generator/lookml.py +++ b/generator/lookml.py @@ -8,6 +8,7 @@ import lkml import yaml from google.cloud import bigquery +from .dashboards import DASHBOARD_TYPES from .explores import EXPLORE_TYPES from .namespaces import _get_glean_apps from .views import VIEW_TYPES, View, ViewDict @@ -54,6 +55,26 @@ def _generate_explores( yield path +def _generate_dashboards( + client, dash_dir: Path, model_dir: Path, namespace: str, dashboards: dict +): + for dashboard_name, dashboard_info in dashboards.items(): + logging.info(f"Generating lookml for dashboard {dashboard_name} in {namespace}") + dashboard = DASHBOARD_TYPES[dashboard_info["type"]].from_dict( + namespace, dashboard_name, dashboard_info + ) + + # A dashboard needs an associated model to render + dashboard_lookml, model_lookml = dashboard.to_lookml(client) + + dash_path = dash_dir / f"{dashboard_name}.dashboard.lookml" + model_path = model_dir / f"{dashboard_name}_data.model.lkml" + + dash_path.write_text(dashboard_lookml) + model_path.write_text(model_lookml) + yield dash_path + + def _get_views_from_dict(views: Dict[str, ViewDict], namespace: str) -> Iterable[View]: for view_name, view_info in views.items(): yield VIEW_TYPES[view_info["type"]].from_dict( # type: ignore @@ -100,6 +121,16 @@ def _lookml(namespaces, glean_apps, target_dir): ): logging.info(f" ...Generating {explore_path}") + logging.info(" Generating dashboards") + namespace_dir = target / namespace + dashboard_dir = namespace_dir / "dashboards" + dashboard_dir.mkdir(parents=True, exist_ok=True) + dashboards = lookml_objects.get("dashboards", {}) + for dashboard_path in _generate_dashboards( + client, dashboard_dir, namespace_dir, namespace, dashboards + ): + logging.info(f" ...Generating {dashboard_path}") + @click.command(help=__doc__) @click.option( diff --git a/generator/namespaces.py b/generator/namespaces.py index db724b6..4eecaf5 100644 --- a/generator/namespaces.py +++ b/generator/namespaces.py @@ -78,11 +78,11 @@ def _get_db_views(uri): def _append_view_and_explore_for_data_type( - om_views_and_explores, project_name, table_prefix, data_type, branches + om_content, project_name, table_prefix, data_type, branches ): project_data_type_id = f"{project_name}_{data_type}" - om_views_and_explores["views"][project_data_type_id] = { + om_content["views"][project_data_type_id] = { "type": f"operational_monitoring_{data_type}_view", "tables": [ { @@ -90,17 +90,17 @@ def _append_view_and_explore_for_data_type( } ], } - om_views_and_explores["explores"][project_data_type_id] = { + om_content["explores"][project_data_type_id] = { "type": "operational_monitoring_explore", "views": {"base_view": f"{project_name}_{data_type}"}, "branches": branches, } -def _get_opmon_views_and_explores(): +def _get_opmon_views_explores_dashboards(): client = storage.Client(PROD_PROJECT) bucket = client.get_bucket(OPMON_BUCKET_NAME) - om_views_and_explores = {"views": {}, "explores": {}} + om_content = {"views": {}, "explores": {}, "dashboards": {}} # Iterating over all defined operational monitoring projects for blob in bucket.list_blobs(prefix=PROJECTS_FOLDER): @@ -117,10 +117,20 @@ def _get_opmon_views_and_explores(): for data_type in DATA_TYPES: _append_view_and_explore_for_data_type( - om_views_and_explores, project_name, table_prefix, data_type, branches + om_content, project_name, table_prefix, data_type, branches ) - return om_views_and_explores + om_content["dashboards"][project_name] = { + "type": "operational_monitoring_dashboard", + "tables": [ + { + "explore": f"{project_name}_{data_type}", + "table": f"{PROD_PROJECT}.operational_monitoring.{table_prefix}_{data_type}", + } + for data_type in DATA_TYPES + ], + } + return om_content def _get_glean_apps( @@ -268,8 +278,8 @@ def namespaces(custom_namespaces, generated_sql_uri, app_listings_uri, disallowl if custom_namespaces is not None: custom_namespaces = yaml.safe_load(custom_namespaces.read()) or {} - views_and_explores = _get_opmon_views_and_explores() - custom_namespaces["operational_monitoring"].update(views_and_explores) + views_explores_dashboards = _get_opmon_views_explores_dashboards() + custom_namespaces["operational_monitoring"].update(views_explores_dashboards) _merge_namespaces(namespaces, custom_namespaces) disallowed_namespaces = yaml.safe_load(disallowlist.read()) or {} diff --git a/generator/views/lookml_utils.py b/generator/views/lookml_utils.py index 38fbcc0..a2c5b2e 100644 --- a/generator/views/lookml_utils.py +++ b/generator/views/lookml_utils.py @@ -4,6 +4,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple import click from google.cloud import bigquery +from jinja2 import Environment, PackageLoader BIGQUERY_TYPE_TO_DIMENSION_TYPE = { "BIGNUMERIC": "string", @@ -176,3 +177,11 @@ def _is_nested_dimension(dimension: dict): and "nested" in dimension and dimension["nested"] ) + + +def render_template(filename, template_folder, **kwargs) -> str: + """Render a given template using Jinja.""" + env = Environment(loader=PackageLoader("generator", f"{template_folder}/templates")) + template = env.get_template(filename) + rendered = template.render(**kwargs) + return rendered diff --git a/generator/views/operational_monitoring_histogram_view.py b/generator/views/operational_monitoring_histogram_view.py index 97ffdee..f6eade1 100644 --- a/generator/views/operational_monitoring_histogram_view.py +++ b/generator/views/operational_monitoring_histogram_view.py @@ -1,23 +1,12 @@ """Class to describe an Operational Monitoring Histogram View.""" from textwrap import dedent -from typing import Any, Dict, Optional, Set +from typing import Any, Dict, Optional +from ..constants import OPMON_EXCLUDED_FIELDS from . import lookml_utils from .operational_monitoring_view import OperationalMonitoringView -# These are fields we don't need in our view -EXCLUDED_FIELDS: Set[str] = { - "submission", - "client_id", - "build_id", - "histogram__VALUES", - "histogram__bucket_count", - "histogram__histogram_type", - "histogram__range", - "histogram__sum", -} - class OperationalMonitoringHistogramView(OperationalMonitoringView): """A view on a scalar operational monitoring table.""" @@ -54,7 +43,7 @@ class OperationalMonitoringHistogramView(OperationalMonitoringView): additional_dimensions = [ dimension for dimension in all_dimensions - if dimension["name"] not in EXCLUDED_FIELDS + if dimension["name"] not in OPMON_EXCLUDED_FIELDS ] self.dimensions.extend(additional_dimensions) diff --git a/generator/views/operational_monitoring_scalar_view.py b/generator/views/operational_monitoring_scalar_view.py index 2d8d561..04695ec 100644 --- a/generator/views/operational_monitoring_scalar_view.py +++ b/generator/views/operational_monitoring_scalar_view.py @@ -1,20 +1,12 @@ """Class to describe an Operational Monitoring Scalar View.""" from textwrap import dedent -from typing import Any, Dict, Optional, Set +from typing import Any, Dict, Optional +from ..constants import OPMON_EXCLUDED_FIELDS from . import lookml_utils from .operational_monitoring_view import OperationalMonitoringView -# These are fields we don't need in our view -EXCLUDED_FIELDS: Set[str] = { - "submission", - "client_id", - "build_id", - "agg_type", - "value", -} - class OperationalMonitoringScalarView(OperationalMonitoringView): """A view on a scalar operational monitoring table.""" @@ -51,7 +43,7 @@ class OperationalMonitoringScalarView(OperationalMonitoringView): additional_dimensions = [ dimension for dimension in all_dimensions - if dimension["name"] not in EXCLUDED_FIELDS + if dimension["name"] not in OPMON_EXCLUDED_FIELDS ] self.dimensions.extend(additional_dimensions) diff --git a/requirements.in b/requirements.in index a77b342..955ffbb 100644 --- a/requirements.in +++ b/requirements.in @@ -4,11 +4,14 @@ google-api-core==1.31.3 # transitive dep that needs dependabot updates google-cloud-bigquery==2.28.1 google-cloud-storage==1.42.3 googleapis-common-protos==1.53.0 # transitive dep that needs dependabot updates +Jinja2==3.0.1 lkml==1.1.1 looker-sdk==21.14.0 mozilla-schema-generator==0.5.1 pip-tools==6.4.0 +pandas==1.3.3 pre-commit==2.15.0 +pyarrow==5.0.0 pytest-black==0.3.12 pytest-flake8==1.0.7 pytest-isort==2.0.0 diff --git a/requirements.txt b/requirements.txt index 9efa16e..4e925c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -265,6 +265,10 @@ isort==5.9.3 \ --hash=sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899 \ --hash=sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2 # via pytest-isort +jinja2==3.0.1 \ + --hash=sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4 \ + --hash=sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4 + # via -r requirements.in jsonschema==3.2.0 \ --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \ --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a @@ -277,6 +281,62 @@ looker-sdk==21.14.0 \ --hash=sha256:9dc896e48e60f69b235e670addd24f93847490d7860ce0280731986c575512ba \ --hash=sha256:e053a0ff8d05832691512f82b455bd65314b7e380082727e8a6ab544209653ed # via -r requirements.in +markupsafe==2.0.1 \ + --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ + --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ + --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \ + --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \ + --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \ + --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \ + --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \ + --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \ + --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \ + --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \ + --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \ + --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \ + --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \ + --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \ + --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \ + --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \ + --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \ + --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \ + --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \ + --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \ + --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \ + --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \ + --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \ + --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \ + --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \ + --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \ + --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \ + --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \ + --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \ + --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \ + --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \ + --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \ + --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \ + --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \ + --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \ + --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \ + --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \ + --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \ + --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \ + --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \ + --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \ + --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \ + --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \ + --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \ + --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \ + --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \ + --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \ + --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \ + --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \ + --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \ + --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \ + --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ + --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ + --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 + # via jinja2 mccabe==0.6.1 \ --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f @@ -320,6 +380,40 @@ nodeenv==1.6.0 \ --hash=sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b \ --hash=sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7 # via pre-commit +numpy==1.21.2 \ + --hash=sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf \ + --hash=sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2 \ + --hash=sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6 \ + --hash=sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad \ + --hash=sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc \ + --hash=sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254 \ + --hash=sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0 \ + --hash=sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218 \ + --hash=sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13 \ + --hash=sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef \ + --hash=sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737 \ + --hash=sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723 \ + --hash=sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3 \ + --hash=sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1 \ + --hash=sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d \ + --hash=sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52 \ + --hash=sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3 \ + --hash=sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5 \ + --hash=sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f \ + --hash=sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496 \ + --hash=sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f \ + --hash=sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724 \ + --hash=sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931 \ + --hash=sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f \ + --hash=sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7 \ + --hash=sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38 \ + --hash=sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557 \ + --hash=sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57 \ + --hash=sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310 \ + --hash=sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419 + # via + # pandas + # pyarrow packaging==21.0 \ --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 @@ -327,6 +421,29 @@ packaging==21.0 \ # google-api-core # google-cloud-bigquery # pytest +pandas==1.3.3 \ + --hash=sha256:272c8cb14aa9793eada6b1ebe81994616e647b5892a370c7135efb2924b701df \ + --hash=sha256:3334a5a9eeaca953b9db1b2b165dcdc5180b5011f3bec3a57a3580c9c22eae68 \ + --hash=sha256:37d63e78e87eb3791da7be4100a65da0383670c2b59e493d9e73098d7a879226 \ + --hash=sha256:3f5020613c1d8e304840c34aeb171377dc755521bf5e69804991030c2a48aec3 \ + --hash=sha256:45649503e167d45360aa7c52f18d1591a6d5c70d2f3a26bc90a3297a30ce9a66 \ + --hash=sha256:49fd2889d8116d7acef0709e4c82b8560a8b22b0f77471391d12c27596e90267 \ + --hash=sha256:4def2ef2fb7fcd62f2aa51bacb817ee9029e5c8efe42fe527ba21f6a3ddf1a9f \ + --hash=sha256:53e2fb11f86f6253bb1df26e3aeab3bf2e000aaa32a953ec394571bec5dc6fd6 \ + --hash=sha256:629138b7cf81a2e55aa29ce7b04c1cece20485271d1f6c469c6a0c03857db6a4 \ + --hash=sha256:68408a39a54ebadb9014ee5a4fae27b2fe524317bc80adf56c9ac59e8f8ea431 \ + --hash=sha256:7326b37de08d42dd3fff5b7ef7691d0fd0bf2428f4ba5a2bdc3b3247e9a52e4c \ + --hash=sha256:7557b39c8e86eb0543a17a002ac1ea0f38911c3c17095bc9350d0a65b32d801c \ + --hash=sha256:86b16b1b920c4cb27fdd65a2c20258bcd9c794be491290660722bb0ea765054d \ + --hash=sha256:a800df4e101b721e94d04c355e611863cc31887f24c0b019572e26518cbbcab6 \ + --hash=sha256:a9f1b54d7efc9df05320b14a48fb18686f781aa66cc7b47bb62fabfc67a0985c \ + --hash=sha256:c399200631db9bd9335d013ec7fce4edb98651035c249d532945c78ad453f23a \ + --hash=sha256:e574c2637c9d27f322e911650b36e858c885702c5996eda8a5a60e35e6648cf2 \ + --hash=sha256:e9bc59855598cb57f68fdabd4897d3ed2bc3a3b3bef7b868a0153c4cd03f3207 \ + --hash=sha256:ebbed7312547a924df0cbe133ff1250eeb94cdff3c09a794dc991c5621c8c735 \ + --hash=sha256:ed2f29b4da6f6ae7c68f4b3708d9d9e59fa89b2f9e87c2b64ce055cbd39f729e \ + --hash=sha256:f7d84f321674c2f0f31887ee6d5755c54ca1ea5e144d6d54b3bbf566dd9ea0cc + # via -r requirements.in pathspec==0.9.0 \ --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 @@ -395,6 +512,36 @@ py==1.10.0 \ --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \ --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a # via pytest +pyarrow==5.0.0 \ + --hash=sha256:1832709281efefa4f199c639e9f429678286329860188e53beeda71750775923 \ + --hash=sha256:1d9485741e497ccc516cb0a0c8f56e22be55aea815be185c3f9a681323b0e614 \ + --hash=sha256:24e64ea33eed07441cc0e80c949e3a1b48211a1add8953268391d250f4d39922 \ + --hash=sha256:2d26186ca9748a1fb89ae6c1fa04fb343a4279b53f118734ea8096f15d66c820 \ + --hash=sha256:357605665fbefb573d40939b13a684c2490b6ed1ab4a5de8dd246db4ab02e5a4 \ + --hash=sha256:4341ac0f552dc04c450751e049976940c7f4f8f2dae03685cc465ebe0a61e231 \ + --hash=sha256:456a4488ae810a0569d1adf87dbc522bcc9a0e4a8d1809b934ca28c163d8edce \ + --hash=sha256:4d8adda1892ef4553c4804af7f67cce484f4d6371564e2d8374b8e2bc85293e2 \ + --hash=sha256:53e550dec60d1ab86cba3afa1719dc179a8bc9632a0e50d9fe91499cf0a7f2bc \ + --hash=sha256:5c0d1b68e67bb334a5af0cecdf9b6a702aaa4cc259c5cbb71b25bbed40fcedaf \ + --hash=sha256:601b0aabd6fb066429e706282934d4d8d38f53bdb8d82da9576be49f07eedf5c \ + --hash=sha256:64f30aa6b28b666a925d11c239344741850eb97c29d3aa0f7187918cf82494f7 \ + --hash=sha256:6e1f0e4374061116f40e541408a8a170c170d0a070b788717e18165ebfdd2a54 \ + --hash=sha256:6e937ce4a40ea0cc7896faff96adecadd4485beb53fbf510b46858e29b2e75ae \ + --hash=sha256:7560332e5846f0e7830b377c14c93624e24a17f91c98f0b25dafb0ca1ea6ba02 \ + --hash=sha256:7c4edd2bacee3eea6c8c28bddb02347f9d41a55ec9692c71c6de6e47c62a7f0d \ + --hash=sha256:99c8b0f7e2ce2541dd4c0c0101d9944bb8e592ae3295fe7a2f290ab99222666d \ + --hash=sha256:9e04d3621b9f2f23898eed0d044203f66c156d880f02c5534a7f9947ebb1a4af \ + --hash=sha256:b1453c2411b5062ba6bf6832dbc4df211ad625f678c623a2ee177aee158f199b \ + --hash=sha256:b3115df938b8d7a7372911a3cb3904196194bcea8bb48911b4b3eafee3ab8d90 \ + --hash=sha256:b6387d2058d95fa48ccfedea810a768187affb62f4a3ef6595fa30bf9d1a65cf \ + --hash=sha256:bbe2e439bec2618c74a3bb259700c8a7353dc2ea0c5a62686b6cf04a50ab1e0d \ + --hash=sha256:c3fc856f107ca2fb3c9391d7ea33bbb33f3a1c2b4a0e2b41f7525c626214cc03 \ + --hash=sha256:c5493d2414d0d690a738aac8dd6d38518d1f9b870e52e24f89d8d7eb3afd4161 \ + --hash=sha256:e9ec80f4a77057498cf4c5965389e42e7f6a618b6859e6dd615e57505c9167a6 \ + --hash=sha256:ed135a99975380c27077f9d0e210aea8618ed9fadcec0e71f8a3190939557afe \ + --hash=sha256:f4db312e9ba80e730cefcae0a05b63ea5befc7634c28df56682b628ad8e1c25c \ + --hash=sha256:ff21711f6ff3b0bc90abc8ca8169e676faeb2401ddc1a0bc1c7dc181708a3406 + # via -r requirements.in pyasn1==0.4.8 \ --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba @@ -475,10 +622,16 @@ pytest-mypy==0.8.1 \ pytest-pydocstyle==2.2.0 \ --hash=sha256:b4e7eee47252fbb7495f4a0126e547a25f5f114f3096c504e738db49a5034333 # via -r requirements.in +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via pandas pytz==2021.1 \ --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 - # via google-api-core + # via + # google-api-core + # pandas pyyaml==6.0 \ --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ @@ -586,6 +739,7 @@ six==1.16.0 \ # grpcio # jsonschema # protobuf + # python-dateutil # virtualenv smmap==4.0.0 \ --hash=sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182 \