Fix date filtering in metric definition views (#921)

generator/__init__.py

@@ -3,6 +3,7 @@
 .. include:: ../README.md
 .. include:: ../architecture/namespaces_yaml.md
 """
+
 __docformat__ = "restructuredtext"
 
 import warnings

generator/__main__.py

@@ -1,6 +1,5 @@
 """Run lookml_generator cli."""
 
-
 from . import cli
 
 

generator/dashboards/__init__.py

@@ -1,4 +1,5 @@
 """All possible dashboard types."""
+
 from .dashboard import Dashboard  # noqa: F401
 from .operational_monitoring_dashboard import OperationalMonitoringDashboard
 

generator/dashboards/dashboard.py

@@ -1,4 +1,5 @@
 """Generic dashboard type."""
+
 from __future__ import annotations
 
 from dataclasses import dataclass, field

generator/dashboards/operational_monitoring_dashboard.py

@@ -1,4 +1,5 @@
 """Class to describe Operational Monitoring Dashboard."""
+
 from __future__ import annotations
 
 from typing import Any, Dict, List
@@ -129,10 +130,13 @@ class OperationalMonitoringDashboard(Dashboard):
                             kwargs["elements"].append(
                                 {
                                     "title": title,
-                                    "metric": summary["metric"]
-                                    if metric_group is None
-                                    else ", ".join(
-                                        f'"{m}"' for m in metric_groups[metric_group]
+                                    "metric": (
+                                        summary["metric"]
+                                        if metric_group is None
+                                        else ", ".join(
+                                            f'"{m}"'
+                                            for m in metric_groups[metric_group]
+                                        )
                                     ),
                                     "statistic": summary["statistic"],
                                     "explore": explore,
@@ -153,10 +157,13 @@ class OperationalMonitoringDashboard(Dashboard):
                             kwargs["elements"].append(
                                 {
                                     "title": f"{title} - By {self.group_by_dimension}",
-                                    "metric": summary["metric"]
-                                    if metric_group is None
-                                    else ", ".join(
-                                        f'"{m}"' for m in metric_groups[metric_group]
+                                    "metric": (
+                                        summary["metric"]
+                                        if metric_group is None
+                                        else ", ".join(
+                                            f'"{m}"'
+                                            for m in metric_groups[metric_group]
+                                        )
                                     ),
                                     "statistic": summary["statistic"],
                                     "explore": explore,

generator/explores/__init__.py

@@ -1,4 +1,5 @@
 """All possible explore types."""
+
 from .explore import Explore  # noqa: F401 isort:skip
 from .client_counts_explore import ClientCountsExplore
 from .events_explore import EventsExplore

generator/explores/client_counts_explore.py

@@ -1,4 +1,5 @@
 """Client Counts explore type."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/explores/events_explore.py

@@ -1,4 +1,5 @@
 """An explore for Events Views."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/explores/explore.py

@@ -1,4 +1,5 @@
 """Generic explore type."""
+
 from __future__ import annotations
 
 from dataclasses import dataclass, field

generator/explores/funnel_analysis_explore.py

@@ -1,4 +1,5 @@
 """Funnel Analysis explore type."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/explores/growth_accounting_explore.py

@@ -1,4 +1,5 @@
 """Growth Accounting explore type."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/explores/metric_definitions_explore.py

@@ -1,4 +1,5 @@
 """Metric Hub metrics explore type."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/explores/operational_monitoring_explore.py

@@ -1,4 +1,5 @@
 """Operational Monitoring Explore type."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/explores/ping_explore.py

@@ -1,4 +1,5 @@
 """Ping explore type."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/explores/table_explore.py

@@ -1,4 +1,5 @@
 """Table explore type."""
+
 from __future__ import annotations
 
 from pathlib import Path

generator/lkml_update.py

@@ -1,4 +1,5 @@
 """An updated lkml parser to handle explore queries."""
+
 from typing import List, Union
 
 from lkml.keys import KEYS_WITH_NAME_FIELDS

generator/lookml.py

@@ -1,4 +1,5 @@
 """Generate lookml from namespaces."""
+
 import logging
 from pathlib import Path
 from typing import Dict, Iterable, Optional

generator/namespaces.py

@@ -1,4 +1,5 @@
 """Generate namespaces.yaml."""
+
 import fnmatch
 import json
 import re
@@ -241,12 +242,16 @@ def _get_glean_apps(
         channels = [
             {
                 "channel": channel.get("app_channel"),
-                "dataset": channel.get("app_name").replace("-", "_")
-                if channel.get("app_channel") == "release"
-                else channel.get("bq_dataset_family"),
-                "source_dataset": channel.get("bq_dataset_family")
-                if channel.get("app_channel") == "release"
-                else channel.get("bq_dataset_family") + "_stable",
+                "dataset": (
+                    channel.get("app_name").replace("-", "_")
+                    if channel.get("app_channel") == "release"
+                    else channel.get("bq_dataset_family")
+                ),
+                "source_dataset": (
+                    channel.get("bq_dataset_family")
+                    if channel.get("app_channel") == "release"
+                    else channel.get("bq_dataset_family") + "_stable"
+                ),
             }
             for channel in variants
             if not channel.get("deprecated")

generator/operational_monitoring_utils.py

@@ -1,4 +1,5 @@
 """Utils for operational monitoring."""
+
 from multiprocessing.pool import ThreadPool
 from typing import Any, Dict, List, Optional, Tuple
 

generator/views/__init__.py

@@ -1,4 +1,5 @@
 """All available Looker views."""
+
 from .client_counts_view import ClientCountsView
 from .events_view import EventsView
 from .funnel_analysis_view import FunnelAnalysisView

generator/views/client_counts_view.py

@@ -1,4 +1,5 @@
 """Class to describe a Client Counts View."""
+
 from __future__ import annotations
 
 from copy import deepcopy

generator/views/events_view.py

@@ -1,4 +1,5 @@
 """Class to describe an Events view."""
+
 from __future__ import annotations
 
 from copy import deepcopy

generator/views/funnel_analysis_view.py

@@ -19,6 +19,7 @@ That lets us find out whether the user completed those funnel steps:
 The `funnel_analysis` view has some nice dimensions to hide these details from the end user,
 e.g. `completed_funnel_step_N`. We can then count those users across dimensions.
 """
+
 from __future__ import annotations
 
 from textwrap import dedent

generator/views/growth_accounting_view.py

@@ -1,4 +1,5 @@
 """Class to describe a Growth Accounting View."""
+
 from __future__ import annotations
 
 from copy import deepcopy

generator/views/lookml_utils.py

@@ -1,4 +1,5 @@
 """Utils for generating lookml."""
+
 import re
 import tarfile
 import urllib.request

generator/views/metric_definitions_view.py

@@ -105,18 +105,30 @@ class MetricDefinitionsView(View):
                 f"base_{d['name']} AS {d['name']},\n"
                 for d in base_view_lkml["views"][0]["dimensions"]
                 if d["name"] not in ignore_base_fields and "hidden" not in d
+            ] + [
+                f"base_{d['sql'].replace('${TABLE}.', '')} AS {d['sql'].replace('${TABLE}.', '')},\n"
+                for d in base_view_lkml["views"][0]["dimension_groups"]
+                if d["name"] not in ignore_base_fields and "hidden" not in d
             ]
 
             base_view_fields = [
                 f"{d['name']},\n"
                 for d in base_view_lkml["views"][0]["dimensions"]
                 if d["name"] not in ignore_base_fields and "hidden" not in d
+            ] + [
+                f"{d['sql'].replace('${TABLE}.', '')},\n"
+                for d in base_view_lkml["views"][0]["dimension_groups"]
+                if d["name"] not in ignore_base_fields and "hidden" not in d
             ]
 
             selected_fields = [
                 f"{d['name'].replace('__', '.')} AS base_{d['name']},\n"
                 for d in base_view_lkml["views"][0]["dimensions"]
                 if d["name"] not in ignore_base_fields and "hidden" not in d
+            ] + [
+                f"{d['sql'].replace('${TABLE}.', '').replace('__', '.')} AS base_{d['sql'].replace('${TABLE}.', '')},\n"
+                for d in base_view_lkml["views"][0]["dimension_groups"]
+                if d["name"] not in ignore_base_fields and "hidden" not in d
             ]
 
             client_id_join = (
@@ -132,17 +144,30 @@ class MetricDefinitionsView(View):
                 {"".join(selected_fields)}
                 FROM
                 {base_table}
+                WHERE
+                submission_date BETWEEN
+                COALESCE(
+                    SAFE_CAST(
+                    {{% date_start {data_source_definition.submission_date_column or "submission_date"} %}} AS DATE),
+                CURRENT_DATE()) AND
+                COALESCE(
+                    SAFE_CAST(
+                        {{% date_end {data_source_definition.submission_date_column or "submission_date"} %}} AS DATE
+                ), CURRENT_DATE())
             ) base
             ON
                 base.base_submission_date = m.{data_source_definition.submission_date_column or "submission_date"}
                 {client_id_join}
             WHERE base.base_submission_date BETWEEN
+            COALESCE(
                 SAFE_CAST(
                     {{% date_start {data_source_definition.submission_date_column or "submission_date"} %}} AS DATE
-                ) AND
+            ), CURRENT_DATE()) AND
+            COALESCE(
                 SAFE_CAST(
                     {{% date_end {data_source_definition.submission_date_column or "submission_date"} %}} AS DATE
-                ) AND
+                ), CURRENT_DATE())
+            AND
                 base.base_sample_id < {{% parameter sampling %}}
             """
 
@@ -192,13 +217,17 @@ class MetricDefinitionsView(View):
                 }
             AS m
             {join_base_view}
-            {'AND' if join_base_view else 'WHERE'} m.submission_date BETWEEN
+            {'AND' if join_base_view else 'WHERE'}
+            m.{data_source_definition.submission_date_column or "submission_date"}
+            BETWEEN
+            COALESCE(
                 SAFE_CAST(
                     {{% date_start {data_source_definition.submission_date_column or "submission_date"} %}} AS DATE
-                ) AND
+                ), CURRENT_DATE()) AND
+            COALESCE(
                 SAFE_CAST(
                     {{% date_end {data_source_definition.submission_date_column or "submission_date"} %}} AS DATE
-                )
+                ), CURRENT_DATE())
             GROUP BY
                 {"".join(base_view_fields)}
                 client_id,
@@ -382,9 +411,11 @@ class MetricDefinitionsView(View):
                     elif statistic_slug == "client_count":
                         measures.append(
                             {
-                                "name": f"{dimension['name']}_{statistic_slug}_sampled"
-                                if sampling
-                                else f"{dimension['name']}_{statistic_slug}",
+                                "name": (
+                                    f"{dimension['name']}_{statistic_slug}_sampled"
+                                    if sampling
+                                    else f"{dimension['name']}_{statistic_slug}"
+                                ),
                                 "type": "count_distinct",
                                 "label": f"{dimension['label']} Client Count",
                                 "group_label": "Statistics",

generator/views/operational_monitoring_view.py

@@ -1,4 +1,5 @@
 """Class to describe an Operational Monitoring View."""
+
 from __future__ import annotations
 
 from typing import Any, Dict, List, Optional, Union

generator/views/ping_view.py

@@ -1,4 +1,5 @@
 """Class to describe a Ping View."""
+
 from __future__ import annotations
 
 from collections import defaultdict

generator/views/table_view.py

@@ -1,4 +1,5 @@
 """Class to describe a Table View."""
+
 from __future__ import annotations
 
 from collections import defaultdict

generator/views/view.py

@@ -1,4 +1,5 @@
 """Generic class to describe Looker views."""
+
 from __future__ import annotations
 
 from typing import Any, Dict, Iterator, List, Optional, Set, TypedDict