Generate BigConfig files for views (#6312)

* Generate BigConfig files for views

* Re-enable monitoring for telemetry.releases

bigquery_etl/cli/monitoring.py

@@ -12,7 +12,15 @@ from bigeye_sdk.client.datawatch_client import datawatch_client_factory
 from bigeye_sdk.client.enum import Method
 from bigeye_sdk.controller.metric_suite_controller import MetricSuiteController
 from bigeye_sdk.exceptions.exceptions import FileLoadException
-from bigeye_sdk.model.big_config import BigConfig, TableDeployment, TableDeploymentSuite
+from bigeye_sdk.model.big_config import (
+    BigConfig,
+    ColumnSelector,
+    RowCreationTimes,
+    TableDeployment,
+    TableDeploymentSuite,
+    TagDeployment,
+    TagDeploymentSuite,
+)
 from bigeye_sdk.model.protobuf_message_facade import (
     SimpleCollection,
     SimpleMetricDefinition,
@@ -130,6 +138,122 @@ def deploy(
             print("No metadata file for: {}.{}.{}".format(project, dataset, table))
 
 
+def _update_table_bigconfig(
+    bigconfig,
+    metadata,
+    project,
+    dataset,
+    table,
+):
+    """Update the BigConfig file to monitor a table."""
+    default_metrics = [
+        SimplePredefinedMetricName.FRESHNESS,
+        SimplePredefinedMetricName.VOLUME,
+    ]
+
+    for collection in bigconfig.table_deployments:
+        for deployment in collection.deployments:
+            for metric in deployment.table_metrics:
+                if metric.metric_type.predefined_metric in default_metrics:
+                    default_metrics.remove(metric.metric_type.predefined_metric)
+
+        if metadata.monitoring.collection and collection.collection is None:
+            collection.collection = SimpleCollection(
+                name=metadata.monitoring.collection
+            )
+
+    if len(default_metrics) > 0:
+        deployments = [
+            TableDeployment(
+                fq_table_name=f"{project}.{project}.{dataset}.{table}",
+                table_metrics=[
+                    SimpleMetricDefinition(
+                        metric_type=SimplePredefinedMetric(
+                            type="PREDEFINED", predefined_metric=metric
+                        ),
+                        metric_schedule=SimpleMetricSchedule(
+                            named_schedule=SimpleNamedSchedule(
+                                name="Default Schedule - 13:00 UTC"
+                            )
+                        ),
+                    )
+                    for metric in default_metrics
+                ],
+            )
+        ]
+
+        collection = None
+        if metadata.monitoring.collection:
+            collection = SimpleCollection(name=metadata.monitoring.collection)
+
+        bigconfig.table_deployments += [
+            TableDeploymentSuite(deployments=deployments, collection=collection)
+        ]
+
+
+def _update_view_bigconfig(
+    bigconfig,
+    metadata,
+    project,
+    dataset,
+    table,
+):
+    """Update the BigConfig file to monitor a view."""
+    default_metrics = [
+        SimplePredefinedMetricName.FRESHNESS_DATA,
+        SimplePredefinedMetricName.VOLUME_DATA,
+    ]
+
+    for collection in bigconfig.tag_deployments:
+        for deployment in collection.deployments:
+            for metric in deployment.metrics:
+                if metric.metric_type.predefined_metric in default_metrics:
+                    default_metrics.remove(metric.metric_type.predefined_metric)
+
+        if metadata.monitoring.collection and collection.collection is None:
+            collection.collection = SimpleCollection(
+                name=metadata.monitoring.collection
+            )
+
+    if len(default_metrics) > 0:
+        deployments = [
+            TagDeployment(
+                column_selectors=[
+                    ColumnSelector(name=f"{project}.{project}.{dataset}.{table}.*")
+                ],
+                metrics=[
+                    SimpleMetricDefinition(
+                        metric_type=SimplePredefinedMetric(
+                            type="PREDEFINED", predefined_metric=metric
+                        ),
+                        metric_schedule=SimpleMetricSchedule(
+                            named_schedule=SimpleNamedSchedule(
+                                name="Default Schedule - 13:00 UTC"
+                            )
+                        ),
+                    )
+                    for metric in default_metrics
+                ],
+            )
+        ]
+
+        collection = None
+        if metadata.monitoring.collection:
+            collection = SimpleCollection(name=metadata.monitoring.collection)
+
+        bigconfig.tag_deployments += [
+            TagDeploymentSuite(deployments=deployments, collection=collection)
+        ]
+
+        bigconfig.row_creation_times = RowCreationTimes(
+            column_selectors=[
+                ColumnSelector(
+                    name=f"{project}.{project}.{dataset}.{table}.{metadata.monitoring.partition_column}"
+                )
+            ]
+        )
+
+
 @monitoring.command(
     help="""
     Update BigConfig files based on monitoring metadata.
@@ -158,55 +282,22 @@ def update(name: str, sql_dir: Optional[str], project_id: Optional[str]) -> None
                 else:
                     bigconfig = BigConfig(type="BIGCONFIG_FILE")
 
-                default_metrics = [
+                if (metadata_file.parent / VIEW_FILE).exists():
-                    SimplePredefinedMetricName.FRESHNESS,
+                    _update_view_bigconfig(
-                    SimplePredefinedMetricName.VOLUME,
+                        bigconfig=bigconfig,
-                ]
+                        metadata=metadata,
-
+                        project=project,
-                for collection in bigconfig.table_deployments:
+                        dataset=dataset,
-                    for deployment in collection.deployments:
+                        table=table,
-                        for metric in deployment.table_metrics:
+                    )
-                            if metric.metric_type.predefined_metric in default_metrics:
+                else:
-                                default_metrics.remove(
+                    _update_table_bigconfig(
-                                    metric.metric_type.predefined_metric
+                        bigconfig=bigconfig,
-                                )
+                        metadata=metadata,
-
+                        project=project,
-                    if metadata.monitoring.collection and collection.collection is None:
+                        dataset=dataset,
-                        collection.collection = SimpleCollection(
+                        table=table,
-                            name=metadata.monitoring.collection
+                    )
-                        )
-
-                if len(default_metrics) > 0:
-                    deployments = [
-                        TableDeployment(
-                            fq_table_name=f"{project}.{project}.{dataset}.{table}",
-                            table_metrics=[
-                                SimpleMetricDefinition(
-                                    metric_type=SimplePredefinedMetric(
-                                        type="PREDEFINED", predefined_metric=metric
-                                    ),
-                                    metric_schedule=SimpleMetricSchedule(
-                                        named_schedule=SimpleNamedSchedule(
-                                            name="Default Schedule - 17:00 UTC"
-                                        )
-                                    ),
-                                )
-                                for metric in default_metrics
-                            ],
-                        )
-                    ]
-
-                    collection = None
-                    if metadata.monitoring.collection:
-                        collection = SimpleCollection(
-                            name=metadata.monitoring.collection
-                        )
-
-                    bigconfig.table_deployments += [
-                        TableDeploymentSuite(
-                            deployments=deployments, collection=collection
-                        )
-                    ]
 
                 bigconfig.save(
                     output_path=bigconfig_file.parent,

sql/moz-fx-data-shared-prod/telemetry/releases/metadata.yaml

@@ -8,5 +8,5 @@ description: |-
 owners:
 - ascholtz@mozilla.com
 monitoring:
-  enabled: false
+  enabled: true
   partition_column: date