Add query for monitoring deletion request volume (#1106)

bigquery_etl/dryrun.py

@@ -26,6 +26,7 @@ SKIP = {
     # Access Denied
     "sql/activity_stream/impression_stats_flat/view.sql",
     "sql/activity_stream/tile_id_types/view.sql",
+    "sql/monitoring/deletion_request_volume_v1/query.sql",
     "sql/monitoring/document_sample_nonprod_v1/query.sql",
     "sql/monitoring/schema_error_counts_v1/view.sql",
     "sql/monitoring/structured_error_counts_v1/view.sql",

dags.yaml

@@ -172,3 +172,12 @@ bqetl_internet_outages:
     email: ['aplacitelli@mozilla.com', 'sguha@mozilla.com']
     retries: 2
     retry_delay: 30m
+
+bqetl_deletion_request_volume:
+  schedule_interval: 0 1 * * *
+  default_args:
+    owner: dthorn@mozilla.com
+    start_date: '2020-06-29'
+    email: ['telemetry-alerts@mozilla.com', 'dthorn@mozilla.com']
+    retries: 2
+    retry_delay: 30m

dags/bqetl_deletion_request_volume.py

@@ -0,0 +1,35 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/master/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.operators.sensors import ExternalTaskSensor
+import datetime
+from utils.gcp import bigquery_etl_query
+
+default_args = {
+    "owner": "dthorn@mozilla.com",
+    "start_date": datetime.datetime(2020, 6, 29, 0, 0),
+    "email": ["telemetry-alerts@mozilla.com", "dthorn@mozilla.com"],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=1800),
+    "email_on_failure": True,
+    "email_on_retry": True,
+    "retries": 2,
+}
+
+with DAG(
+    "bqetl_deletion_request_volume",
+    default_args=default_args,
+    schedule_interval="0 1 * * *",
+) as dag:
+
+    monitoring__deletion_request_volume__v1 = bigquery_etl_query(
+        task_id="monitoring__deletion_request_volume__v1",
+        destination_table="deletion_request_volume_v1",
+        dataset_id="monitoring",
+        project_id="moz-fx-data-shared-prod",
+        owner="dthorn@mozilla.com",
+        email=["dthorn@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )

sql/monitoring/deletion_request_volume_v1/metadata.yaml

@@ -0,0 +1,9 @@
+friendly_name: Deletion Request Volume
+description: >
+  A daily count of deletion request pings by document namespace
+owners:
+  - dthorn@mozilla.com
+labels:
+  schedule: daily
+scheduling:
+  dag_name: bqetl_deletion_request_volume

sql/monitoring/deletion_request_volume_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  DATE(submission_timestamp) AS submission_date,
+  REGEXP_EXTRACT(_TABLE_SUFFIX, "(?:stub_)?(?:[^_]+_)(.*)__") AS dataset_id,
+  COUNT(*) AS num_rows,
+FROM
+  `payload_bytes_decoded.*`
+WHERE
+  DATE(submission_timestamp) < CURRENT_DATE
+  AND (@submission_date IS NULL OR @submission_date = DATE(submission_timestamp))
+  AND _TABLE_SUFFIX LIKE '%__deletion_request_v%'
+GROUP BY
+  submission_date,
+  dataset_id