Include crash fields in clients_daily_joined (#3275)

dags.yaml

@@ -819,22 +819,3 @@ bqetl_desktop_mobile_search_monthly:
   tags:
     - impact/tier_1
     - repo/bigquery-etl
-
-bqetl_telemetry_crashes_daily:
-  default_args:
-    depends_on_past: false
-    email:
-      - telemetry-alerts@mozilla.com
-      - frank@mozilla.com
-    email_on_failure: true
-    email_on_retry: true
-    end_date: null
-    owner: frank@mozilla.com
-    retries: 2
-    retry_delay: 30m
-    start_date: '2022-08-30'
-  description: Telemetry crashes daily
-  schedule_interval: 0 1 * * *
-  tags:
-    - impact/tier_2
-    - repo/bigquery-etl

dags/bqetl_main_summary.py

@@ -67,6 +67,23 @@ with DAG(
         task_concurrency=1,
     )
 
+    crashes_daily_v1 = bigquery_etl_query(
+        task_id="crashes_daily_v1",
+        destination_table="crashes_daily_v1",
+        dataset_id="telemetry_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="frank@mozilla.com",
+        email=[
+            "dthorn@mozilla.com",
+            "frank@mozilla.com",
+            "jklukas@mozilla.com",
+            "telemetry-alerts@mozilla.com",
+        ],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        priority_weight=85,
+    )
+
     firefox_desktop_exact_mau28_by_client_count_dimensions = bigquery_etl_query(
         task_id="firefox_desktop_exact_mau28_by_client_count_dimensions",
         destination_table="firefox_desktop_exact_mau28_by_client_count_dimensions_v1",
@@ -185,9 +202,10 @@ with DAG(
         destination_table="clients_daily_joined_v1",
         dataset_id="telemetry_derived",
         project_id="moz-fx-data-shared-prod",
-        owner="jklukas@mozilla.com",
+        owner="frank@mozilla.com",
         email=[
             "dthorn@mozilla.com",
+            "frank@mozilla.com",
             "jklukas@mozilla.com",
             "telemetry-alerts@mozilla.com",
         ],
@@ -536,6 +554,8 @@ with DAG(
 
     telemetry_derived__clients_daily_event__v1.set_upstream(wait_for_event_events)
 
+    telemetry_derived__clients_daily_joined__v1.set_upstream(crashes_daily_v1)
+
     telemetry_derived__clients_daily_joined__v1.set_upstream(
         telemetry_derived__clients_daily__v6
     )

dags/bqetl_telemetry_crashes_daily.py

@@ -1,56 +0,0 @@
-# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py
-
-from airflow import DAG
-from airflow.sensors.external_task import ExternalTaskMarker
-from airflow.sensors.external_task import ExternalTaskSensor
-from airflow.utils.task_group import TaskGroup
-import datetime
-from utils.constants import ALLOWED_STATES, FAILED_STATES
-from utils.gcp import bigquery_etl_query, gke_command
-
-docs = """
-### bqetl_telemetry_crashes_daily
-
-Built from bigquery-etl repo, [`dags/bqetl_telemetry_crashes_daily.py`](https://github.com/mozilla/bigquery-etl/blob/main/dags/bqetl_telemetry_crashes_daily.py)
-
-#### Description
-
-Telemetry crashes daily
-#### Owner
-
-frank@mozilla.com
-"""
-
-
-default_args = {
-    "owner": "frank@mozilla.com",
-    "start_date": datetime.datetime(2022, 8, 30, 0, 0),
-    "end_date": None,
-    "email": ["telemetry-alerts@mozilla.com", "frank@mozilla.com"],
-    "depends_on_past": False,
-    "retry_delay": datetime.timedelta(seconds=1800),
-    "email_on_failure": True,
-    "email_on_retry": True,
-    "retries": 2,
-}
-
-tags = ["impact/tier_2", "repo/bigquery-etl"]
-
-with DAG(
-    "bqetl_telemetry_crashes_daily",
-    default_args=default_args,
-    schedule_interval="0 1 * * *",
-    doc_md=docs,
-    tags=tags,
-) as dag:
-
-    crashes_daily_v1 = bigquery_etl_query(
-        task_id="crashes_daily_v1",
-        destination_table="crashes_daily_v1",
-        dataset_id="telemetry_derived",
-        project_id="moz-fx-data-shared-prod",
-        owner="frank@mozilla.com",
-        email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"],
-        date_partition_parameter="submission_date",
-        depends_on_past=False,
-    )

sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_joined_v1/metadata.yaml

@@ -9,7 +9,7 @@ description: |-
   we received a main ping from that client on that day. The first_seen_date
   and second_seen_date is added to each client from clients_last_seen.
 owners:
-- jklukas@mozilla.com
+- frank@mozilla.com
 labels:
   application: firefox
   schedule: daily
@@ -22,7 +22,9 @@ bigquery:
     field: submission_date
     type: day
     require_partition_filter: true
+    expiration_days: null
   clustering:
     fields:
     - normalized_channel
     - sample_id
+references: {}

sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_joined_v1/query.sql

@@ -18,6 +18,23 @@ daily_events AS (
     telemetry_derived.clients_daily_event_v1
   WHERE
     submission_date = @submission_date
+),
+crashes_daily AS (
+  SELECT
+    submission_date,
+    sample_id,
+    client_id,
+    main_crash_count,
+    content_crash_count,
+    gpu_crash_count,
+    rdd_crash_count,
+    socket_crash_count,
+    utility_crash_count,
+    vr_crash_count,
+  FROM
+    telemetry.crashes_daily
+  WHERE
+    submission_date = @submission_date
 )
 SELECT
   *
@@ -31,5 +48,9 @@ LEFT JOIN
   first_seen
 USING
   (submission_date, sample_id, client_id)
+LEFT JOIN
+  crashes_daily
+USING
+  (submission_date, sample_id, client_id)
 WHERE
   cd.submission_date = @submission_date

sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_joined_v1/schema.yaml

@@ -2049,3 +2049,24 @@ fields:
 - mode: NULLABLE
   name: logins_migrations_quantity_all
   type: INTEGER
+- mode: NULLABLE
+  name: main_crash_count
+  type: INTEGER
+- mode: NULLABLE
+  name: content_crash_count
+  type: INTEGER
+- mode: NULLABLE
+  name: gpu_crash_count
+  type: INTEGER
+- mode: NULLABLE
+  name: rdd_crash_count
+  type: INTEGER
+- mode: NULLABLE
+  name: socket_crash_count
+  type: INTEGER
+- mode: NULLABLE
+  name: utility_crash_count
+  type: INTEGER
+- mode: NULLABLE
+  name: vr_crash_count
+  type: INTEGER

sql/moz-fx-data-shared-prod/telemetry_derived/crashes_daily_v1/metadata.yaml

@@ -11,8 +11,9 @@ owners:
 labels:
   incremental: true
 scheduling:
-  dag_name: bqetl_telemetry_crashes_daily
+  dag_name: bqetl_main_summary
   task_name: crashes_daily_v1
+  priority: 85
 bigquery:
   time_partitioning:
     field: submission_date