bqetl_nondesktop DAG

dags.yaml

@@ -63,6 +63,15 @@ bqetl_core:
     retries: 1
     retry_delay: 5m
 
+bqetl_nondesktop:
+  schedule_interval: 0 1 * * *
+  default_args:
+    owner: jklukas@mozilla.com
+    start_date: '2019-07-25'
+    email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
+    retries: 1
+    retry_delay: 5m
+
 # DAG for exporting query data marked as public to GCS
 # queries should not be explicitly assigned to this DAG (it's done automatically)
 bqetl_public_data_json:

dags/bqetl_kpi_dashboard.py

@@ -56,6 +56,28 @@ with DAG(
         dag=dag,
     )
 
+    wait_for_telemetry_derived__smoot_usage_nondesktop__v2 = ExternalTaskSensor(
+        task_id="wait_for_telemetry_derived__smoot_usage_nondesktop__v2",
+        external_dag_id="bqetl_nondesktop",
+        external_task_id="telemetry_derived__smoot_usage_nondesktop__v2",
+        check_existence=True,
+    )
+
+    telemetry_derived__smoot_usage_new_profiles__v2.set_upstream(
+        wait_for_telemetry_derived__smoot_usage_nondesktop__v2
+    )
+
     telemetry_derived__smoot_usage_new_profiles_compressed__v2.set_upstream(
         telemetry_derived__smoot_usage_new_profiles__v2
     )
+
+    wait_for_telemetry__firefox_nondesktop_exact_mau28_raw__v1 = ExternalTaskSensor(
+        task_id="wait_for_telemetry__firefox_nondesktop_exact_mau28_raw__v1",
+        external_dag_id="bqetl_nondesktop",
+        external_task_id="telemetry__firefox_nondesktop_exact_mau28_raw__v1",
+        check_existence=True,
+    )
+
+    kpi_dashboard.set_upstream(
+        wait_for_telemetry__firefox_nondesktop_exact_mau28_raw__v1
+    )

dags/bqetl_nondesktop.py

@@ -0,0 +1,108 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/master/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.operators.sensors import ExternalTaskSensor
+import datetime
+from utils.gcp import bigquery_etl_query
+
+default_args = {
+    "owner": "jklukas@mozilla.com",
+    "start_date": datetime.datetime(2019, 7, 25, 0, 0),
+    "email": ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=300),
+    "email_on_failure": True,
+    "email_on_retry": True,
+    "retries": 1,
+}
+
+with DAG(
+    "bqetl_nondesktop", default_args=default_args, schedule_interval="0 1 * * *"
+) as dag:
+
+    telemetry_derived__firefox_nondesktop_day_2_7_activation__v1 = bigquery_etl_query(
+        task_id="telemetry_derived__firefox_nondesktop_day_2_7_activation__v1",
+        destination_table="firefox_nondesktop_day_2_7_activation_v1",
+        dataset_id="telemetry_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="gkaberere@mozilla.com",
+        email=["gkaberere@mozilla.com", "jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    telemetry_derived__smoot_usage_nondesktop_compressed__v2 = bigquery_etl_query(
+        task_id="telemetry_derived__smoot_usage_nondesktop_compressed__v2",
+        destination_table="smoot_usage_nondesktop_compressed_v2",
+        dataset_id="telemetry_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    telemetry_derived__firefox_nondesktop_exact_mau28_by_client_count_dimensions__v1 = bigquery_etl_query(
+        task_id="telemetry_derived__firefox_nondesktop_exact_mau28_by_client_count_dimensions__v1",
+        destination_table="firefox_nondesktop_exact_mau28_by_client_count_dimensions_v1",
+        dataset_id="telemetry_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    telemetry_derived__smoot_usage_nondesktop__v2 = bigquery_etl_query(
+        task_id="telemetry_derived__smoot_usage_nondesktop__v2",
+        destination_table="smoot_usage_nondesktop_v2",
+        dataset_id="telemetry_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    telemetry__firefox_nondesktop_exact_mau28_raw__v1 = bigquery_etl_query(
+        task_id="telemetry__firefox_nondesktop_exact_mau28_raw__v1",
+        destination_table="firefox_nondesktop_exact_mau28_raw_v1",
+        dataset_id="telemetry",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    wait_for_telemetry_derived__core_clients_last_seen__v1 = ExternalTaskSensor(
+        task_id="wait_for_telemetry_derived__core_clients_last_seen__v1",
+        external_dag_id="bqetl_core",
+        external_task_id="telemetry_derived__core_clients_last_seen__v1",
+        check_existence=True,
+    )
+
+    telemetry_derived__firefox_nondesktop_day_2_7_activation__v1.set_upstream(
+        wait_for_telemetry_derived__core_clients_last_seen__v1
+    )
+
+    telemetry_derived__smoot_usage_nondesktop_compressed__v2.set_upstream(
+        telemetry_derived__smoot_usage_nondesktop__v2
+    )
+
+    telemetry_derived__firefox_nondesktop_exact_mau28_by_client_count_dimensions__v1.set_upstream(
+        wait_for_telemetry_derived__core_clients_last_seen__v1
+    )
+
+    telemetry_derived__smoot_usage_nondesktop__v2.set_upstream(
+        wait_for_telemetry_derived__core_clients_last_seen__v1
+    )
+
+    telemetry__firefox_nondesktop_exact_mau28_raw__v1.set_upstream(
+        wait_for_telemetry_derived__core_clients_last_seen__v1
+    )

sql/telemetry/firefox_nondesktop_exact_mau28_raw_v1/metadata.yaml

@@ -0,0 +1,11 @@
+friendly_name: Firefox Non-desktop Exact MAU Raw
+description: >
+  Monthly active users with non-desktop clients.
+owners:
+  - jklukas@mozilla.com
+labels:
+  application: nondesktop
+  schedule: daily
+  incremental: true
+scheduling:
+  dag_name: bqetl_nondesktop

sql/telemetry_derived/firefox_nondesktop_day_2_7_activation_v1/metadata.yaml

@@ -8,3 +8,6 @@ labels:
   application: nondesktop
   incremental: true
   schedule: daily
+scheduling:
+  dag_name: bqetl_nondesktop
+  email: ['jklukas@mozilla.com']

sql/telemetry_derived/firefox_nondesktop_exact_mau28_by_client_count_dimensions_v1/metadata.yaml

@@ -0,0 +1,11 @@
+friendly_name: Firefox Non-desktop Exact MAU by Client Count Dimensions
+description: >
+  Monthly active users with non-desktop clients aggregated across unique sets of dimensions.
+owners:
+  - jklukas@mozilla.com
+labels:
+  application: nondesktop
+  schedule: daily
+  incremental: true
+scheduling:
+  dag_name: bqetl_nondesktop

sql/telemetry_derived/smoot_usage_nondesktop_compressed_v2/metadata.yaml

@@ -0,0 +1,11 @@
+friendly_name: Smoot Usage Non-desktop Compressed
+description: >
+  Compressed usage metrics for non-desktop clients.
+owners:
+  - jklukas@mozilla.com
+labels:
+  application: nondesktop
+  incremental: true
+  schedule: daily
+scheduling:
+  dag_name: bqetl_nondesktop

sql/telemetry_derived/smoot_usage_nondesktop_v2/metadata.yaml

@@ -0,0 +1,11 @@
+friendly_name: Smoot Usage Non-desktop
+description: >
+  Usage metrics for non-desktop clients.
+owners:
+  - jklukas@mozilla.com
+labels:
+  application: nondesktop
+  incremental: true
+  schedule: daily
+scheduling:
+  dag_name: bqetl_nondesktop