Create DAG for events rollup (#1301)

* Create DAG for events rollup

* Update sql/org_mozilla_firefox_derived/events_daily_v1/metadata.yaml

Co-authored-by: Anna Scholtz <anna@scholtzan.net>

Co-authored-by: Anna Scholtz <anna@scholtzan.net>

dags.yaml

@@ -199,3 +199,12 @@ bqetl_deletion_request_volume:
     email: ['telemetry-alerts@mozilla.com', 'dthorn@mozilla.com']
     retries: 2
     retry_delay: 30m
+
+bqetl_fenix_event_rollup:   # name of the DAG; must start with bqetl_
+  schedule_interval: 0 2 * * *    # query schedule
+  default_args:
+    owner: frank@mozilla.com
+    start_date: '2020-09-09'  # YYYY-MM-DD
+    email: ['frank@mozilla.com']
+    retries: 2    # number of retries if the query execution fails
+    retry_delay: 30m

dags/bqetl_fenix_event_rollup.py

@@ -0,0 +1,67 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/master/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.operators.sensors import ExternalTaskSensor
+import datetime
+from utils.gcp import bigquery_etl_query
+
+default_args = {
+    "owner": "frank@mozilla.com",
+    "start_date": datetime.datetime(2020, 9, 9, 0, 0),
+    "email": ["frank@mozilla.com"],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=1800),
+    "email_on_failure": True,
+    "email_on_retry": True,
+    "retries": 2,
+}
+
+with DAG(
+    "bqetl_fenix_event_rollup", default_args=default_args, schedule_interval="0 2 * * *"
+) as dag:
+
+    org_mozilla_firefox_derived__event_types__v1 = bigquery_etl_query(
+        task_id="org_mozilla_firefox_derived__event_types__v1",
+        destination_table="event_types_v1",
+        dataset_id="org_mozilla_firefox_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="frank@mozilla.com",
+        email=["frank@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=True,
+        dag=dag,
+    )
+
+    org_mozilla_firefox_derived__events_daily__v1 = bigquery_etl_query(
+        task_id="org_mozilla_firefox_derived__events_daily__v1",
+        destination_table="events_daily_v1",
+        dataset_id="org_mozilla_firefox_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="frank@mozilla.com",
+        email=["frank@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    wait_for_copy_deduplicate_all = ExternalTaskSensor(
+        task_id="wait_for_copy_deduplicate_all",
+        external_dag_id="copy_deduplicate",
+        external_task_id="copy_deduplicate_all",
+        execution_delta=datetime.timedelta(seconds=3600),
+        check_existence=True,
+        mode="reschedule",
+        pool="DATA_ENG_EXTERNALTASKSENSOR",
+    )
+
+    org_mozilla_firefox_derived__event_types__v1.set_upstream(
+        wait_for_copy_deduplicate_all
+    )
+
+    org_mozilla_firefox_derived__events_daily__v1.set_upstream(
+        wait_for_copy_deduplicate_all
+    )
+
+    org_mozilla_firefox_derived__events_daily__v1.set_upstream(
+        org_mozilla_firefox_derived__event_types__v1
+    )

sql/org_mozilla_firefox_derived/event_types_v1/metadata.yaml

@@ -6,3 +6,6 @@ labels:
   application: firefox-android
   incremental: false
   schedule: daily
+scheduling:
+  dag_name: bqetl_fenix_event_rollup
+  depends_on_past: true

sql/org_mozilla_firefox_derived/events_daily_v1/metadata.yaml

@@ -6,3 +6,5 @@ labels:
   application: fenix
   schedule: daily
   incremental: true
+scheduling:
+  dag_name: bqetl_fenix_event_rollup