DAG for vrbrowser queries

bigquery_etl/query_scheduling/templates/airflow_dag.j2

@@ -40,7 +40,9 @@ with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None
 {% for task in tasks %}
     {% for dependency in task.dependencies -%}
     {% if dependency.dag_name == name -%}
+    {% if dependency.task_name != task.task_name %}
     {{ task.task_name }}.set_upstream({{ dependency.task_name }})
+    {% endif -%}
     {% else -%}
     {% if (dependency.dag_name, dependency.task_name) not in wait_for_seen -%}
     wait_for_{{ dependency.task_name }} = ExternalTaskSensor(

dags.yaml

@@ -45,6 +45,15 @@ bqetl_amo_stats:
       retries: 2
       retry_delay: 30m
 
+bqetl_vrbrowser:
+  schedule_interval: 0 1 * * *
+  default_args:
+    owner: jklukas@mozilla.com
+    start_date: '2019-07-25'
+    email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com', 'ascholtz@mozilla.com']
+    retries: 1
+    retry_delay: 5m
+
 # DAG for exporting query data marked as public to GCS
 # queries should not be explicitly assigned to this DAG (it's done automatically)
 bqetl_public_data_json:

dags/bqetl_vrbrowser.py

@@ -0,0 +1,101 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/master/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.operators.sensors import ExternalTaskSensor
+import datetime
+from utils.gcp import bigquery_etl_query
+
+default_args = {
+    "owner": "jklukas@mozilla.com",
+    "start_date": datetime.datetime(2019, 7, 25, 0, 0),
+    "email": [
+        "telemetry-alerts@mozilla.com",
+        "jklukas@mozilla.com",
+        "ascholtz@mozilla.com",
+    ],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=300),
+    "email_on_failure": True,
+    "email_on_retry": True,
+    "retries": 1,
+}
+
+with DAG(
+    "bqetl_vrbrowser", default_args=default_args, schedule_interval="0 1 * * *"
+) as dag:
+
+    org_mozilla_vrbrowser_derived__baseline_daily__v1 = bigquery_etl_query(
+        task_id="org_mozilla_vrbrowser_derived__baseline_daily__v1",
+        destination_table="baseline_daily_v1",
+        dataset_id="org_mozilla_vrbrowser_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["ascholtz@mozilla.com", "jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    org_mozilla_vrbrowser_derived__metrics_daily__v1 = bigquery_etl_query(
+        task_id="org_mozilla_vrbrowser_derived__metrics_daily__v1",
+        destination_table="metrics_daily_v1",
+        dataset_id="org_mozilla_vrbrowser_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["ascholtz@mozilla.com", "jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    org_mozilla_vrbrowser_derived__clients_last_seen__v1 = bigquery_etl_query(
+        task_id="org_mozilla_vrbrowser_derived__clients_last_seen__v1",
+        destination_table="clients_last_seen_v1",
+        dataset_id="org_mozilla_vrbrowser_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["ascholtz@mozilla.com", "jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    org_mozilla_vrbrowser_derived__clients_daily__v1 = bigquery_etl_query(
+        task_id="org_mozilla_vrbrowser_derived__clients_daily__v1",
+        destination_table="clients_daily_v1",
+        dataset_id="org_mozilla_vrbrowser_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="jklukas@mozilla.com",
+        email=["ascholtz@mozilla.com", "jklukas@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    wait_for_copy_deduplicate_copy_deduplicate_all = ExternalTaskSensor(
+        task_id="wait_for_copy_deduplicate_copy_deduplicate_all",
+        external_dag_id="copy_deduplicate",
+        external_task_id="copy_deduplicate_all",
+        check_existence=True,
+        dag=dag,
+    )
+
+    org_mozilla_vrbrowser_derived__baseline_daily__v1.set_upstream(
+        wait_for_copy_deduplicate_copy_deduplicate_all
+    )
+
+    org_mozilla_vrbrowser_derived__metrics_daily__v1.set_upstream(
+        wait_for_copy_deduplicate_copy_deduplicate_all
+    )
+
+    org_mozilla_vrbrowser_derived__clients_last_seen__v1.set_upstream(
+        org_mozilla_vrbrowser_derived__clients_daily__v1
+    )
+
+    org_mozilla_vrbrowser_derived__clients_daily__v1.set_upstream(
+        org_mozilla_vrbrowser_derived__baseline_daily__v1
+    )
+
+    org_mozilla_vrbrowser_derived__clients_daily__v1.set_upstream(
+        org_mozilla_vrbrowser_derived__metrics_daily__v1
+    )

sql/org_mozilla_vrbrowser_derived/baseline_daily_v1/metadata.yaml

@@ -8,3 +8,8 @@ owners:
 labels:
   application: firefox-reality
   schedule: daily
+scheduling:
+  dag_name: bqetl_vrbrowser
+  depends_on:
+    - dag_name: copy_deduplicate
+      task_id: copy_deduplicate_all

sql/org_mozilla_vrbrowser_derived/clients_daily_v1/metadata.yaml

@@ -8,3 +8,5 @@ owners:
 labels:
   application: firefox-reality
   schedule: daily
+scheduling:
+  dag_name: bqetl_vrbrowser

sql/org_mozilla_vrbrowser_derived/clients_last_seen_v1/metadata.yaml

@@ -8,3 +8,5 @@ owners:
 labels:
   application: firefox-reality
   schedule: daily
+scheduling:
+  dag_name: bqetl_vrbrowser

sql/org_mozilla_vrbrowser_derived/metrics_daily_v1/metadata.yaml

@@ -8,3 +8,8 @@ owners:
 labels:
   application: firefox-reality
   schedule: daily
+scheduling:
+  dag_name: bqetl_vrbrowser
+  depends_on:
+    - dag_name: copy_deduplicate
+      task_id: copy_deduplicate_all

tests/data/dags/test_dag_with_dependencies

@@ -53,5 +53,7 @@ with DAG('bqetl_test_dag', default_args=default_args, schedule_interval='@daily'
     )
     
     {{ temporary_dataset }}__query__v1.set_upstream(wait_for_{{ temporary_dataset }}__external_table__v1)
+    
     {{ temporary_dataset }}__query__v1.set_upstream({{ temporary_dataset }}__table1__v1)
+    
     {{ temporary_dataset }}__query__v1.set_upstream({{ temporary_dataset }}__table2__v1)