Add bqetl_search DAG

dags.yaml

@@ -97,6 +97,15 @@ bqetl_activity_stream:
     email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
     retries: 1
     retry_delay: 5m
+
+bqetl_search:
+  schedule_interval: 0 1 * * *
+  default_args:
+    owner: bewu@mozilla.com
+    start_date: '2018-11-27'
+    email: ['telemetry-alerts@mozilla.com', 'bewu@mozilla.com', 'frank@mozilla.com']
+    retries: 2
+    retry_delay: 30m
     
 # DAG for exporting query data marked as public to GCS
 # queries should not be explicitly assigned to this DAG (it's done automatically)

dags/bqetl_search.py

@@ -0,0 +1,78 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/master/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.operators.sensors import ExternalTaskSensor
+import datetime
+from utils.gcp import bigquery_etl_query
+
+default_args = {
+    "owner": "bewu@mozilla.com",
+    "start_date": datetime.datetime(2018, 11, 27, 0, 0),
+    "email": ["telemetry-alerts@mozilla.com", "bewu@mozilla.com", "frank@mozilla.com"],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=1800),
+    "email_on_failure": True,
+    "email_on_retry": True,
+    "retries": 2,
+}
+
+with DAG(
+    "bqetl_search", default_args=default_args, schedule_interval="0 1 * * *"
+) as dag:
+
+    search_derived__search_aggregates__v8 = bigquery_etl_query(
+        task_id="search_derived__search_aggregates__v8",
+        destination_table="search_aggregates_v8",
+        dataset_id="search_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="bewu@mozilla.com",
+        email=["bewu@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    search_derived__search_clients_last_seen__v1 = bigquery_etl_query(
+        task_id="search_derived__search_clients_last_seen__v1",
+        destination_table="search_clients_last_seen_v1",
+        dataset_id="search_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="frank@mozilla.com",
+        email=["frank@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=True,
+        dag=dag,
+    )
+
+    search_derived__search_clients_daily__v8 = bigquery_etl_query(
+        task_id="search_derived__search_clients_daily__v8",
+        destination_table="search_clients_daily_v8",
+        dataset_id="search_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="bewu@mozilla.com",
+        email=["bewu@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+        dag=dag,
+    )
+
+    search_derived__search_aggregates__v8.set_upstream(
+        search_derived__search_clients_daily__v8
+    )
+
+    search_derived__search_clients_last_seen__v1.set_upstream(
+        search_derived__search_clients_daily__v8
+    )
+
+    wait_for_main_summary_main_summary = ExternalTaskSensor(
+        task_id="wait_for_main_summary_main_summary",
+        external_dag_id="main_summary",
+        external_task_id="main_summary",
+        check_existence=True,
+        mode="reschedule",
+        dag=dag,
+    )
+
+    search_derived__search_clients_daily__v8.set_upstream(
+        wait_for_main_summary_main_summary
+    )

sql/search_derived/search_aggregates_v8/metadata.yaml

@@ -0,0 +1,10 @@
+friendly_name: Search Aggregates
+description: >-
+  Daily search clients, aggregated across unique sets of dimensions
+  and partitioned by day
+owners:
+  - bewu@mozilla.com
+labels:
+  schedule: daily
+scheduling:
+  dag_name: bqetl_search

sql/search_derived/search_clients_daily_v8/metadata.yaml

@@ -0,0 +1,11 @@
+friendly_name: Search Clients Daily
+description: A daily aggregate of search clients, partitioned by day
+owners:
+  - bewu@mozilla.com
+labels:
+  schedule: daily
+scheduling:
+  dag_name: bqetl_search
+  depends_on:
+    - dag_name: main_summary
+      task_id: main_summary

sql/search_derived/search_clients_last_seen_v1/metadata.yaml

@@ -0,0 +1,11 @@
+friendly_name: Search Clients Last Seen
+description: >-
+  Captures history of activity of each search client in 28 day
+  windows for each submission date.
+owners:
+  - frank@mozilla.com
+labels:
+  schedule: daily
+scheduling:
+  dag_name: bqetl_search
+  depends_on_past: true