2020-10-30 03:24:55 +03:00
|
|
|
---
|
2020-06-03 20:49:11 +03:00
|
|
|
bqetl_error_aggregates:
|
|
|
|
|
schedule_interval: 3h
|
|
|
|
|
default_args:
|
2022-01-03 14:53:05 +03:00
|
|
|
owner: wkahngreene@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
email:
|
|
|
|
|
[
|
|
|
|
|
"telemetry-alerts@mozilla.com",
|
2022-01-03 14:53:05 +03:00
|
|
|
"wkahngreene@mozilla.com",
|
2020-10-30 03:24:55 +03:00
|
|
|
]
|
|
|
|
|
start_date: "2019-11-01"
|
2020-06-03 20:49:11 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 20m
|
|
|
|
|
depends_on_past: false
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-05-22 03:01:42 +03:00
|
|
|
|
2020-05-30 01:42:22 +03:00
|
|
|
bqetl_ssl_ratios:
|
2020-07-10 20:33:34 +03:00
|
|
|
schedule_interval: 0 2 * * *
|
2021-01-30 02:04:42 +03:00
|
|
|
description: The DAG schedules SSL ratios queries.
|
2020-05-30 01:42:22 +03:00
|
|
|
default_args:
|
2020-10-30 03:24:55 +03:00
|
|
|
owner: chutten@mozilla.com
|
|
|
|
|
start_date: "2019-07-20"
|
|
|
|
|
email: ["telemetry-alerts@mozilla.com", "chutten@mozilla.com"]
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2020-05-30 01:42:22 +03:00
|
|
|
|
2020-06-03 16:44:02 +03:00
|
|
|
bqetl_amo_stats:
|
2020-07-09 19:27:02 +03:00
|
|
|
schedule_interval: 0 3 * * *
|
2021-02-02 22:53:13 +03:00
|
|
|
# yamllint disable rule:line-length
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-02-02 22:53:13 +03:00
|
|
|
Add-on download and install statistics to power the
|
|
|
|
|
[addons.mozilla.org](https://addons.mozilla.org) (AMO) stats pages.
|
|
|
|
|
|
|
|
|
|
See the [post on the Add-Ons Blog](https://blog.mozilla.org/addons/2020/06/10/improvements-to-statistics-processing-on-amo/).
|
|
|
|
|
# yamllint enable rule:line-length
|
2020-06-03 16:44:02 +03:00
|
|
|
default_args:
|
2023-05-31 12:49:11 +03:00
|
|
|
owner: kik@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2020-06-01"
|
2023-05-31 12:49:11 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "kik@mozilla.com"]
|
2020-10-30 03:24:55 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-06-03 16:44:02 +03:00
|
|
|
|
2020-06-05 21:36:53 +03:00
|
|
|
bqetl_core:
|
2020-07-10 20:33:34 +03:00
|
|
|
schedule_interval: 0 2 * * *
|
2021-02-02 22:53:13 +03:00
|
|
|
description:
|
|
|
|
|
Tables derived from the legacy telemetry `core` ping sent by various
|
|
|
|
|
mobile applications.
|
2020-06-05 21:36:53 +03:00
|
|
|
default_args:
|
2023-05-30 20:05:40 +03:00
|
|
|
owner: ascholtz@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2019-07-25"
|
2023-05-30 20:05:40 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "ascholtz@mozilla.com"]
|
2020-06-05 21:36:53 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 5m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-06-05 21:36:53 +03:00
|
|
|
|
2020-06-05 22:41:31 +03:00
|
|
|
bqetl_nondesktop:
|
2020-07-10 22:35:45 +03:00
|
|
|
schedule_interval: 0 3 * * *
|
2020-06-05 22:41:31 +03:00
|
|
|
default_args:
|
2023-05-30 20:05:40 +03:00
|
|
|
owner: "ascholtz@mozilla.com"
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2019-07-25"
|
2022-01-03 14:53:05 +03:00
|
|
|
email: [
|
|
|
|
|
"telemetry-alerts@mozilla.com",
|
|
|
|
|
]
|
2020-06-05 22:41:31 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 5m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-06-05 22:41:31 +03:00
|
|
|
|
2020-06-08 23:04:50 +03:00
|
|
|
bqetl_mobile_search:
|
2020-07-10 22:35:45 +03:00
|
|
|
schedule_interval: 0 2 * * *
|
2020-06-08 23:04:50 +03:00
|
|
|
default_args:
|
2021-12-13 19:12:53 +03:00
|
|
|
owner: anicholson@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2019-07-25"
|
2021-12-13 19:12:53 +03:00
|
|
|
email:
|
|
|
|
|
- "telemetry-alerts@mozilla.com"
|
|
|
|
|
- "anicholson@mozilla.com"
|
|
|
|
|
- "akomar@mozilla.com"
|
2023-02-01 22:29:49 +03:00
|
|
|
- "cmorales@mozilla.com"
|
2020-06-08 23:04:50 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 5m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-06-08 23:04:50 +03:00
|
|
|
|
2020-07-14 14:58:11 +03:00
|
|
|
bqetl_fxa_events:
|
|
|
|
|
schedule_interval: 30 1 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-02-02 22:53:13 +03:00
|
|
|
Copies data from a Firefox Accounts (FxA) project. Those source tables
|
|
|
|
|
are populated via Cloud Logging (Stackdriver). We hash various fields
|
|
|
|
|
as part of the import.
|
|
|
|
|
|
|
|
|
|
The DAG also provides daily aggregations on top of the raw log data,
|
|
|
|
|
which eventually power high-level reporting about FxA usage.
|
2021-04-26 17:50:00 +03:00
|
|
|
|
|
|
|
|
Tasks here have occasionally failed due to incompatible schema changes
|
|
|
|
|
in the tables populated by Cloud Logging.
|
|
|
|
|
See https://github.com/mozilla/bigquery-etl/issues/1684 for an example
|
|
|
|
|
mitigation.
|
2020-07-14 14:58:11 +03:00
|
|
|
default_args:
|
2022-06-23 21:36:20 +03:00
|
|
|
owner: dthorn@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2019-03-01"
|
2022-06-23 21:36:20 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "dthorn@mozilla.com"]
|
2020-07-14 14:58:11 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 10m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-07-14 14:58:11 +03:00
|
|
|
|
2023-09-22 16:27:07 +03:00
|
|
|
bqetl_accounts_backend_external:
|
|
|
|
|
schedule_interval: 30 1 * * *
|
|
|
|
|
description: |
|
|
|
|
|
Copies data from Firefox Accounts (FxA) CloudSQL databases.
|
|
|
|
|
|
|
|
|
|
This DAG is under active development.
|
|
|
|
|
default_args:
|
|
|
|
|
owner: akomar@mozilla.com
|
|
|
|
|
start_date: "2023-09-19"
|
2023-10-25 10:50:25 +03:00
|
|
|
email: ["akomar@mozilla.com", "telemetry-alerts@mozilla.com"]
|
2023-09-22 16:27:07 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 10m
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
- repo/bigquery-etl
|
|
|
|
|
|
2021-07-21 23:39:01 +03:00
|
|
|
bqetl_subplat:
|
2021-02-02 17:07:16 +03:00
|
|
|
schedule_interval: 45 1 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-07-21 23:39:01 +03:00
|
|
|
Daily imports for Subscription Platform data from Stripe and the Mozilla VPN
|
|
|
|
|
operational DB as well as derived tables based on that data.
|
2021-03-29 16:15:24 +03:00
|
|
|
|
2021-07-21 23:39:01 +03:00
|
|
|
Depends on `bqetl_fxa_events`, so is scheduled to run a bit after that.
|
2022-06-01 16:54:19 +03:00
|
|
|
|
|
|
|
|
Stripe data retrieved by stripe_external__itemized_payout_reconciliation__v5
|
|
|
|
|
task has highly viariable availability timing, so it is possible for it to
|
|
|
|
|
fail with the following type of error:
|
|
|
|
|
`Error: Request req_OTssZ0Zv1cEmmm: Data for the report type
|
|
|
|
|
payout_reconciliation.itemized.5 is only available through
|
|
|
|
|
2022-05-08 12:00:00 UTC; you requested `interval_end`
|
|
|
|
|
= 2022-05-09 00:00:00 UTC.`
|
|
|
|
|
In such cases the failure is expected, the task will continue to retry every
|
|
|
|
|
30 minutes until the data becomes available. If failure observed looks
|
|
|
|
|
different then it should be reported using the Airflow triage process.
|
|
|
|
|
|
2021-02-02 17:07:16 +03:00
|
|
|
default_args:
|
2022-06-30 19:07:33 +03:00
|
|
|
owner: srose@mozilla.com
|
2021-07-21 23:39:01 +03:00
|
|
|
start_date: "2021-07-20"
|
2022-06-30 19:07:33 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "srose@mozilla.com"]
|
2021-02-02 17:07:16 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-02-02 17:07:16 +03:00
|
|
|
|
2021-04-22 23:48:59 +03:00
|
|
|
bqetl_mozilla_vpn_site_metrics:
|
2021-04-27 22:19:23 +03:00
|
|
|
schedule_interval: 0 15 * * *
|
2021-04-22 23:48:59 +03:00
|
|
|
description: |
|
|
|
|
|
Daily extracts from the Google Analytics tables for Mozilla VPN as well as
|
|
|
|
|
derived tables based on that data.
|
|
|
|
|
|
|
|
|
|
Depends on Google Analytics exports, which have highly variable timing, so
|
2021-04-27 22:19:23 +03:00
|
|
|
queries depend on site_metrics_empty_check_v1, which retries every 30
|
|
|
|
|
minutes to wait for data to be available.
|
2021-04-22 23:48:59 +03:00
|
|
|
default_args:
|
2022-06-30 19:07:33 +03:00
|
|
|
owner: srose@mozilla.com
|
2021-04-22 23:48:59 +03:00
|
|
|
start_date: "2021-04-22"
|
2022-06-30 19:07:33 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "srose@mozilla.com"]
|
2021-04-22 23:48:59 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2021-04-22 23:48:59 +03:00
|
|
|
|
2020-06-09 01:39:19 +03:00
|
|
|
bqetl_gud:
|
2020-07-09 19:27:02 +03:00
|
|
|
schedule_interval: 0 3 * * *
|
2021-03-30 02:23:50 +03:00
|
|
|
description: Optimized tables that power the
|
2021-02-02 22:53:13 +03:00
|
|
|
[Mozilla Growth and Usage Dashboard](https://gud.telemetry.mozilla.org).
|
2020-06-09 01:39:19 +03:00
|
|
|
default_args:
|
|
|
|
|
owner: jklukas@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2019-07-25"
|
|
|
|
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
2020-06-09 01:39:19 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 5m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-06-10 23:18:48 +03:00
|
|
|
|
|
|
|
|
bqetl_messaging_system:
|
2020-07-10 22:35:45 +03:00
|
|
|
schedule_interval: 0 2 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-02-02 22:53:13 +03:00
|
|
|
Daily aggregations on top of pings sent for the `messaging_system`
|
|
|
|
|
namespace by desktop Firefox.
|
2020-06-10 23:18:48 +03:00
|
|
|
default_args:
|
|
|
|
|
owner: najiang@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2019-07-25"
|
|
|
|
|
email: ["telemetry-alerts@mozilla.com", "najiang@mozilla.com"]
|
2020-06-10 23:18:48 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 5m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2020-06-12 02:19:20 +03:00
|
|
|
|
|
|
|
|
bqetl_activity_stream:
|
2020-07-10 22:35:45 +03:00
|
|
|
schedule_interval: 0 2 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-02-02 22:53:13 +03:00
|
|
|
Daily aggregations on top of pings sent for the `activity_stream`
|
|
|
|
|
namespace by desktop Firefox. These are largely related to activity
|
|
|
|
|
on the newtab page and engagement with Pocket content.
|
2020-06-12 02:19:20 +03:00
|
|
|
default_args:
|
2023-05-30 20:05:40 +03:00
|
|
|
owner: anicholson@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2019-07-25"
|
2023-05-30 20:05:40 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "anicholson@mozilla.com"]
|
2020-06-12 02:19:20 +03:00
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 5m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2020-06-12 01:54:40 +03:00
|
|
|
|
|
|
|
|
bqetl_search:
|
2020-07-09 19:27:02 +03:00
|
|
|
schedule_interval: 0 3 * * *
|
2020-06-12 01:54:40 +03:00
|
|
|
default_args:
|
2021-12-13 19:12:53 +03:00
|
|
|
owner: anicholson@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2018-11-27"
|
|
|
|
|
email:
|
2021-12-13 19:12:53 +03:00
|
|
|
- "telemetry-alerts@mozilla.com"
|
|
|
|
|
- "anicholson@mozilla.com"
|
|
|
|
|
- "akomar@mozilla.com"
|
2023-02-01 22:29:49 +03:00
|
|
|
- "cmorales@mozilla.com"
|
2020-06-12 01:54:40 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-06-13 00:14:57 +03:00
|
|
|
|
|
|
|
|
bqetl_addons:
|
2021-03-29 16:15:24 +03:00
|
|
|
schedule_interval: 0 4 * * *
|
|
|
|
|
description: |
|
|
|
|
|
Daily rollups of addon data from `main` pings.
|
|
|
|
|
|
|
|
|
|
Depends on `bqetl_search`, so is scheduled after that DAG.
|
2020-06-13 00:14:57 +03:00
|
|
|
default_args:
|
2023-05-31 12:49:11 +03:00
|
|
|
owner: kik@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2018-11-27"
|
2023-05-31 12:49:11 +03:00
|
|
|
email:
|
|
|
|
|
- "telemetry-alerts@mozilla.com"
|
|
|
|
|
- "kik@mozilla.com"
|
2020-06-13 00:14:57 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2020-06-23 19:22:41 +03:00
|
|
|
|
|
|
|
|
bqetl_devtools:
|
2020-07-09 19:27:02 +03:00
|
|
|
schedule_interval: 0 3 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-02-02 22:53:13 +03:00
|
|
|
Summarizes usage of the Dev Tools component of desktop Firefox.
|
2020-06-23 19:22:41 +03:00
|
|
|
default_args:
|
2023-05-30 20:05:40 +03:00
|
|
|
owner: ascholtz@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2018-11-27"
|
2023-05-30 20:05:40 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "ascholtz@mozilla.com"]
|
2020-06-23 19:22:41 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2020-06-23 19:22:41 +03:00
|
|
|
|
2020-06-24 23:40:24 +03:00
|
|
|
bqetl_main_summary:
|
2020-07-10 22:35:45 +03:00
|
|
|
schedule_interval: 0 2 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-02-02 22:53:13 +03:00
|
|
|
General-purpose derived tables for analyzing usage of desktop Firefox.
|
|
|
|
|
This is one of our highest-impact DAGs and should be handled carefully.
|
2020-06-24 23:40:24 +03:00
|
|
|
default_args:
|
|
|
|
|
owner: dthorn@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2018-11-27"
|
|
|
|
|
email:
|
|
|
|
|
[
|
|
|
|
|
"telemetry-alerts@mozilla.com",
|
|
|
|
|
"dthorn@mozilla.com",
|
|
|
|
|
]
|
2020-06-24 23:40:24 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-06-24 23:40:24 +03:00
|
|
|
|
2020-06-25 22:44:37 +03:00
|
|
|
bqetl_experiments_daily:
|
2020-07-09 19:27:02 +03:00
|
|
|
schedule_interval: 0 3 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-01-30 02:04:42 +03:00
|
|
|
The DAG schedules queries that query experimentation related
|
|
|
|
|
metrics (enrollments, search, ...) from stable tables to finalize
|
|
|
|
|
numbers of experiment monitoring datasets for a specific date.
|
2020-06-25 00:42:55 +03:00
|
|
|
default_args:
|
2021-01-14 00:53:32 +03:00
|
|
|
owner: ascholtz@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2018-11-27"
|
2021-01-14 00:53:32 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "ascholtz@mozilla.com"]
|
2020-06-25 00:42:55 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-12-08 19:12:11 +03:00
|
|
|
|
2020-05-30 01:42:22 +03:00
|
|
|
# DAG for exporting query data marked as public to GCS
|
2020-10-30 03:24:55 +03:00
|
|
|
# queries should not be explicitly assigned to this DAG (done automatically)
|
2020-06-01 22:40:03 +03:00
|
|
|
bqetl_public_data_json:
|
2021-03-29 16:15:24 +03:00
|
|
|
schedule_interval: 0 5 * * *
|
|
|
|
|
description: |
|
|
|
|
|
Daily exports of query data marked as public to GCS.
|
|
|
|
|
|
|
|
|
|
Depends on the results of several upstream DAGs, the latest of which
|
|
|
|
|
runs at 04:00 UTC.
|
2020-05-30 01:42:22 +03:00
|
|
|
default_args:
|
|
|
|
|
owner: ascholtz@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2020-04-14"
|
|
|
|
|
email: ["telemetry-alerts@mozilla.com", "ascholtz@mozilla.com"]
|
2020-05-30 01:42:22 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2020-07-01 07:44:40 +03:00
|
|
|
|
|
|
|
|
bqetl_internet_outages:
|
2022-02-28 15:01:20 +03:00
|
|
|
schedule_interval: 0 7 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-01-30 02:25:05 +03:00
|
|
|
DAG for building the internet outages datasets.
|
|
|
|
|
See [bug 1640204](https://bugzilla.mozilla.org/show_bug.cgi?id=1640204).
|
2020-07-01 07:44:40 +03:00
|
|
|
default_args:
|
|
|
|
|
owner: aplacitelli@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2020-01-01"
|
2021-08-31 21:38:22 +03:00
|
|
|
email: ["aplacitelli@mozilla.com"]
|
2020-07-01 07:44:40 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2020-07-13 21:11:06 +03:00
|
|
|
|
|
|
|
|
bqetl_deletion_request_volume:
|
|
|
|
|
schedule_interval: 0 1 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
owner: dthorn@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2020-06-29"
|
|
|
|
|
email: ["telemetry-alerts@mozilla.com", "dthorn@mozilla.com"]
|
2020-07-13 21:11:06 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2020-09-11 01:50:45 +03:00
|
|
|
|
2020-09-23 22:28:25 +03:00
|
|
|
bqetl_fenix_event_rollup:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
2020-09-11 01:50:45 +03:00
|
|
|
default_args:
|
2021-09-16 18:04:26 +03:00
|
|
|
owner: wlachance@mozilla.com
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2020-09-09"
|
2021-09-16 18:04:26 +03:00
|
|
|
email: ["wlachance@mozilla.com"]
|
2020-09-23 22:28:25 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-09-23 22:28:25 +03:00
|
|
|
|
2020-10-17 00:37:30 +03:00
|
|
|
bqetl_org_mozilla_fenix_derived:
|
2021-03-29 16:15:24 +03:00
|
|
|
schedule_interval: 0 2 * * *
|
2020-10-17 00:37:30 +03:00
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
2020-10-30 03:24:55 +03:00
|
|
|
- amiyaguchi@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
2020-10-17 00:37:30 +03:00
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: amiyaguchi@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2020-10-30 03:24:55 +03:00
|
|
|
start_date: "2020-10-18"
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-10-29 00:40:45 +03:00
|
|
|
|
2022-12-01 19:21:54 +03:00
|
|
|
bqetl_org_mozilla_firefox_derived:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- frank@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: frank@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2022-11-30"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
|
2023-02-23 01:51:06 +03:00
|
|
|
bqetl_org_mozilla_focus_derived:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- dthorn@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: dthorn@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2023-02-22"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
|
2020-10-29 00:40:45 +03:00
|
|
|
bqetl_google_analytics_derived:
|
2020-11-04 22:47:32 +03:00
|
|
|
schedule_interval: 0 23 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
|
|
|
|
Daily aggregations of data exported from Google Analytics.
|
|
|
|
|
|
|
|
|
|
The GA export runs at 15:00 UTC, so there's an effective 2-day delay
|
|
|
|
|
for user activity to appear in these tables.
|
2020-10-29 00:40:45 +03:00
|
|
|
default_args:
|
2021-09-01 22:51:29 +03:00
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
email:
|
|
|
|
|
- ascholtz@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
2021-03-30 02:23:50 +03:00
|
|
|
start_date: "2020-10-31"
|
2020-10-29 00:40:45 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-11-10 23:44:03 +03:00
|
|
|
|
|
|
|
|
bqetl_monitoring:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-01-30 02:25:05 +03:00
|
|
|
This DAG schedules queries and scripts for populating datasets
|
|
|
|
|
used for monitoring of the data platform.
|
2020-11-10 23:44:03 +03:00
|
|
|
default_args:
|
|
|
|
|
owner: ascholtz@mozilla.com
|
2021-03-30 02:23:50 +03:00
|
|
|
email: ["ascholtz@mozilla.com"]
|
|
|
|
|
start_date: "2018-10-30"
|
2020-11-12 17:10:07 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-11-12 17:10:07 +03:00
|
|
|
|
2022-10-12 12:57:24 +03:00
|
|
|
bqetl_monitoring_airflow:
|
2023-09-08 18:53:07 +03:00
|
|
|
schedule_interval: 0 10 * * *
|
2022-10-12 12:57:24 +03:00
|
|
|
description: |
|
|
|
|
|
This DAG schedules queries and scripts for populating datasets
|
|
|
|
|
used for monitoring of Airflow DAGs.
|
|
|
|
|
default_args:
|
2023-08-14 17:34:12 +03:00
|
|
|
owner: kik@mozilla.com
|
|
|
|
|
email: ["kik@mozilla.com"]
|
2022-10-12 12:57:24 +03:00
|
|
|
start_date: "2022-09-01"
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
|
|
|
|
|
2020-11-12 17:10:07 +03:00
|
|
|
bqetl_event_rollup:
|
|
|
|
|
schedule_interval: 0 3 * * *
|
2023-01-30 20:35:12 +03:00
|
|
|
description: |
|
|
|
|
|
Desktop tables (`telemetry_derived.events_daily_v1` and upstream) are deprecated and paused
|
|
|
|
|
(have their scheduling metadata commented out) per https://bugzilla.mozilla.org/show_bug.cgi?id=1805722#c10
|
2020-11-12 17:10:07 +03:00
|
|
|
default_args:
|
2021-09-16 18:04:26 +03:00
|
|
|
owner: wlachance@mozilla.com
|
2020-11-12 17:10:07 +03:00
|
|
|
start_date: "2020-11-03"
|
2021-09-16 18:04:26 +03:00
|
|
|
email: ["wlachance@mozilla.com"]
|
2020-11-10 23:44:03 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2020-11-30 23:34:55 +03:00
|
|
|
|
2021-07-06 23:31:29 +03:00
|
|
|
bqetl_iprospect:
|
|
|
|
|
schedule_interval: 0 4 * * *
|
|
|
|
|
description: |
|
|
|
|
|
This DAG imports iProspect data from moz-fx-data-marketing-prod-iprospect.
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
default_args:
|
|
|
|
|
owner: ascholtz@mozilla.com
|
2021-11-03 22:32:58 +03:00
|
|
|
email:
|
|
|
|
|
[
|
|
|
|
|
"ascholtz@mozilla.com",
|
|
|
|
|
"echo@mozilla.com",
|
|
|
|
|
"shong@mozilla.com"
|
|
|
|
|
]
|
2021-07-06 23:31:29 +03:00
|
|
|
start_date: "2021-04-19"
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-07-06 23:31:29 +03:00
|
|
|
|
2020-12-15 02:27:14 +03:00
|
|
|
bqetl_search_dashboard:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
2021-03-10 01:25:01 +03:00
|
|
|
- telemetry-alerts@mozilla.com
|
2021-09-01 22:51:29 +03:00
|
|
|
- akomar@mozilla.com
|
2020-12-15 02:27:14 +03:00
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
2021-09-01 22:51:29 +03:00
|
|
|
owner: akomar@mozilla.com
|
2020-12-15 02:27:14 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-03-30 02:23:50 +03:00
|
|
|
start_date: "2020-12-14"
|
2023-11-06 19:55:12 +03:00
|
|
|
schedule_interval: 30 4 * * *
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2020-12-15 02:27:14 +03:00
|
|
|
|
2020-12-11 23:05:50 +03:00
|
|
|
bqetl_desktop_platform:
|
|
|
|
|
schedule_interval: 0 3 * * *
|
|
|
|
|
default_args:
|
2023-05-30 20:05:40 +03:00
|
|
|
owner: ascholtz@mozilla.com
|
2020-12-15 20:41:30 +03:00
|
|
|
start_date: "2018-11-01"
|
2020-12-11 23:05:50 +03:00
|
|
|
email:
|
|
|
|
|
[
|
|
|
|
|
"telemetry-alerts@mozilla.com",
|
2023-05-30 20:05:40 +03:00
|
|
|
"ascholtz@mozilla.com",
|
2020-12-11 23:05:50 +03:00
|
|
|
"yzenevich@mozilla.com",
|
|
|
|
|
]
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2021-02-11 17:43:34 +03:00
|
|
|
|
|
|
|
|
bqetl_internal_tooling:
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-02-11 17:43:34 +03:00
|
|
|
This DAG schedules queries for populating queries related to Mozilla's
|
|
|
|
|
internal developer tooling (e.g. mozregression).
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
2021-03-10 01:25:01 +03:00
|
|
|
- wlachance@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
2021-02-11 17:43:34 +03:00
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: wlachance@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-03-30 02:23:50 +03:00
|
|
|
start_date: "2020-06-01"
|
2021-02-11 17:43:34 +03:00
|
|
|
schedule_interval: 0 4 * * *
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_3
|
2021-03-10 22:27:32 +03:00
|
|
|
|
|
|
|
|
bqetl_release_criteria:
|
|
|
|
|
schedule_interval: daily
|
|
|
|
|
default_args:
|
|
|
|
|
owner: perf-pmo@mozilla.com
|
|
|
|
|
start_date: "2020-12-03"
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- dthorn@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-03-12 00:55:21 +03:00
|
|
|
|
|
|
|
|
bqetl_pocket:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
2023-05-31 12:49:11 +03:00
|
|
|
- kik@mozilla.com
|
2021-03-12 00:55:21 +03:00
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
2023-05-31 12:49:11 +03:00
|
|
|
owner: kik@mozilla.com
|
2023-03-04 02:00:19 +03:00
|
|
|
# Retry more than normal because the files from Pocket may not always be available on time.
|
|
|
|
|
retries: 10
|
|
|
|
|
retry_delay: 60m
|
2021-03-30 02:23:50 +03:00
|
|
|
start_date: "2021-03-10"
|
2021-03-12 00:55:21 +03:00
|
|
|
description: |
|
|
|
|
|
Import of data from Pocket's Snowflake warehouse.
|
|
|
|
|
|
|
|
|
|
Originally created for [Bug 1695336](
|
|
|
|
|
https://bugzilla.mozilla.org/show_bug.cgi?id=1695336).
|
|
|
|
|
schedule_interval: 0 12 * * *
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2021-03-17 21:03:54 +03:00
|
|
|
|
2021-03-16 18:49:17 +03:00
|
|
|
bqetl_desktop_funnel:
|
2021-03-29 16:15:24 +03:00
|
|
|
description: |
|
2021-03-16 18:49:17 +03:00
|
|
|
This DAG schedules desktop funnel queries used to power the
|
|
|
|
|
[Numbers that Matter dashboard](https://protosaur.dev/numbers-that-matter/)
|
2021-03-19 22:20:05 +03:00
|
|
|
schedule_interval: 0 4 * * *
|
2021-03-16 18:49:17 +03:00
|
|
|
default_args:
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
start_date: "2021-01-01"
|
|
|
|
|
email:
|
|
|
|
|
[
|
|
|
|
|
"telemetry-alerts@mozilla.com",
|
|
|
|
|
"ascholtz@mozilla.com",
|
|
|
|
|
]
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-03-30 02:23:50 +03:00
|
|
|
|
|
|
|
|
bqetl_firefox_ios:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
2023-05-31 12:49:11 +03:00
|
|
|
- kik@mozilla.com
|
2023-12-04 18:48:56 +03:00
|
|
|
- frank@mozilla.com
|
2021-03-30 02:23:50 +03:00
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
2023-05-31 12:49:11 +03:00
|
|
|
owner: kik@mozilla.com
|
2021-03-30 02:23:50 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2021-03-18"
|
|
|
|
|
description: Schedule daily ios firefox ETL
|
|
|
|
|
schedule_interval: 0 4 * * *
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-04-14 22:54:28 +03:00
|
|
|
|
|
|
|
|
bqetl_releases:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- ascholtz@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2021-04-14"
|
|
|
|
|
description: |
|
|
|
|
|
Schedule release data import from https://product-details.mozilla.org/1.0
|
2021-04-15 19:28:14 +03:00
|
|
|
|
|
|
|
|
For more context, see
|
|
|
|
|
https://wiki.mozilla.org/Release_Management/Product_details
|
2021-04-14 22:54:28 +03:00
|
|
|
schedule_interval: 0 4 * * *
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2021-05-05 21:32:20 +03:00
|
|
|
|
|
|
|
|
bqetl_ctxsvc_derived:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
2022-07-21 23:18:23 +03:00
|
|
|
- ctroy@mozilla.com
|
|
|
|
|
- wstuckey@mozilla.com
|
2021-05-05 21:32:20 +03:00
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
2022-07-21 23:18:23 +03:00
|
|
|
owner: ctroy@mozilla.com
|
2021-05-05 21:32:20 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2021-05-01'
|
|
|
|
|
description: Contextual services derived tables
|
|
|
|
|
schedule_interval: 0 3 * * *
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2021-06-15 01:16:38 +03:00
|
|
|
|
2021-09-22 16:01:24 +03:00
|
|
|
bqetl_search_terms_daily:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
2022-07-21 23:18:23 +03:00
|
|
|
- ctroy@mozilla.com
|
|
|
|
|
- wstuckey@mozilla.com
|
2021-09-22 16:01:24 +03:00
|
|
|
- rburwei@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
2022-07-21 23:18:23 +03:00
|
|
|
owner: ctroy@mozilla.com
|
2021-09-22 16:01:24 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2021-09-20'
|
|
|
|
|
description: |
|
|
|
|
|
Derived tables on top of search terms data.
|
2022-01-10 17:24:49 +03:00
|
|
|
|
2022-07-21 23:18:23 +03:00
|
|
|
Note that the tasks for populating `suggest_impression_sanitized_v*` are
|
2022-01-10 17:24:49 +03:00
|
|
|
particularly important because the source unsanitized dataset has only
|
|
|
|
|
a 2-day retention period, so errors fairly quickly become unrecoverable
|
|
|
|
|
and can impact reporting to partners. If this task errors out, it could
|
|
|
|
|
indicate trouble with an upstream task that runs in a restricted project
|
2022-07-21 23:18:23 +03:00
|
|
|
outside of Airflow. Contact `ctroy`, `wstuckey`, `whd`, and `jbuck`.
|
2021-09-22 16:01:24 +03:00
|
|
|
schedule_interval: 0 3 * * *
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2022-01-10 17:24:49 +03:00
|
|
|
- impact/tier_1
|
2021-09-22 16:01:24 +03:00
|
|
|
|
2021-07-22 01:40:18 +03:00
|
|
|
bqetl_experimenter_experiments_import:
|
|
|
|
|
schedule_interval: "*/10 * * * *"
|
|
|
|
|
description: |
|
|
|
|
|
Imports experiments from the Experimenter V4 and V6 API.
|
|
|
|
|
|
|
|
|
|
Imported experiment data is used for experiment monitoring in
|
|
|
|
|
[Grafana](https://grafana.telemetry.mozilla.org/d/XspgvdxZz/experiment-enrollment).
|
|
|
|
|
default_args:
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
start_date: "2020-10-09"
|
|
|
|
|
retries: 0
|
|
|
|
|
email:
|
|
|
|
|
- ascholtz@mozilla.com
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2021-08-03 19:18:50 +03:00
|
|
|
|
|
|
|
|
bqetl_feature_usage:
|
|
|
|
|
schedule_interval: 0 5 * * *
|
|
|
|
|
description: |
|
|
|
|
|
Daily aggregation of browser features usages from `main` pings,
|
|
|
|
|
`event` pings and addon data.
|
|
|
|
|
|
|
|
|
|
Depends on `bqetl_addons` and `bqetl_main_summary`, so is scheduled after.
|
|
|
|
|
default_args:
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
start_date: "2021-01-01"
|
|
|
|
|
email:
|
|
|
|
|
- "telemetry-alerts@mozilla.com"
|
|
|
|
|
- "ascholtz@mozilla.com"
|
|
|
|
|
- "loines@mozilla.com"
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-09-02 01:43:51 +03:00
|
|
|
|
|
|
|
|
bqetl_urlbar:
|
|
|
|
|
schedule_interval: 0 3 * * *
|
|
|
|
|
description: |
|
|
|
|
|
Daily aggregation of metrics related to urlbar usage.
|
|
|
|
|
default_args:
|
2021-12-13 19:12:53 +03:00
|
|
|
owner: anicholson@mozilla.com
|
2021-09-02 01:43:51 +03:00
|
|
|
start_date: "2021-08-01"
|
|
|
|
|
email:
|
|
|
|
|
- "telemetry-alerts@mozilla.com"
|
2021-12-13 19:12:53 +03:00
|
|
|
- "anicholson@mozilla.com"
|
2021-09-02 01:43:51 +03:00
|
|
|
- "akomar@mozilla.com"
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-15 16:21:56 +03:00
|
|
|
- impact/tier_2
|
2021-10-13 01:59:24 +03:00
|
|
|
|
|
|
|
|
bqetl_unified:
|
|
|
|
|
schedule_interval: 0 3 * * *
|
|
|
|
|
description: |
|
|
|
|
|
Schedule queries that unify metrics across all products.
|
|
|
|
|
default_args:
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
start_date: "2021-10-12"
|
|
|
|
|
email:
|
|
|
|
|
- "telemetry-alerts@mozilla.com"
|
|
|
|
|
- "ascholtz@mozilla.com"
|
|
|
|
|
- "loines@mozilla.com"
|
2022-05-12 17:23:50 +03:00
|
|
|
- "lvargas@mozilla.com"
|
2021-10-13 01:59:24 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
2021-12-08 19:12:11 +03:00
|
|
|
tags:
|
2021-12-14 14:46:50 +03:00
|
|
|
- impact/tier_1
|
2021-12-16 21:14:51 +03:00
|
|
|
|
|
|
|
|
bqetl_regrets_reporter_summary:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
2023-08-14 17:34:12 +03:00
|
|
|
- kik@mozilla.com
|
2021-12-16 21:14:51 +03:00
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
2023-08-14 17:34:12 +03:00
|
|
|
owner: kik@mozilla.com
|
2021-12-16 21:14:51 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2021-12-12'
|
|
|
|
|
description: Measure usage of the regrets reporter addon
|
|
|
|
|
schedule_interval: 0 4 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
2022-03-26 00:56:11 +03:00
|
|
|
|
|
|
|
|
bqetl_cjms_nonprod:
|
2022-04-07 00:31:21 +03:00
|
|
|
schedule_interval: 0 * * * *
|
2022-03-26 00:56:11 +03:00
|
|
|
description: |
|
|
|
|
|
Hourly ETL for cjms nonprod.
|
|
|
|
|
default_args:
|
2022-06-30 19:07:33 +03:00
|
|
|
owner: srose@mozilla.com
|
2022-03-26 00:56:11 +03:00
|
|
|
start_date: "2022-03-24"
|
2022-06-30 19:07:33 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "srose@mozilla.com"]
|
2022-03-26 00:56:11 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 5m
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
2022-05-04 12:40:48 +03:00
|
|
|
|
|
|
|
|
bqetl_acoustic_contact_export:
|
2022-06-02 16:18:41 +03:00
|
|
|
schedule_interval: 0 9 * * *
|
2022-05-04 12:40:48 +03:00
|
|
|
description: |
|
|
|
|
|
Processing data loaded by
|
|
|
|
|
fivetran_acoustic_contact_export
|
|
|
|
|
DAG to clean up the data loaded from Acoustic.
|
|
|
|
|
default_args:
|
2023-08-14 17:34:12 +03:00
|
|
|
owner: kik@mozilla.com
|
2022-05-04 12:40:48 +03:00
|
|
|
start_date: "2021-03-01"
|
2023-08-14 17:34:12 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "kik@mozilla.com"]
|
2023-07-31 19:12:09 +03:00
|
|
|
retries: 1
|
2022-05-04 12:40:48 +03:00
|
|
|
retry_delay: 5m
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
|
|
|
|
|
bqetl_acoustic_raw_recipient_export:
|
2022-06-02 16:18:41 +03:00
|
|
|
schedule_interval: 0 9 * * *
|
2022-05-04 12:40:48 +03:00
|
|
|
description: |
|
|
|
|
|
Processing data loaded by
|
|
|
|
|
fivetran_acoustic_raw_recipient_export
|
|
|
|
|
DAG to clean up the data loaded from Acoustic.
|
|
|
|
|
default_args:
|
2023-08-14 17:34:12 +03:00
|
|
|
owner: kik@mozilla.com
|
2022-05-04 12:40:48 +03:00
|
|
|
start_date: "2022-03-01"
|
2023-08-14 17:34:12 +03:00
|
|
|
email: ["telemetry-alerts@mozilla.com", "kik@mozilla.com"]
|
2023-07-31 19:12:09 +03:00
|
|
|
retries: 1
|
2022-05-04 12:40:48 +03:00
|
|
|
retry_delay: 5m
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
2022-05-12 17:23:50 +03:00
|
|
|
|
|
|
|
|
bqetl_analytics_aggregations:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- "telemetry-alerts@mozilla.com"
|
|
|
|
|
- "lvargas@mozilla.com"
|
|
|
|
|
- "gkaberere@mozilla.com"
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: lvargas@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2022-05-12'
|
|
|
|
|
description: Scheduler to populate the aggregations required for
|
|
|
|
|
analytics engineering and reports optimization.
|
|
|
|
|
It provides data to build growth, search and usage metrics, as well
|
|
|
|
|
as acquisition and retention KPIs, in a model that facilitates
|
|
|
|
|
reporting in Looker.
|
2023-11-06 19:55:12 +03:00
|
|
|
schedule_interval: 15 4 * * *
|
2022-05-12 17:23:50 +03:00
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
2022-06-02 00:00:29 +03:00
|
|
|
|
|
|
|
|
bqetl_fog_decision_support:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- pmcmanis@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: pmcmanis@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2022-05-25'
|
|
|
|
|
description: This DAG schedules queries for calculating FOG decision
|
|
|
|
|
support metrics
|
|
|
|
|
schedule_interval: 0 4 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
- repo/bigquery-etl
|
2022-07-18 23:32:03 +03:00
|
|
|
|
|
|
|
|
bqetl_newtab:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- anicholson@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: anicholson@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2022-07-01'
|
|
|
|
|
description: Schedules newtab related queries.
|
|
|
|
|
schedule_interval: daily
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
2022-08-18 14:47:22 +03:00
|
|
|
|
|
|
|
|
bqetl_desktop_mobile_search_monthly:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- akommasani@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: akommasani@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2019-01-01'
|
|
|
|
|
description: Generate mnthly client data from daily search table
|
|
|
|
|
schedule_interval: "0 5 2 * *"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
2022-10-20 01:13:43 +03:00
|
|
|
|
|
|
|
|
bqetl_domain_meta:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- wstuckey@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: wstuckey@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2022-10-13'
|
|
|
|
|
description: Domain metadata
|
|
|
|
|
schedule_interval: monthly
|
|
|
|
|
tags:
|
2023-04-12 18:28:19 +03:00
|
|
|
- impact/tier_3
|
|
|
|
|
- triage/no_triage
|
2022-10-20 01:13:43 +03:00
|
|
|
- repo/bigquery-etl
|
2022-10-31 17:50:11 +03:00
|
|
|
|
|
|
|
|
bqetl_sponsored_tiles_clients_daily:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- skahmann@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: skahmann@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2022-09-13'
|
|
|
|
|
description: daily run of sponsored tiles related fields
|
|
|
|
|
schedule_interval: 0 4 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
- repo/bigquery-etl
|
2022-12-01 03:08:17 +03:00
|
|
|
|
|
|
|
|
bqetl_mobile_activation:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- vsabino@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: vsabino@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2021-01-01'
|
|
|
|
|
description: Queries related to the mobile activation metric used by Marketing
|
|
|
|
|
schedule_interval: daily
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
Create table, view, DAG for Firefox Android Clients. (#3379)
* Create table, view, DAG for Firefox Android Clients.
* Normalization of adjust_network and install_source in the view, typo. Move dataset from telemetry_derived to fenix_derived.
* DENG-178_ Update logic to use fenix.first_session instead of org_mozilla_firefox, consider pings received after client's first seen date. Use first_run_date as datetime for comparison with metrics datetimes. Remove condition of metrics ping without adjust data to differentiate when a ping is not received and when is empty. Health check date types. Update clustering as adjust data is more likely to be null. Find deterministic values for fisr_session ping data. Remove uniqueness validation of clients in different metrics pings. Create UDF for the logic of finding the first adjust (value,datetime) pair.
* DENG-178_ Refactor query for readibility and consistency with existing datasets.
* Update clustering and description in metadata. Collect only `first seen` clients & metrics ping data for channel release. Update LEAST for COALESCE to avoid NULL returned. Collect core dimensions from `baseline_clients_first_seen` for data completeness when first_session ping is not reported.
* Delete schema file.
* Reducing duplicated logic, readability. Adding sample_id to the table and the clustering.
* Compare to find first value also for first_seen_date,submission_date, first_run_date, first_reported_country, first_reported_isp and channel.
* DAG partition parameter NULL. FUll outer join in init to collect ping's data for not yet first_seen clients.
Co-authored-by: Lucia Vargas <lvargas@mozilla.com>
2022-12-13 23:21:10 +03:00
|
|
|
|
|
|
|
|
bqetl_analytics_tables:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- lvargas@mozilla.com
|
|
|
|
|
- gkaberere@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: lvargas@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2022-12-01'
|
|
|
|
|
description: Scheduled queries for analytics tables.
|
|
|
|
|
engineering.
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
2023-01-05 18:15:16 +03:00
|
|
|
bqetl_fivetran_google_ads:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- frank@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: frank@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-01-01'
|
|
|
|
|
description: Queries for Google Ads data
|
|
|
|
|
coming from Fivetran. Fivetran
|
|
|
|
|
updates these tables every hour.
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
|
|
|
|
- repo/bigquery-etl
|
2023-01-12 00:58:10 +03:00
|
|
|
bqetl_campaign_cost_breakdowns:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- ctroy@mozilla.com
|
|
|
|
|
- frank@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: ctroy@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2021-09-20'
|
|
|
|
|
description: |
|
|
|
|
|
Derived tables on top of fenix installation and DOU metrics,
|
|
|
|
|
as well as Google ads campaign data.
|
|
|
|
|
schedule_interval: 0 3 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
|
|
|
|
- repo/bigquery-etl
|
2023-01-31 18:27:29 +03:00
|
|
|
bqetl_fivetran_costs:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- lschiestl@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: lschiestl@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-01-18'
|
|
|
|
|
description: |
|
|
|
|
|
Derived tables for analyzing the Fivetran Costs. Data coming from Fivetran.
|
|
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: 0 5 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
2023-02-23 12:38:12 +03:00
|
|
|
|
|
|
|
|
bqetl_mdn_yari:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
2023-03-27 22:02:25 +03:00
|
|
|
- mdn-infra@mozilla.com
|
2023-02-23 12:38:12 +03:00
|
|
|
- fmerz@mozilla.com
|
2023-08-14 17:34:12 +03:00
|
|
|
- kik@mozilla.com
|
2023-02-23 12:38:12 +03:00
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: false
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: fmerz@mozilla.com
|
|
|
|
|
retries: 1
|
|
|
|
|
retry_delay: 5m
|
|
|
|
|
start_date: '2023-02-01'
|
|
|
|
|
description: |
|
|
|
|
|
Monthly data exports of MDN 'Popularities'. This aggregates and counts total
|
|
|
|
|
page visits and normalizes them agains the max.
|
|
|
|
|
schedule_interval: 0 0 1 * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
- triage/record_only
|
2023-04-06 01:24:09 +03:00
|
|
|
|
|
|
|
|
bqetl_status_check:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: false
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
retries: 0
|
|
|
|
|
start_date: '2023-04-01'
|
|
|
|
|
description: |
|
|
|
|
|
This DAG checks if bigquery-etl is working properly. Dummy ETL tasks are executed to detect
|
|
|
|
|
breakages as soon as possible.
|
|
|
|
|
|
|
|
|
|
*Triage notes*
|
|
|
|
|
|
|
|
|
|
None of these tasks should fail. If they do it is very likely that other/all ETL tasks will
|
|
|
|
|
subsequently fail as well. Any failures should be communicated to the Data Infra Working Group
|
|
|
|
|
as soon as possible.
|
|
|
|
|
schedule_interval: "1h"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
2023-04-26 21:47:34 +03:00
|
|
|
|
2023-05-04 07:07:16 +03:00
|
|
|
|
|
|
|
|
bqetl_adjust:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
Ds 2944 create external adjust table redux (#3907)
* add metadata, schema yamls and query.py
* created adjust_derived namespace
* add query.py, metadata, schema, dataset for testing
* delete extraneous file, update DAG name
* modify bqetl_adjust DAG redux
* update DAG name, take out '_derived'
* update table name in view
* standardize table names across files
* regenerate DAG
* update schema in both locations
* add query.py, metadata, schema yaml files
* take put extraneous print statements, update datasets to be 'adjust' or 'adjust_derived'
* add submission date to date_partition_parameter
* update table name to be just one table
* add DAG for adjust_derived
* add bq_etl adjust_derived DAG to yaml file
* add note about API token
* revert changes to bqetl.adjust.py
* use proper tast_id
* fix start dates
* add python command and docker image
* add python command and docker image
* delete extraneous code
* comment out docker part in old adjust dag
* add whitespace, delete extraneous code
* Update sql/moz-fx-data-shared-prod/adjust/adjust_derived/view.sql
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* updated logic to check if response dictionary is not empty, moved view out of nested folder, added token ownership statement to metadata file, turned off email retry in dags.yaml, separated out clean up of json to its own function
* take out extraneous if statement and move else statement
* reorder where comment is to make more sense
* more description as to why we're using mhirose's API token
* take out periods
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/metadata.yaml
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
* combine adjust DAGs
* change logic for query_export check loop continuance, adapt metadata.yamls
* add blank parameters test
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/metadata.yaml
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
* add arguments to metadata.yaml
* remove external table reference
* refactor to add date parameter
* refactor based on Circle CI's advice
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* take out TODO comment
---------
Co-authored-by: kik-kik <kignasiak@mozilla.com>
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
2023-07-08 01:32:03 +03:00
|
|
|
- mhirose@mozilla.com
|
2023-05-04 07:07:16 +03:00
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
Ds 2944 create external adjust table redux (#3907)
* add metadata, schema yamls and query.py
* created adjust_derived namespace
* add query.py, metadata, schema, dataset for testing
* delete extraneous file, update DAG name
* modify bqetl_adjust DAG redux
* update DAG name, take out '_derived'
* update table name in view
* standardize table names across files
* regenerate DAG
* update schema in both locations
* add query.py, metadata, schema yaml files
* take put extraneous print statements, update datasets to be 'adjust' or 'adjust_derived'
* add submission date to date_partition_parameter
* update table name to be just one table
* add DAG for adjust_derived
* add bq_etl adjust_derived DAG to yaml file
* add note about API token
* revert changes to bqetl.adjust.py
* use proper tast_id
* fix start dates
* add python command and docker image
* add python command and docker image
* delete extraneous code
* comment out docker part in old adjust dag
* add whitespace, delete extraneous code
* Update sql/moz-fx-data-shared-prod/adjust/adjust_derived/view.sql
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* updated logic to check if response dictionary is not empty, moved view out of nested folder, added token ownership statement to metadata file, turned off email retry in dags.yaml, separated out clean up of json to its own function
* take out extraneous if statement and move else statement
* reorder where comment is to make more sense
* more description as to why we're using mhirose's API token
* take out periods
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/metadata.yaml
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
* combine adjust DAGs
* change logic for query_export check loop continuance, adapt metadata.yamls
* add blank parameters test
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/metadata.yaml
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
* add arguments to metadata.yaml
* remove external table reference
* refactor to add date parameter
* refactor based on Circle CI's advice
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* take out TODO comment
---------
Co-authored-by: kik-kik <kignasiak@mozilla.com>
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
2023-07-08 01:32:03 +03:00
|
|
|
owner: mhirose@mozilla.com
|
2023-05-04 07:07:16 +03:00
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
Ds 2944 create external adjust table redux (#3907)
* add metadata, schema yamls and query.py
* created adjust_derived namespace
* add query.py, metadata, schema, dataset for testing
* delete extraneous file, update DAG name
* modify bqetl_adjust DAG redux
* update DAG name, take out '_derived'
* update table name in view
* standardize table names across files
* regenerate DAG
* update schema in both locations
* add query.py, metadata, schema yaml files
* take put extraneous print statements, update datasets to be 'adjust' or 'adjust_derived'
* add submission date to date_partition_parameter
* update table name to be just one table
* add DAG for adjust_derived
* add bq_etl adjust_derived DAG to yaml file
* add note about API token
* revert changes to bqetl.adjust.py
* use proper tast_id
* fix start dates
* add python command and docker image
* add python command and docker image
* delete extraneous code
* comment out docker part in old adjust dag
* add whitespace, delete extraneous code
* Update sql/moz-fx-data-shared-prod/adjust/adjust_derived/view.sql
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* updated logic to check if response dictionary is not empty, moved view out of nested folder, added token ownership statement to metadata file, turned off email retry in dags.yaml, separated out clean up of json to its own function
* take out extraneous if statement and move else statement
* reorder where comment is to make more sense
* more description as to why we're using mhirose's API token
* take out periods
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/metadata.yaml
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
* combine adjust DAGs
* change logic for query_export check loop continuance, adapt metadata.yamls
* add blank parameters test
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/metadata.yaml
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
* add arguments to metadata.yaml
* remove external table reference
* refactor to add date parameter
* refactor based on Circle CI's advice
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* Update sql/moz-fx-data-shared-prod/adjust_derived/adjust_derived_v1/query.py
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
* take out TODO comment
---------
Co-authored-by: kik-kik <kignasiak@mozilla.com>
Co-authored-by: Lucia <30448600+lucia-vargas-a@users.noreply.github.com>
Co-authored-by: kik-kik <42538694+kik-kik@users.noreply.github.com>
Co-authored-by: Anna Scholtz <anna@scholtzan.net>
2023-07-08 01:32:03 +03:00
|
|
|
start_date: '2023-07-06'
|
|
|
|
|
description: |
|
|
|
|
|
Derived tables built on Adjust data downloaded from https://api.adjust.com/kpis/v1/<app_token>
|
|
|
|
|
Using mhirose's API token - no Adjust API token for service accounts, just users.
|
2023-05-04 07:07:16 +03:00
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: 0 4 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
|
|
|
|
- repo/bigquery-etl
|
|
|
|
|
|
2023-04-26 21:47:34 +03:00
|
|
|
bqetl_download_funnel_attribution:
|
|
|
|
|
description: Daily aggregations of data exported from Google Analytics joined with Firefox download data.
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- gleonard@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: gleonard@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-04-10'
|
|
|
|
|
schedule_interval: 0 23 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
2023-05-09 05:24:10 +03:00
|
|
|
|
|
|
|
|
bqetl_fenix_external:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- frank@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: frank@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2023-05-07"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
2023-05-26 18:32:19 +03:00
|
|
|
bqetl_fivetran_apple_ads:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- frank@mozilla.com
|
|
|
|
|
- kik@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: kik@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-05-25'
|
|
|
|
|
description: |
|
|
|
|
|
Copies over apple_ads data coming from Fivetran
|
|
|
|
|
into our data BQ project. Fivetran syncs this data
|
|
|
|
|
every hour. We copy the data every 3 hours to our project.
|
|
|
|
|
schedule_interval: 0 3 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
2023-07-07 21:24:03 +03:00
|
|
|
bqetl_fivetran_copied_tables:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- frank@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: frank@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-07-04'
|
|
|
|
|
description: |
|
|
|
|
|
Copy over Fivetran data to shared-prod.
|
|
|
|
|
schedule_interval: 0 3 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
2023-07-31 20:47:15 +03:00
|
|
|
bqetl_kpis_shredder:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- lvargas@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: lvargas@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-05-16'
|
2023-09-12 16:15:23 +03:00
|
|
|
description: |
|
|
|
|
|
This DAG calculates KPIs for shredder client_ids
|
2023-07-31 20:47:15 +03:00
|
|
|
repo: bigquery-etl
|
2023-10-12 19:18:24 +03:00
|
|
|
schedule_interval: 0 2 */28 * *
|
2023-07-31 20:47:15 +03:00
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
- repo/bigquery-etl
|
2023-08-29 15:32:52 +03:00
|
|
|
bqetl_default:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: false
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: telemetry-alerts@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-09-01'
|
|
|
|
|
description: This is a default DAG to schedule tasks with lower business impact
|
2023-08-31 13:15:15 +03:00
|
|
|
or that don't require a new or existing DAG. Queries are automatically scheduled
|
|
|
|
|
in this DAG during creation when no dag name is specified using option --dag.
|
|
|
|
|
See [related documentation in the cookbooks](https://mozilla.github.io/bigquery-etl/cookbooks/creating_a_derived_dataset/)
|
2023-08-29 15:32:52 +03:00
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: 0 4 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
- triage/no_triage
|
2023-09-20 18:55:09 +03:00
|
|
|
|
|
|
|
|
bqetl_reference:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- cmorales@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: cmorales@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-09-18'
|
|
|
|
|
description: DAG to build reference data
|
|
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: daily
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
2023-10-13 00:05:08 +03:00
|
|
|
|
|
|
|
|
bqetl_generated_funnels:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- ascholtz@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-10-14'
|
|
|
|
|
description: DAG scheduling funnels defined in sql_generators/funnels
|
|
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: 0 5 * * *
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_3
|
|
|
|
|
- triage/no_triage
|
|
|
|
|
|
|
|
|
|
bqetl_serp:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- akommasani@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: akommasani@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-10-01'
|
|
|
|
|
description: DAG to build serp events data
|
|
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: daily
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
2023-10-18 23:57:15 +03:00
|
|
|
|
|
|
|
|
bqetl_review_checker:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- akommasani@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: akommasani@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-10-01'
|
|
|
|
|
description: DAG to build review checker data
|
|
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: daily
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
2023-10-24 19:51:14 +03:00
|
|
|
|
|
|
|
|
bqetl_ads:
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
- cmorales@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
end_date: null
|
|
|
|
|
owner: cmorales@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: '2023-10-10'
|
|
|
|
|
description: Tables related to ads
|
|
|
|
|
repo: bigquery-etl
|
|
|
|
|
schedule_interval: daily
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
|
|
|
|
- repo/bigquery-etl
|
2023-11-16 23:58:33 +03:00
|
|
|
|
|
|
|
|
bqetl_mozilla_org_derived:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- frank@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: frank@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2023-11-13"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
2023-11-30 20:48:17 +03:00
|
|
|
|
|
|
|
|
bqetl_glean_usage:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- ascholtz@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2023-11-20"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_1
|
2023-12-08 01:56:55 +03:00
|
|
|
|
|
|
|
|
bqetl_glam_export:
|
|
|
|
|
schedule_interval: 0 8 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- ascholtz@mozilla.com
|
|
|
|
|
- efilho@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: true
|
|
|
|
|
owner: ascholtz@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2023-11-28"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
|
|
|
|
description: DAG to prepare GLAM data for public export.
|
2023-12-12 19:57:52 +03:00
|
|
|
|
|
|
|
|
bqetl_crash:
|
|
|
|
|
schedule_interval: 0 2 * * *
|
|
|
|
|
default_args:
|
|
|
|
|
depends_on_past: false
|
|
|
|
|
email:
|
|
|
|
|
- dthorn@mozilla.com
|
|
|
|
|
- telemetry-alerts@mozilla.com
|
|
|
|
|
email_on_failure: true
|
|
|
|
|
email_on_retry: false
|
|
|
|
|
owner: dthorn@mozilla.com
|
|
|
|
|
retries: 2
|
|
|
|
|
retry_delay: 30m
|
|
|
|
|
start_date: "2023-12-10"
|
|
|
|
|
tags:
|
|
|
|
|
- impact/tier_2
|
