DENG-871 - Add installs_by_country for Fenix (#3767)

* Add installs_by_country for Fenix

* Dont dryrun the query

* Dont dryrun view

* Add chanel to query

bigquery_etl/dryrun.py

@@ -193,6 +193,8 @@ SKIP = {
     ),
     "sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v1/query.sql",
     "sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v2/query.sql",
+    "sql/moz-fx-data-shared-prod/fenix_external/installs_by_country_v1/query.sql",
+    "sql/moz-fx-data-shared-prod/fenix/installs_by_country/view.sql",
     # Materialized views
     "sql/moz-fx-data-shared-prod/telemetry_derived/experiment_search_events_live_v1/init.sql",  # noqa E501
     "sql/moz-fx-data-shared-prod/telemetry_derived/experiment_events_live_v1/init.sql",  # noqa E501

dags.yaml

@@ -1080,3 +1080,20 @@ bqetl_download_funnel_attribution:
   tags:
     - impact/tier_1
     - repo/bigquery-etl
+
+bqetl_fenix_external:
+  schedule_interval: 0 2 * * *
+  default_args:
+    depends_on_past: false
+    email:
+      - frank@mozilla.com
+      - telemetry-alerts@mozilla.com
+    email_on_failure: true
+    email_on_retry: true
+    owner: frank@mozilla.com
+    retries: 2
+    retry_delay: 30m
+    start_date: "2023-05-07"
+  tags:
+    - impact/tier_1
+    - repo/bigquery-etl

dags/bqetl_fenix_external.py

@@ -0,0 +1,67 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.sensors.external_task import ExternalTaskMarker
+from airflow.sensors.external_task import ExternalTaskSensor
+from airflow.utils.task_group import TaskGroup
+import datetime
+from utils.constants import ALLOWED_STATES, FAILED_STATES
+from utils.gcp import bigquery_etl_query, gke_command
+
+docs = """
+### bqetl_fenix_external
+
+Built from bigquery-etl repo, [`dags/bqetl_fenix_external.py`](https://github.com/mozilla/bigquery-etl/blob/main/dags/bqetl_fenix_external.py)
+
+#### Owner
+
+frank@mozilla.com
+"""
+
+
+default_args = {
+    "owner": "frank@mozilla.com",
+    "start_date": datetime.datetime(2023, 5, 7, 0, 0),
+    "end_date": None,
+    "email": ["frank@mozilla.com", "telemetry-alerts@mozilla.com"],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=1800),
+    "email_on_failure": True,
+    "email_on_retry": True,
+    "retries": 2,
+}
+
+tags = ["impact/tier_1", "repo/bigquery-etl"]
+
+with DAG(
+    "bqetl_fenix_external",
+    default_args=default_args,
+    schedule_interval="0 2 * * *",
+    doc_md=docs,
+    tags=tags,
+) as dag:
+    fenix_external__installs_by_country__v1 = bigquery_etl_query(
+        task_id="fenix_external__installs_by_country__v1",
+        destination_table="installs_by_country_v1",
+        dataset_id="fenix_external",
+        project_id="moz-fx-data-shared-prod",
+        owner="frank@mozilla.com",
+        email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
+    wait_for_play_store_export = ExternalTaskSensor(
+        task_id="wait_for_play_store_export",
+        external_dag_id="play_store_export",
+        external_task_id="play_store_export",
+        execution_delta=datetime.timedelta(seconds=7200),
+        check_existence=True,
+        mode="reschedule",
+        allowed_states=ALLOWED_STATES,
+        failed_states=FAILED_STATES,
+        pool="DATA_ENG_EXTERNALTASKSENSOR",
+    )
+
+    fenix_external__installs_by_country__v1.set_upstream(wait_for_play_store_export)

sql/moz-fx-data-shared-prod/fenix/installs_by_country/metadata.yaml

@@ -0,0 +1,8 @@
+friendly_name: Installs By Country
+description: |-
+  Please provide a description for the view
+owners:
+- example@mozilla.com
+labels: {}
+bigquery: null
+references: {}

sql/moz-fx-data-shared-prod/fenix/installs_by_country/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.fenix.installs_by_country`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.fenix_external.installs_by_country_v1`

sql/moz-fx-data-shared-prod/fenix_external/installs_by_country_v1/metadata.yaml

@@ -0,0 +1,22 @@
+friendly_name: Installs By Country
+description: |-
+  Google Play Store installs, broken down by country.
+  Grain is (date, channel, country_code).
+owners:
+- frank@mozilla.com
+labels:
+  application: firefox_android
+  incremental: false
+  schedule: daily
+scheduling:
+  dag_name: bqetl_fenix_external
+  depends_on_past: false
+  date_partition_parameter: null
+  depends_on:
+  - task_id: play_store_export
+    dag_name: play_store_export
+    execution_delta: 2h
+bigquery:
+  clustering:
+    fields: [channel, date]
+references: {}

sql/moz-fx-data-shared-prod/fenix_external/installs_by_country_v1/query.sql

@@ -0,0 +1,36 @@
+WITH app_data AS (
+  SELECT
+    Date AS date,
+    mozfun.norm.firefox_android_package_name_to_channel(Package_Name) AS channel,
+    Country AS code,
+    SUM(Install_events) AS install_count,
+    SUM(Uninstall_events) AS uninstall_count,
+    SUM(Update_events) AS update_count,
+  FROM
+    `moz-fx-data-marketing-prod`.google_play_store.p_Installs_country_v1
+  WHERE
+    mozfun.norm.firefox_android_package_name_to_channel(Package_Name) IS NOT NULL
+  GROUP BY
+    date,
+    channel,
+    code
+)
+SELECT
+  date,
+  channel,
+  code AS country_code,
+  country_details.name AS country_name,
+  country_details.region_name AS region_name,
+  country_details.subregion_name AS subregion_name,
+  install_count,
+  uninstall_count,
+  update_count,
+FROM
+  app_data
+LEFT JOIN
+  mozdata.static.country_codes_v1 AS country_details
+USING
+  (code)
+ORDER BY
+  country_code,
+  date