Urlbar events: nested (long) instead of wide (#4373)

* feat: urlbar events final release * feat: new result types * feat: add interaction and group * fix: date * fix: use BQ builtin for UUIDs * Add the view_v2' * Add new table to the DAG * fix CI error fix ci error * remove teon brooks * Incorporate feedback by Curtis Incorporate feedback from Curtis --------- Co-authored-by: Alekhya Kommasani <akommasani@mozilla.com> Co-authored-by: Alekhya <88394696+alekhyamoz@users.noreply.github.com>
2023-10-25 11:55:00 -05:00 · 2023-10-25 11:55:00 -05:00 · 73a5535b67
--- a/dags.yaml
+++ b/dags.yaml
@ -711,7 +711,6 @@ bqetl_urlbar:
      - "telemetry-alerts@mozilla.com"
      - "anicholson@mozilla.com"
      - "akomar@mozilla.com"
-      - "tbrooks@mozilla.com"
    retries: 2
    retry_delay: 30m
  tags:
--- a/dags/bqetl_urlbar.py
+++ b/dags/bqetl_urlbar.py
@ -31,7 +31,6 @@ default_args = {
        "telemetry-alerts@mozilla.com",
        "anicholson@mozilla.com",
        "akomar@mozilla.com",
-        "tbrooks@mozilla.com",
    ],
    "depends_on_past": False,
    "retry_delay": datetime.timedelta(seconds=1800),
@ -61,7 +60,24 @@ with DAG(
            "anicholson@mozilla.com",
            "dzeber@mozilla.com",
            "rburwei@mozilla.com",
-            "tbrooks@mozilla.com",
+            "telemetry-alerts@mozilla.com",
+        ],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+    )
+
+    firefox_desktop_urlbar_events__v2 = bigquery_etl_query(
+        task_id="firefox_desktop_urlbar_events__v2",
+        destination_table="urlbar_events_v2",
+        dataset_id="firefox_desktop_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="akommasani@mozilla.com",
+        email=[
+            "akomar@mozilla.com",
+            "akommasani@mozilla.com",
+            "anicholson@mozilla.com",
+            "dzeber@mozilla.com",
+            "rburwei@mozilla.com",
            "telemetry-alerts@mozilla.com",
        ],
        date_partition_parameter="submission_date",
@ -98,6 +114,8 @@ with DAG(

    firefox_desktop_urlbar_events__v1.set_upstream(wait_for_copy_deduplicate_all)

+    firefox_desktop_urlbar_events__v2.set_upstream(wait_for_copy_deduplicate_all)
+
    wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskSensor(
        task_id="wait_for_telemetry_derived__clients_daily_joined__v1",
        external_dag_id="bqetl_main_summary",
--- a/sql/moz-fx-data-shared-prod/firefox_desktop/urlbar_events_temp_v2/view.sql
+++ b/sql/moz-fx-data-shared-prod/firefox_desktop/urlbar_events_temp_v2/view.sql
@ -0,0 +1,8 @@
+--- User-facing view. Generated via sql_generators.active_users.
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.urlbar_events_temp_v2`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.urlbar_events_v2`
--- a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/metadata.yaml
@ -0,0 +1,21 @@
+friendly_name: Urlbar Events
+description: |-
+  Each row in this table represents one engagement (https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/metrics/urlbar_engagement)
+  or abandonment(https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/metrics/urlbar_abandonment) event. A urlbar session ends
+  when the urlbar dropdown menu closes, however, not all events mark the end of a urlbar session. To identify the events marking the end of a
+  urlbar session, filter on the field is_terminal = true.
+owners:
+- akommasani@mozilla.com
+- rburwei@mozilla.com
+- dzeber@mozilla.com
+labels:
+  incremental: true
+  schedule: daily
+scheduling:
+  dag_name: bqetl_urlbar
+  task_name: firefox_desktop_urlbar_events__v2
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
--- a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql
+++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql
@ -0,0 +1,110 @@
+CREATE TEMP FUNCTION enumerated_array(results ARRAY<STRING>, _groups ARRAY<STRING>) AS (
+  ARRAY(
+    SELECT
+      STRUCT(
+        off + 1 AS position,
+        r AS result_type,
+        `mozfun.norm.result_type_to_product_name`(r) AS product_result_type,
+        _groups[SAFE_OFFSET(off)] AS result_group
+      )
+    FROM
+      UNNEST(results) AS r
+      WITH OFFSET off
+  )
+);
+
+CREATE TEMP FUNCTION get_event_action(event_name STRING, engagement_type STRING) AS (
+  CASE
+    WHEN event_name = 'engagement'
+      AND (engagement_type IN ("click", "drop_go", "enter", "go_button", "paste_go"))
+      THEN 'engaged'
+    WHEN event_name = 'abandonment'
+      THEN 'abandoned'
+    WHEN event_name = 'engagement'
+      AND (engagement_type NOT IN ("click", "drop_go", "enter", "go_button", "paste_go"))
+      THEN "annoyance"
+    ELSE NULL
+  END
+);
+
+CREATE TEMP FUNCTION get_is_terminal(selected_result STRING, engagement_type STRING) AS (
+  --events where the urlbar dropdown menu remains open (i.e., the urlbar session did not end)
+  COALESCE(
+    NOT (selected_result = 'tab_to_search' AND engagement_type IN ('click', 'enter', 'go_button'))
+    AND NOT (
+      selected_result = 'tip_dismissal_acknowledgement'
+      AND engagement_type IN ('click', 'enter')
+    )
+    AND NOT (
+      engagement_type IN (
+        'dismiss',
+        'inaccurate_location',
+        'not_interested',
+        'not_relevant',
+        'show_less_frequently'
+      )
+    ),
+    TRUE
+  )
+);
+
+WITH events_unnested AS (
+  SELECT
+    DATE(submission_timestamp) AS submission_date,
+    client_info.client_id AS glean_client_id,
+    metrics.uuid.legacy_telemetry_client_id AS legacy_telemetry_client_id,
+    sample_id,
+    name AS event_name,
+    timestamp AS event_timestamp,
+    GENERATE_UUID() AS event_id,
+    ping_info.experiments,
+    ping_info.seq,
+    normalized_channel,
+    normalized_country_code,
+    `moz-fx-data-shared-prod`.udf.normalize_search_engine(
+      mozfun.map.get_key(extra, "search_engine_default_id")
+    ) AS normalized_engine,
+    COALESCE(metrics.boolean.urlbar_pref_suggest_data_collection, FALSE) AS pref_data_collection,
+    COALESCE(metrics.boolean.urlbar_pref_suggest_sponsored, FALSE) AS pref_sponsored_suggestions,
+    COALESCE(metrics.boolean.urlbar_pref_suggest_nonsponsored, FALSE) AS pref_fx_suggestions,
+    mozfun.map.get_key(extra, "engagement_type") AS engagement_type,
+    mozfun.map.get_key(extra, "interaction") AS interaction,
+    CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed,
+    CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results,
+  --If 0, then no result was selected.
+    NULLIF(CAST(mozfun.map.get_key(extra, "selected_position") AS int), 0) AS selected_position,
+    mozfun.map.get_key(extra, "selected_result") AS selected_result,
+    enumerated_array(
+      SPLIT(mozfun.map.get_key(extra, "results"), ','),
+      SPLIT(mozfun.map.get_key(extra, "groups"), ',')
+    ) AS results,
+  FROM
+    `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1`,
+    UNNEST(events) AS event
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+    AND event.category = 'urlbar'
+    AND event.name IN ('engagement', 'abandonment')
+)
+SELECT
+  *,
+  `mozfun.norm.result_type_to_product_name`(selected_result) AS product_selected_result,
+  get_event_action(event_name, engagement_type) AS event_action,
+  get_is_terminal(selected_result, engagement_type) AS is_terminal,
+  CASE
+    WHEN get_event_action(event_name, engagement_type) IN ('engaged', 'annoyance')
+      THEN selected_result
+    ELSE NULL
+  END AS engaged_result_type,
+  CASE
+    WHEN get_event_action(event_name, engagement_type) IN ('engaged', 'annoyance')
+      THEN `mozfun.norm.result_type_to_product_name`(selected_result)
+    ELSE NULL
+  END AS product_engaged_result_type,
+  CASE
+    WHEN get_event_action(event_name, engagement_type) = 'annoyance'
+      THEN engagement_type
+    ELSE NULL
+  END AS annoyance_signal_type,
+FROM
+  events_unnested
--- a/sql/mozfun/norm/result_type_to_product_name/udf.sql
+++ b/sql/mozfun/norm/result_type_to_product_name/udf.sql
@ -3,6 +3,9 @@ CREATE OR REPLACE FUNCTION norm.result_type_to_product_name(
 )
 RETURNS STRING AS (
    CASE
+    WHEN res IN ('autofill_origin', 'autofill_url') THEN 'autofill'
+    WHEN res IN ('addon') THEN 'xchannels_add_on'
+    WHEN res IN ('rs_amo') THEN 'suggest_add_on'
    WHEN res IN ('search_suggest', 'search_history', 'search_suggest_rich') THEN 'default_partner_search_suggestion'
    WHEN res IN ('search_engine') THEN 'search_engine'
    WHEN res IN ('trending_search', 'trending_search_rich') THEN 'trending_suggestion'
@ -11,7 +14,6 @@ RETURNS STRING AS (
    WHEN res IN ('tab') THEN 'open_tab'
    WHEN res IN ('merino_adm_sponsored', 'rs_adm_sponsored', 'suggest_sponsor') THEN 'admarketplace_sponsored'
    WHEN res IN ('merino_top_picks') THEN 'navigational'
-    WHEN res IN ('addon', 'rs_amo') THEN 'add_on'
    WHEN res IN ('rs_adm_nonsponsored',
    'merino_adm_nonsponsored',
    'suggest_non_sponsor') THEN 'wikipedia_enhanced'
@ -19,7 +21,7 @@ RETURNS STRING AS (
    WHEN res IN ('weather') THEN 'weather'
    WHEN res IN ( 'action', 'intervention_clear', 'intervention_refresh', 'intervention_unknown', 'intervention_update' ) THEN 'quick_action'
    WHEN res IN ('rs_pocket') THEN 'pocket_collection'
-    ELSE NULL
+    ELSE 'other'
    END
 );