Newtab table default UI pocket (#5784)

* Add pocket tile ID * Add new weather, wallpaper telemetry * Update newtab_visits_v1 tests * Fix tests again * Update metadata * Add new boolean to test JSON * Fix tests * Add category metric to test values * test --------- Co-authored-by: Chelsey Beck <64881557+chelseybeck@users.noreply.github.com>
2024-06-17 12:26:59 -07:00 · 2024-06-17 12:26:59 -07:00 · 5507129f6e
--- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/metadata.yaml
@ -18,6 +18,7 @@ bigquery:
    field: submission_date
    require_partition_filter: true
    expiration_days: null
+  range_partitioning: null
  clustering:
    fields:
    - channel
--- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql
+++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql
@ -20,16 +20,6 @@ WITH events_unnested AS (
  WHERE
    DATE(submission_timestamp) = @submission_date
    AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket')
-    AND name IN (
-      'closed',
-      'opened',
-      'impression',
-      'issued',
-      'click',
-      'save',
-      'topic_click',
-      'dismiss'
-    )
 ),
 visit_metadata AS (
  SELECT
@ -54,6 +44,7 @@ visit_metadata AS (
    ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category,
    ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category,
    ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled,
+    ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled,
    ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows,
    ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors,
    ANY_VALUE(ping_info.experiments) AS experiments,
@ -110,6 +101,7 @@ search_events AS (
    events_unnested
  WHERE
    event_category IN ('newtab.search', 'newtab.search.ad')
+    AND event_name IN ('click', 'impression', 'issued')
  GROUP BY
    newtab_visit_id,
    search_engine,
@ -180,6 +172,7 @@ topsites_events AS (
    )
  WHERE
    event_category = 'topsites'
+    AND event_name IN ('dismiss', 'click', 'impression')
  GROUP BY
    newtab_visit_id,
    topsite_tile_position,
@ -218,6 +211,8 @@ pocket_events AS (
  SELECT
    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
    SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position,
+    mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id,
+    mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id,
    COUNTIF(event_name = 'save') AS pocket_saves,
    COUNTIF(event_name = 'click') AS pocket_clicks,
    COUNTIF(event_name = 'impression') AS pocket_impressions,
@ -249,9 +244,12 @@ pocket_events AS (
    events_unnested
  WHERE
    event_category = 'pocket'
+    AND event_name IN ('impression', 'click', 'save')
  GROUP BY
    newtab_visit_id,
-    pocket_story_position
+    pocket_story_position,
+    pocket_tile_id,
+    pocket_recommendation_id
 ),
 pocket_summary AS (
  SELECT
@ -259,6 +257,8 @@ pocket_summary AS (
    ARRAY_AGG(
      STRUCT(
        pocket_story_position,
+        pocket_tile_id,
+        pocket_recommendation_id,
        pocket_impressions,
        sponsored_pocket_impressions,
        organic_pocket_impressions,
@ -275,6 +275,99 @@ pocket_summary AS (
  GROUP BY
    newtab_visit_id
 ),
+wallpaper_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper,
+    COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks,
+    COUNTIF(
+      event_name = 'wallpaper_click'
+      AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true"
+    ) AS wallpaper_clicks_had_previous_wallpaper,
+    COUNTIF(
+      event_name = 'wallpaper_click'
+      AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false"
+    ) AS wallpaper_clicks_first_selected_wallpaper,
+    COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks,
+    COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals,
+    COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks
+  FROM
+    events_unnested
+  WHERE
+    event_category = 'newtab'
+    AND event_name IN (
+      'wallpaper_click',
+      'wallpaper_category_click',
+      'wallpaper_highlight_cta_clicks',
+      'wallpaper_highlight_dismissed'
+    )
+  GROUP BY
+    newtab_visit_id,
+    wallpaper_selected_wallpaper
+),
+wallpaper_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        wallpaper_selected_wallpaper,
+        wallpaper_clicks,
+        wallpaper_clicks_had_previous_wallpaper,
+        wallpaper_clicks_first_selected_wallpaper,
+        wallpaper_category_clicks,
+        wallpaper_highlight_dismissals,
+        wallpaper_highlight_cta_clicks
+      )
+    ) AS wallpaper_interactions
+  FROM
+    wallpaper_events
+  GROUP BY
+    newtab_visit_id
+),
+weather_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions,
+    COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks,
+    COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors,
+    COUNTIF(
+      event_name = 'weather_change_display'
+      AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed"
+    ) AS weather_widget_change_display_to_detailed,
+    COUNTIF(
+      event_name = 'weather_change_display'
+      AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple"
+    ) AS weather_widget_change_display_to_simple
+  FROM
+    events_unnested
+  WHERE
+    event_category = 'newtab'
+    AND event_name IN (
+      'weather_impression',
+      'weather_open_provider_url',
+      'weather_load_error',
+      'weather_change_display'
+    )
+  GROUP BY
+    newtab_visit_id
+),
+weather_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        weather_widget_impressions,
+        weather_widget_clicks,
+        weather_widget_load_errors,
+        weather_widget_change_display_to_detailed,
+        weather_widget_change_display_to_simple
+      )
+    ) AS weather_interactions
+  FROM
+    weather_events
+  GROUP BY
+    newtab_visit_id
+),
 combined_newtab_activity AS (
  SELECT
    *
@ -289,6 +382,12 @@ combined_newtab_activity AS (
  LEFT JOIN
    pocket_summary
    USING (newtab_visit_id)
+  LEFT JOIN
+    wallpaper_summary
+    USING (newtab_visit_id)
+  LEFT JOIN
+    weather_summary
+    USING (newtab_visit_id)
  WHERE
   -- Keep only rows with interactions, unless we receive a valid newtab.opened event.
   -- This is meant to drop only interactions that only have a newtab.closed event on the same partition
@ -297,6 +396,8 @@ combined_newtab_activity AS (
    OR search_interactions IS NOT NULL
    OR topsite_tile_interactions IS NOT NULL
    OR pocket_interactions IS NOT NULL
+    OR wallpaper_interactions IS NOT NULL
+    OR weather_interactions IS NOT NULL
 ),
 client_profile_info AS (
  SELECT
@ -311,7 +412,15 @@ client_profile_info AS (
    client_id
 )
 SELECT
-  *
+  *,
+  CASE
+    WHEN (
+        (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled")
+        OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled")
+      )
+      THEN "default"
+    ELSE "non-default"
+  END AS newtab_default_ui,
 FROM
  combined_newtab_activity
 LEFT JOIN
--- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml
+++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml
@ -215,6 +215,12 @@ fields:
  - mode: NULLABLE
    name: organic_pocket_saves
    type: INTEGER
+  - name: pocket_tile_id
+    type: STRING
+    mode: NULLABLE
+  - name: pocket_recommendation_id
+    type: STRING
+    mode: NULLABLE
  mode: REPEATED
  name: pocket_interactions
  type: RECORD
@ -224,3 +230,53 @@ fields:
 - mode: NULLABLE
  name: activity_segment
  type: STRING
+- mode: NULLABLE
+  name: newtab_default_ui
+  type: STRING
+- name: newtab_weather_widget_enabled
+  type: BOOLEAN
+  mode: NULLABLE
+- name: wallpaper_interactions
+  type: RECORD
+  mode: REPEATED
+  fields:
+  - name: wallpaper_selected_wallpaper
+    type: STRING
+    mode: NULLABLE
+  - name: wallpaper_clicks
+    type: INTEGER
+    mode: NULLABLE
+  - name: wallpaper_clicks_had_previous_wallpaper
+    type: INTEGER
+    mode: NULLABLE
+  - name: wallpaper_clicks_first_selected_wallpaper
+    type: INTEGER
+    mode: NULLABLE
+  - name: wallpaper_category_clicks
+    type: INTEGER
+    mode: NULLABLE
+  - name: wallpaper_highlight_dismissals
+    type: INTEGER
+    mode: NULLABLE
+  - name: wallpaper_highlight_cta_clicks
+    type: INTEGER
+    mode: NULLABLE
+- name: weather_interactions
+  type: RECORD
+  mode: REPEATED
+  fields:
+  - name: weather_widget_impressions
+    type: INTEGER
+    mode: NULLABLE
+  - name: weather_widget_clicks
+    type: INTEGER
+    mode: NULLABLE
+  - name: weather_widget_load_errors
+    type: INTEGER
+    mode: NULLABLE
+  - name: weather_widget_change_display_to_detailed
+    type: INTEGER
+    mode: NULLABLE
+  - name: weather_widget_change_display_to_simple
+    type: INTEGER
+    mode: NULLABLE
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1.schema.json
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1.schema.json
@ -418,6 +418,11 @@
            "name": "topsites_sponsored_enabled",
            "type": "BOOLEAN",
            "mode": "NULLABLE"
+          },
+          {
+            "name": "newtab_weather_enabled",
+            "type": "BOOLEAN",
+            "mode": "NULLABLE"
          }
        ]
      },
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/test_aggregation/expect.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/test_aggregation/expect.yaml
@ -11,6 +11,8 @@
  browser_name: Firefox Desktop
  had_non_impression_engagement: true
  had_non_search_engagement: true
+  newtab_default_ui: default
+  newtab_newtab_category: enabled
  search_interactions:
    - search_engine: Engine1
      search_access_point: urlbar
@ -45,3 +47,17 @@
      pocket_saves: 0
      sponsored_pocket_saves: 0
      organic_pocket_saves: 0
+  wallpaper_interactions:
+    - wallpaper_selected_wallpaper: wallpaper_1
+      wallpaper_clicks: 1
+      wallpaper_clicks_had_previous_wallpaper: 0
+      wallpaper_clicks_first_selected_wallpaper: 1
+      wallpaper_category_clicks: 0
+      wallpaper_highlight_dismissals: 0
+      wallpaper_highlight_cta_clicks: 0
+  weather_interactions:
+    - weather_widget_impressions: 1
+      weather_widget_clicks: 0
+      weather_widget_load_errors: 0
+      weather_widget_change_display_to_detailed: 0
+      weather_widget_change_display_to_simple: 0
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1.yaml
@ -5,6 +5,8 @@
  metrics:
    uuid:
      legacy_telemetry_client_id: legacy_id_1
+    string:
+      newtab_newtab_category: enabled
  events:
    - category: newtab
      name: opened
@ -57,3 +59,17 @@
          value: 1
        - key: is_sponsored
          value: false
+    - category: newtab
+      name: wallpaper_click
+      extra:
+        - key: newtab_visit_id
+          value: visit_id_1
+        - key: selected_wallpaper
+          value: wallpaper_1
+        - key: had_previous_wallpaper
+          value: false
+    - category: newtab
+      name: weather_impression
+      extra:
+        - key: newtab_visit_id
+          value: visit_id_1