Add topic selection fields to newtab_visits (#6004)

* Add topic selection fields to newtab_visits

* Remove duplicate from schema

* Add new metric to test schema

* Fix event name in test yaml

* Test yaml fix

* Add visit id to test case

---------

Co-authored-by: Chelsey Beck <64881557+chelseybeck@users.noreply.github.com>

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql

@@ -54,7 +54,8 @@ visit_metadata AS (
       IF(event_name = "opened", mozfun.map.get_key(event_details, "source"), NULL)
     ) AS newtab_open_source,
     LOGICAL_OR(event_name IN ("click", "issued", "save")) AS had_non_impression_engagement,
-    LOGICAL_OR(event_name IN ("click", "save")) AS had_non_search_engagement
+    LOGICAL_OR(event_name IN ("click", "save")) AS had_non_search_engagement,
+    ANY_VALUE(metrics.string_list.newtab_selected_topics) AS newtab_selected_topics,
   FROM
     events_unnested
   GROUP BY
@@ -240,16 +241,43 @@ pocket_events AS (
       event_name = 'save'
       AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
     ) AS organic_pocket_saves,
+    COUNTIF(
+      event_name = 'dismiss'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_pocket_dismissals,
+    COUNTIF(
+      event_name = 'dismiss'
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+    ) AS organic_pocket_dismissals,
+    COUNTIF(
+      event_name = 'thumb_voting_interaction'
+      AND mozfun.map.get_key(event_details, "thumbs_up") = "true"
+    ) AS pocket_thumbs_up,
+    COUNTIF(
+      event_name = 'thumb_voting_interaction'
+      AND mozfun.map.get_key(event_details, "thumbs_down") = "true"
+    ) AS pocket_thumbs_down,
+    mozfun.map.get_key(event_details, "received_rank") AS pocket_received_rank,
+    mozfun.map.get_key(
+      event_details,
+      "scheduled_corpus_item_id"
+    ) AS pocket_scheduled_corpus_item_id,
+    mozfun.map.get_key(event_details, "topic") AS pocket_topic,
+    mozfun.map.get_key(event_details, "matches_selected_topic") AS pocket_matches_selected_topic,
   FROM
     events_unnested
   WHERE
     event_category = 'pocket'
-    AND event_name IN ('impression', 'click', 'save')
+    AND event_name IN ('impression', 'click', 'save', 'dismiss')
   GROUP BY
     newtab_visit_id,
     pocket_story_position,
     pocket_tile_id,
-    pocket_recommendation_id
+    pocket_recommendation_id,
+    pocket_received_rank,
+    pocket_scheduled_corpus_item_id,
+    pocket_topic,
+    pocket_matches_selected_topic
 ),
 pocket_summary AS (
   SELECT
@@ -371,6 +399,53 @@ weather_summary AS (
   GROUP BY
     newtab_visit_id
 ),
+topic_selection_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    mozfun.map.get_key(event_details, "previous_topics") AS previous_topics,
+    mozfun.map.get_key(event_details, "topics") AS topics,
+    COUNTIF(event_name = 'topic_selection_open') AS topic_selection_open,
+    COUNTIF(event_name = 'topic_selection_dismiss') AS topic_selection_dismiss,
+    COUNTIF(
+      event_name = 'topic_selection_topics_saved'
+      AND mozfun.map.get_key(event_details, "first_save") = "true"
+    ) AS topic_selection_topics_first_saved,
+    COUNTIF(
+      event_name = 'topic_selection_topics_saved'
+      AND mozfun.map.get_key(event_details, "first_save") != "true"
+    ) AS topic_selection_topics_updated,
+  FROM
+    events_unnested
+  WHERE
+    event_category = 'newtab'
+    AND event_name IN (
+      'topic_selection_dismiss',
+      'topic_selection_open',
+      'topic_selection_topics_saved'
+    )
+  GROUP BY
+    newtab_visit_id,
+    previous_topics,
+    topics
+),
+topic_selection_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        previous_topics,
+        topics,
+        topic_selection_open,
+        topic_selection_dismiss,
+        topic_selection_topics_first_saved,
+        topic_selection_topics_updated
+      )
+    ) AS topic_selection_interactions
+  FROM
+    topic_selection_events
+  GROUP BY
+    newtab_visit_id
+),
 combined_newtab_activity AS (
   SELECT
     *
@@ -391,6 +466,9 @@ combined_newtab_activity AS (
   LEFT JOIN
     weather_summary
     USING (newtab_visit_id)
+  LEFT JOIN
+    topic_selection_summary
+    USING (newtab_visit_id)
   WHERE
    -- Keep only rows with interactions, unless we receive a valid newtab.opened event.
    -- This is meant to drop only interactions that only have a newtab.closed event on the same partition
@@ -401,6 +479,7 @@ combined_newtab_activity AS (
     OR pocket_interactions IS NOT NULL
     OR wallpaper_interactions IS NOT NULL
     OR weather_interactions IS NOT NULL
+    OR topic_selection_interactions IS NOT NULL
 ),
 client_profile_info AS (
   SELECT

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml

@@ -221,6 +221,30 @@ fields:
   - name: pocket_recommendation_id
     type: STRING
     mode: NULLABLE
+  - mode: NULLABLE
+    name: sponsored_pocket_dismissals
+    type: INTEGER
+  - mode: NULLABLE
+    name: organic_pocket_dismissals
+    type: INTEGER
+  - mode: NULLABLE
+    name: pocket_thumbs_up
+    type: INTEGER
+  - mode: NULLABLE
+    name: pocket_thumbs_down
+    type: INTEGER
+  - mode: NULLABLE
+    name: pocket_received_rank
+    type: INTEGER
+  - name: pocket_scheduled_corpus_item_id
+    type: STRING
+    mode: NULLABLE
+  - name: pocket_topic
+    type: STRING
+    mode: NULLABLE
+  - name: pocket_matches_selected_topic
+    type: STRING
+    mode: NULLABLE
   mode: REPEATED
   name: pocket_interactions
   type: RECORD
@@ -283,3 +307,28 @@ fields:
   - name: weather_widget_change_display_to_simple
     type: INTEGER
     mode: NULLABLE
+- name: newtab_selected_topics
+  type: STRING
+  mode: REPEATED
+- name: topic_selection_interactions
+  type: RECORD
+  mode: REPEATED
+  fields:
+  - name: previous_topics
+    type: STRING
+    mode: NULLABLE
+  - name: topics
+    type: STRING
+    mode: NULLABLE
+  - name: topic_selection_open
+    type: INTEGER
+    mode: NULLABLE
+  - name: topic_selection_dismiss
+    type: INTEGER
+    mode: NULLABLE
+  - name: topic_selection_topics_first_saved
+    type: INTEGER
+    mode: NULLABLE
+  - name: topic_selection_topics_updated
+    type: INTEGER
+    mode: NULLABLE

tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1.schema.json

@@ -452,6 +452,11 @@
             "name": "newtab_sov_allocation",
             "type": "STRING",
             "mode": "REPEATED"
+          },
+          {
+            "name": "newtab_selected_topics",
+            "type": "STRING",
+            "mode": "REPEATED"
           }
         ]
       }

tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/test_aggregation/expect.yaml

@@ -6,6 +6,7 @@
   legacy_telemetry_client_id: legacy_id_1
   experiments: []
   newtab_blocked_sponsors: []
+  newtab_selected_topics: []
   is_new_profile: true
   activity_segment: lots_of_activity
   browser_name: Firefox Desktop
@@ -62,3 +63,10 @@
       weather_widget_change_display_to_detailed: 0
       weather_widget_change_display_to_simple: 0
       weather_widget_location_selected: 0
+  topic_selection_interactions:
+    - previous_topics: "[topic_1]"
+      topics: "[topic_1, topic_2]"
+      topic_selection_open: 0
+      topic_selection_dismiss: 0
+      topic_selection_topics_first_saved: 0
+      topic_selection_topics_updated: 1

tests/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1.yaml

@@ -73,3 +73,14 @@
       extra:
         - key: newtab_visit_id
           value: visit_id_1
+    - category: newtab
+      name: topic_selection_topics_saved
+      extra:
+        - key: newtab_visit_id
+          value: visit_id_1
+        - key: topics
+          value: "[topic_1, topic_2]"
+        - key: first_save
+          value: false
+        - key: previous_topics
+          value: "[topic_1]"