AD-310 Add mobile suggest to event_aggregates and event_aggregates_suggest (#5565)

* Add mobile suggest to event_aggregates and event_aggregates_suggest

* Rename test files

* Update expected values

* Fix tests

* Omit null values from expect.yaml

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql

@@ -96,6 +96,70 @@ combined AS (
     -- For firefox 116+ use firefox_desktop.quick_suggest instead
     -- https://bugzilla.mozilla.org/show_bug.cgi?id=1836283
     SAFE_CAST(metadata.user_agent.version AS INT64) < 116
+  UNION ALL
+  -- Suggest Android
+  SELECT
+    metrics.uuid.fx_suggest_context_id AS context_id,
+    DATE(submission_timestamp) AS submission_date,
+    'suggest' AS source,
+    IF(
+      metrics.string.fx_suggest_ping_type = "fxsuggest-click",
+      "click",
+      "impression"
+    ) AS event_type,
+    'phone' AS form_factor,
+    normalized_country_code AS country,
+    metadata.geo.subdivision1 AS subdivision1,
+    metrics.string.fx_suggest_advertiser AS advertiser,
+    client_info.app_channel AS release_channel,
+    metrics.quantity.fx_suggest_position AS position,
+    -- Only remote settings is in use on mobile
+    'remote settings' AS provider,
+    -- Only standard suggestions are in use on mobile
+    'firefox-suggest' AS match_type,
+    SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
+    -- This is the opt-in for Merino, not in use on mobile
+    CAST(NULL AS BOOLEAN) AS suggest_data_sharing_enabled,
+    blocks.query_type,
+  FROM
+    `moz-fx-data-shared-prod.fenix.fx_suggest` fs
+  LEFT JOIN
+    blocks
+    ON fs.metrics.quantity.fx_suggest_block_id = blocks.id
+  WHERE
+    metrics.string.fx_suggest_ping_type IN ("fxsuggest-click", "fxsuggest-impression")
+  UNION ALL
+  -- Suggest iOS
+  SELECT
+    metrics.uuid.fx_suggest_context_id AS context_id,
+    DATE(submission_timestamp) AS submission_date,
+    'suggest' AS source,
+    IF(
+      metrics.string.fx_suggest_ping_type = "fxsuggest-click",
+      "click",
+      "impression"
+    ) AS event_type,
+    'phone' AS form_factor,
+    normalized_country_code AS country,
+    metadata.geo.subdivision1 AS subdivision1,
+    metrics.string.fx_suggest_advertiser AS advertiser,
+    client_info.app_channel AS release_channel,
+    metrics.quantity.fx_suggest_position AS position,
+    -- Only remote settings is in use on mobile
+    'remote settings' AS provider,
+    -- Only standard suggestions are in use on mobile
+    'firefox-suggest' AS match_type,
+    SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
+    -- This is the opt-in for Merino, not in use on mobile
+    CAST(NULL AS BOOLEAN) AS suggest_data_sharing_enabled,
+    blocks.query_type,
+  FROM
+    `moz-fx-data-shared-prod.firefox_ios.fx_suggest` fs
+  LEFT JOIN
+    blocks
+    ON fs.metrics.quantity.fx_suggest_block_id = blocks.id
+  WHERE
+    metrics.string.fx_suggest_ping_type IN ("fxsuggest-click", "fxsuggest-impression")
 ),
 with_event_count AS (
   SELECT

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql

@@ -106,6 +106,70 @@ combined AS (
     -- https://bugzilla.mozilla.org/show_bug.cgi?id=1836283
     SAFE_CAST(metadata.user_agent.version AS INT64) < 116
   UNION ALL
+  -- Suggest Android
+  SELECT
+    metrics.uuid.fx_suggest_context_id AS context_id,
+    DATE(submission_timestamp) AS submission_date,
+    'suggest' AS source,
+    IF(
+      metrics.string.fx_suggest_ping_type = "fxsuggest-click",
+      "click",
+      "impression"
+    ) AS event_type,
+    'phone' AS form_factor,
+    normalized_country_code AS country,
+    metadata.geo.subdivision1 AS subdivision1,
+    metrics.string.fx_suggest_advertiser AS advertiser,
+    client_info.app_channel AS release_channel,
+    metrics.quantity.fx_suggest_position AS position,
+    -- Only remote settings is in use on mobile
+    'remote settings' AS provider,
+    -- Only standard suggestions are in use on mobile
+    'firefox-suggest' AS match_type,
+    SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
+    -- This is the opt-in for Merino, not in use on mobile
+    CAST(NULL AS BOOLEAN) AS suggest_data_sharing_enabled,
+    blocks.query_type,
+  FROM
+    `moz-fx-data-shared-prod.fenix.fx_suggest` fs
+  LEFT JOIN
+    blocks
+    ON fs.metrics.quantity.fx_suggest_block_id = blocks.id
+  WHERE
+    metrics.string.fx_suggest_ping_type IN ("fxsuggest-click", "fxsuggest-impression")
+  UNION ALL
+  -- Suggest iOS
+  SELECT
+    metrics.uuid.fx_suggest_context_id AS context_id,
+    DATE(submission_timestamp) AS submission_date,
+    'suggest' AS source,
+    IF(
+      metrics.string.fx_suggest_ping_type = "fxsuggest-click",
+      "click",
+      "impression"
+    ) AS event_type,
+    'phone' AS form_factor,
+    normalized_country_code AS country,
+    metadata.geo.subdivision1 AS subdivision1,
+    metrics.string.fx_suggest_advertiser AS advertiser,
+    client_info.app_channel AS release_channel,
+    metrics.quantity.fx_suggest_position AS position,
+    -- Only remote settings is in use on mobile
+    'remote settings' AS provider,
+    -- Only standard suggestions are in use on mobile
+    'firefox-suggest' AS match_type,
+    SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
+    -- This is the opt-in for Merino, not in use on mobile
+    CAST(NULL AS BOOLEAN) AS suggest_data_sharing_enabled,
+    blocks.query_type,
+  FROM
+    `moz-fx-data-shared-prod.firefox_ios.fx_suggest` fs
+  LEFT JOIN
+    blocks
+    ON fs.metrics.quantity.fx_suggest_block_id = blocks.id
+  WHERE
+    metrics.string.fx_suggest_ping_type IN ("fxsuggest-click", "fxsuggest-impression")
+  UNION ALL
   SELECT
     metrics.uuid.top_sites_context_id AS context_id,
     DATE(submission_timestamp) AS submission_date,
@@ -126,7 +190,7 @@ combined AS (
     NULL AS match_type,
     SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
     -- 'suggest_data_sharing_enabled' is only available for `quicksuggest_*` tables
-    NULL AS suggest_data_sharing_enabled,
+    CAST(NULL AS BOOLEAN) AS suggest_data_sharing_enabled,
     CAST(NULL AS STRING) AS query_type,
   FROM
     firefox_desktop.top_sites

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml

@@ -18,6 +18,24 @@
   query_type: branded
 - <<: *suggest_base
   event_type: click
+- &suggest_mobile
+  submission_date: "2030-01-01"
+  source: suggest
+  provider: remote settings
+  event_type: impression
+  form_factor: phone
+  country: US
+  subdivision1: DC
+  advertiser: ad4
+  release_channel: release
+  match_type: "firefox-suggest"
+  normalized_os: Android
+  position: 1
+  event_count: 1
+  user_count: 1
+  query_type: branded
+- <<: *suggest_mobile
+  normalized_os: iOS
 - &topsites_base
   submission_date: "2030-01-01"
   source: topsites

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml

@@ -0,0 +1,26 @@
+---
+- &suggest_base
+  submission_timestamp: "2030-01-01 01:00:00"
+  metrics: &suggest_metrics
+    quantity:
+      fx_suggest_block_id: 123
+      fx_suggest_position: 1
+    string:
+      fx_suggest_advertiser: ad4
+      fx_suggest_ping_type: fxsuggest-impression
+    uuid:
+      fx_suggest_context_id: fenix-b
+  metadata:
+    geo:
+      subdivision1: DC
+    user_agent:
+      os: Android
+  client_info:
+    app_channel: release
+  normalized_country_code: US
+- <<: *suggest_base
+  metrics:
+    <<: *suggest_metrics
+    string:
+      fx_suggest_advertiser: ad4
+      fx_suggest_ping_type: fxsuggest-someotherevent

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_ios.fx_suggest.yaml

@@ -0,0 +1,19 @@
+---
+- submission_timestamp: "2030-01-01 01:00:00"
+  metrics:
+    quantity:
+      fx_suggest_block_id: 123
+      fx_suggest_position: 1
+    string:
+      fx_suggest_advertiser: ad4
+      fx_suggest_ping_type: fxsuggest-impression
+    uuid:
+      fx_suggest_context_id: fenix-b
+  metadata:
+    geo:
+      subdivision1: DC
+    user_agent:
+      os: iOS 17
+  client_info:
+    app_channel: release
+  normalized_country_code: US

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/expect.yaml

@@ -20,6 +20,24 @@
   event_type: click
   event_count: 1
   user_count: 1
+- &suggest_mobile
+  submission_date: "2030-01-01"
+  source: suggest
+  provider: remote settings
+  event_type: click
+  form_factor: phone
+  country: US
+  subdivision1: DC
+  advertiser: ad4
+  release_channel: release
+  match_type: "firefox-suggest"
+  normalized_os: Android
+  position: 1
+  event_count: 1
+  user_count: 1
+  query_type: branded
+- <<: *suggest_mobile
+  normalized_os: iOS
 - &topsites_base
   submission_date: "2030-01-01"
   source: topsites

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml

@@ -0,0 +1,75 @@
+---
+- &click
+  submission_timestamp: "2030-01-01 01:00:00"
+  metrics: &suggest_metrics
+    quantity:
+      fx_suggest_block_id: 123
+      fx_suggest_position: 1
+    string:
+      fx_suggest_advertiser: ad4
+      fx_suggest_ping_type: fxsuggest-click
+    uuid:
+      fx_suggest_context_id: fenix-b
+  metadata:
+    geo:
+      subdivision1: DC
+    user_agent:
+      os: Android
+  client_info:
+    app_channel: release
+  normalized_country_code: US
+- <<: *click
+  metrics:
+    <<: *suggest_metrics
+    uuid:
+      fx_suggest_context_id: fenix-c
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_ios.fx_suggest.yaml

@@ -0,0 +1,75 @@
+---
+- &click
+  submission_timestamp: "2030-01-01 01:00:00"
+  metrics: &suggest_metrics
+    quantity:
+      fx_suggest_block_id: 123
+      fx_suggest_position: 1
+    string:
+      fx_suggest_advertiser: ad4
+      fx_suggest_ping_type: fxsuggest-click
+    uuid:
+      fx_suggest_context_id: fenix-b
+  metadata:
+    geo:
+      subdivision1: DC
+    user_agent:
+      os: iOS 17
+  client_info:
+    app_channel: release
+  normalized_country_code: US
+- <<: *click
+  metrics:
+    <<: *suggest_metrics
+    uuid:
+      fx_suggest_context_id: fenix-c
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click
+- *click