Bug 1698578 - Parse fenix tagged ad click keys (#1906)

sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/query.sql

@@ -18,6 +18,7 @@ SELECT
   SUM(tagged_sap) AS tagged_sap,
   SUM(tagged_follow_on) AS tagged_follow_on,
   SUM(ad_click) AS ad_click,
+  SUM(ad_click_organic) AS ad_click_organic,
   SUM(search_with_ads) AS search_with_ads,
   SUM(unknown) AS unknown,
 FROM

sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql

@@ -133,11 +133,14 @@ fenix_flattened_searches AS (
     WHEN
       search.search_type = 'in-content'
     THEN
+      -- key format is engine.in-content.type.code
       SPLIT(search.key, '.')[SAFE_OFFSET(0)]
     WHEN
       search.search_type = 'ad-click' OR search.search_type = 'search-with-ads'
     THEN
-      search.key
+      -- ad-click key format is engine.in-content.type.code for builds starting 2021-03-16
+      -- otherwise key is engine
+      SPLIT(search.key, '.')[SAFE_OFFSET(0)]
     ELSE
       NULL
     END AS engine,
@@ -158,6 +161,14 @@ fenix_flattened_searches AS (
         NULL,
         SUBSTR(search.key, STRPOS(search.key, '.') + 1)
       )
+    WHEN
+      search.search_type = 'ad-click'
+    THEN
+      IF(
+        REGEXP_CONTAINS(search.key, '\\.'),
+        SUBSTR(search.key, STRPOS(search.key, '.') + 1),
+        search.search_type
+      )
     ELSE
       search.search_type
     END AS source,
@@ -219,6 +230,10 @@ combined_search_clients AS (
     source,
 
     CASE
+    WHEN
+      search_type = 'ad-click'
+    THEN
+      IF(STARTS_WITH(source, 'in-content.organic'), 'ad-click-organic', search_type)
     WHEN
       STARTS_WITH(source, 'in-content.sap.')
     THEN
@@ -269,6 +284,7 @@ unfiltered_search_clients AS (
     IF(search_count > 10000, NULL, source) AS source,
     app_name,
     normalized_app_name,
+    -- Filter out results with aggregated search count > 10000
     SUM(
       IF(search_type != 'sap' OR engine IS NULL OR search_count > 10000, 0, search_count)
     ) AS search_count,
@@ -284,6 +300,9 @@ unfiltered_search_clients AS (
     SUM(
       IF(search_type != 'ad-click' OR engine IS NULL OR search_count > 10000, 0, search_count)
     ) AS ad_click,
+    SUM(
+      IF(search_type != 'ad-click-organic' OR engine IS NULL OR search_count > 10000, 0, search_count)
+    ) AS ad_click_organic,
     SUM(
       IF(search_type != 'search-with-ads' OR engine IS NULL OR search_count > 10000, 0, search_count)
     ) AS search_with_ads,

sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql

@@ -334,12 +334,15 @@ fenix_flattened_searches AS (
     WHEN
       search.search_type = 'in-content'
     THEN
+      -- key format is engine.in-content.type.code
       SPLIT(search.key, '.')[SAFE_OFFSET(0)]
     WHEN
       search.search_type = 'ad-click'
       OR search.search_type = 'search-with-ads'
     THEN
-      search.key
+      -- ad-click key format is engine.in-content.type.code for builds starting 2021-03-16
+      -- otherwise key is engine
+      SPLIT(search.key, '.')[SAFE_OFFSET(0)]
     ELSE
       NULL
     END
@@ -360,6 +363,14 @@ fenix_flattened_searches AS (
         NULL,
         SUBSTR(search.key, STRPOS(search.key, '.') + 1)
       )
+    WHEN
+      search.search_type = 'ad-click'
+    THEN
+      IF(
+        REGEXP_CONTAINS(search.key, '\\.'),
+        SUBSTR(search.key, STRPOS(search.key, '.') + 1),
+        search.search_type
+      )
     ELSE
       search.search_type
     END
@@ -416,6 +427,10 @@ combined_search_clients AS (
     engine,
     source,
     CASE
+    WHEN
+      search_type = 'ad-click'
+    THEN
+      IF(STARTS_WITH(source, 'in-content.organic'), 'ad-click-organic', search_type)
     WHEN
       STARTS_WITH(source, 'in-content.sap.')
     THEN
@@ -466,6 +481,7 @@ unfiltered_search_clients AS (
     IF(search_count > 10000, NULL, source) AS source,
     app_name,
     normalized_app_name,
+    -- Filter out results with aggregated search count > 10000
     SUM(
       IF(search_type != 'sap' OR engine IS NULL OR search_count > 10000, 0, search_count)
     ) AS search_count,
@@ -487,6 +503,15 @@ unfiltered_search_clients AS (
     SUM(
       IF(search_type != 'ad-click' OR engine IS NULL OR search_count > 10000, 0, search_count)
     ) AS ad_click,
+    SUM(
+      IF(
+        search_type != 'ad-click-organic'
+        OR engine IS NULL
+        OR search_count > 10000,
+        0,
+        search_count
+      )
+    ) AS ad_click_organic,
     SUM(
       IF(
         search_type != 'search-with-ads'

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.yaml

@@ -17,6 +17,7 @@
   tagged_sap: 0
   tagged_follow_on: 0
   ad_click: 0
+  ad_click_organic: 0
   search_with_ads: 0
   unknown: 0
 - <<: *base

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.schema.json

@@ -54,6 +54,11 @@
     "name": "ad_click",
     "mode": "NULLABLE"
   },
+  {
+    "type": "INTEGER",
+    "name": "ad_click_organic",
+    "mode": "NULLABLE"
+  },
   {
     "type": "INTEGER",
     "name": "search_with_ads",

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.yaml

@@ -8,6 +8,7 @@
   tagged_sap: 0
   tagged_follow_on: 0
   ad_click: 0
+  ad_click_organic: 0
   search_with_ads: 0
   unknown: 0
   locale: fr-FR

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml

@@ -9,6 +9,7 @@
   tagged_sap: 0
   tagged_follow_on: 0
   ad_click: 0
+  ad_click_organic: 0
   search_with_ads: 0
   unknown: 0
   app_version: 68.2.1
@@ -173,6 +174,12 @@
   search_count: 0
   ad_click: 3
   total_uri_count: 23
+- <<: *new_field_base
+  engine: engine4
+  source: in-content.sap.code
+  search_count: 0
+  ad_click: 2
+  total_uri_count: 22
 - <<: *new_field_base
   engine: engine3
   source: search-with-ads
@@ -200,6 +207,12 @@
   source: in-content.sap-follow-on.none.ts
   search_count: 0
   tagged_follow_on: 3
+- <<: *new_field_base
+  engine: engine5
+  source: in-content.organic.none
+  search_count: 0
+  ad_click_organic: 3
+  total_uri_count: 22
 - <<: *base
   client_id: e
   channel: beta

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.yaml

@@ -102,6 +102,10 @@
           value: 3
         - key: engine4
           value: 2
+        - key: engine4.in-content.sap.code
+          value: 2
+        - key: engine5.in-content.organic.none
+          value: 3
       browser_search_with_ads:
         - key: engine3
           value: 1

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml

@@ -7,6 +7,7 @@
   tagged_sap: 0
   tagged_follow_on: 0
   ad_click: 0
+  ad_click_organic: 0
   search_with_ads: 0
   unknown: 0
   country: US

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml

@@ -7,6 +7,7 @@
   tagged_sap: 0
   tagged_follow_on: 0
   ad_click: 0
+  ad_click_organic: 0
   search_with_ads: 0
   unknown: 0
   locale: fr-FR