RS-788 Add support for organic searches with ads to the mobile search counts tables (#5598)

* adding organic searches with ads to this table

* updating mobile_search_aggregates table with search_with_ads_organic column

* updating the search revenue lever table

- include search_with_ads_organic columns for Bing, Google and DDG

* Fix CI issues

* Fix tests CI failure

* fix tests

* Fix test sql failure

* Update query.sql

reverting back to original code for search_revenue_levers table

---------

Co-authored-by: Alekhya Kommasani <akommasani@mozilla.com>
Co-authored-by: Alekhya <88394696+alekhyamoz@users.noreply.github.com>

sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/query.sql

@@ -22,6 +22,7 @@ SELECT
   SUM(search_with_ads) AS search_with_ads,
   SUM(unknown) AS unknown,
   CAST(NULL AS string) normalized_engine,
+  SUM(search_with_ads_organic) AS search_with_ads_organic,
 FROM
   mobile_search_clients_daily_v1
 WHERE

sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/schema.yaml

@@ -68,3 +68,6 @@ fields:
 - mode: NULLABLE
   name: ad_click_organic
   type: INTEGER
+- mode: NULLABLE
+  name: search_with_ads_organic
+  type: INTEGER

sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql

@@ -722,6 +722,12 @@ glean_flattened_searches AS (
             SUBSTR(search.key, STRPOS(search.key, '.') + 1),
             search.search_type
           )
+      WHEN search.search_type = 'search-with-ads'
+        THEN IF(
+            REGEXP_CONTAINS(search.key, '\\.'),
+            SUBSTR(search.key, STRPOS(search.key, '.') + 1),
+            search.search_type
+          )
       ELSE search.search_type
     END AS source,
     search.value AS search_count,
@@ -774,6 +780,8 @@ combined_search_clients AS (
     CASE
       WHEN search_type = 'ad-click'
         THEN IF(STARTS_WITH(source, 'in-content.organic'), 'ad-click-organic', search_type)
+      WHEN search_type = 'search-with-ads'
+        THEN IF(STARTS_WITH(source, 'in-content.organic'), 'search-with-ads-organic', search_type)
       WHEN STARTS_WITH(source, 'in-content.sap.')
         THEN 'tagged-sap'
       WHEN REGEXP_CONTAINS(source, '^in-content.*-follow-on')
@@ -855,6 +863,15 @@ unfiltered_search_clients AS (
         search_count
       )
     ) AS search_with_ads,
+    SUM(
+      IF(
+        search_type != 'search-with-ads-organic'
+        OR engine IS NULL
+        OR search_count > 10000,
+        0,
+        search_count
+      )
+    ) AS search_with_ads_organic,
     SUM(
       IF(search_type != 'unknown' OR engine IS NULL OR search_count > 10000, 0, search_count)
     ) AS unknown,

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/mobile_search_clients_daily_v1.schema.json

@@ -64,6 +64,11 @@
     "name": "search_with_ads",
     "mode": "NULLABLE"
   },
+  {
+    "type": "INTEGER",
+    "name": "search_with_ads_organic",
+    "mode": "NULLABLE"
+  },
   {
     "type": "INTEGER",
     "name": "unknown",

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.yaml

@@ -19,6 +19,7 @@
   ad_click: 0
   ad_click_organic: 0
   search_with_ads: 0
+  search_with_ads_organic: 0
   unknown: 0
 - <<: *base
   engine: engine1

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.yaml

@@ -10,6 +10,7 @@
   ad_click: 0
   ad_click_organic: 0
   search_with_ads: 0
+  search_with_ads_organic: 0
   unknown: 0
   locale: fr-FR
   app_version: 68.2.1

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml

@@ -11,6 +11,7 @@
   ad_click: 0
   ad_click_organic: 0
   search_with_ads: 0
+  search_with_ads_organic: 0
   unknown: 0
   app_version: 68.2.1
   app_name: Fennec

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/expect.yaml

@@ -9,6 +9,7 @@
   ad_click: 0
   ad_click_organic: 0
   search_with_ads: 0
+  search_with_ads_organic: 0
   unknown: 0
   locale: en-US
   app_version: "30.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml

@@ -9,6 +9,7 @@
   ad_click: 0
   ad_click_organic: 0
   search_with_ads: 0
+  search_with_ads_organic: 0
   unknown: 0
   country: US
   os_version: '29'

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml

@@ -9,6 +9,7 @@
   ad_click: 0
   ad_click_organic: 0
   search_with_ads: 0
+  search_with_ads_organic: 0
   unknown: 0
   locale: fr-FR
   app_version: 68.2.1