Add has_adblocker_addon to search_clients_daily (#3558)

* feat: adblocker addons field

* Update sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql

Co-authored-by: Curtis Morales <cmorales@mozilla.com>

* fix: use private table

* fix: where clause for private table

* Reference static addons table

* Switch to new monetization_blocking_addons table

* Drop project name in reference to monetization_blocking_addons

* Don't dry-run search_clients_daily

* Add has_adblocker_addons to search_clients_daily_v8 tests

---------

Co-authored-by: Curtis Morales <cmorales@mozilla.com>

bqetl_project.yaml

@@ -125,6 +125,7 @@ dry_run:
   - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql
   - sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql
   - sql/moz-fx-data-shared-prod/fenix_derived/google_ads_campaign_cost_breakdowns_v1/query.sql
+  - sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql
   - sql/moz-fx-data-shared-prod/search_terms*/**/*.sql
   - sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql
   - sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql

sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql

@@ -20,9 +20,42 @@ CREATE TEMP FUNCTION add_access_point(
   ARRAY(SELECT AS STRUCT CONCAT(key, '.', access_point) AS key, value, FROM UNNEST(entries))
 );
 
-WITH combined_access_point AS (
+-- List of Ad Blocking Addons produced using this logic: https://github.com/mozilla/search-adhoc-analysis/tree/master/monetization-blocking-addons
+WITH adblocker_addons AS (
   SELECT
-    *,
+    addon_id,
+    addon_name
+  FROM
+    revenue.monetization_blocking_addons
+  WHERE
+    blocks_monetization
+),
+clients_with_adblocker_addons AS (
+  SELECT
+    client_id,
+    submission_date,
+    TRUE AS has_adblocker_addon
+  FROM
+    telemetry.clients_daily
+  CROSS JOIN
+    UNNEST(active_addons) a
+  INNER JOIN
+    adblocker_addons
+  USING
+    (addon_id)
+  WHERE
+    submission_date = @submission_date
+    AND NOT a.user_disabled
+    AND NOT a.app_disabled
+    AND NOT a.blocklisted
+  GROUP BY
+    client_id,
+    submission_date
+),
+combined_access_point AS (
+  SELECT
+    * EXCEPT (has_adblocker_addon),
+    COALESCE(has_adblocker_addon, FALSE) AS has_adblocker_addon,
     ARRAY_CONCAT(
       add_access_point(search_content_urlbar_sum, 'urlbar'),
       add_access_point(search_content_urlbar_persisted_sum, 'urlbar_persisted'),
@@ -70,6 +103,10 @@ WITH combined_access_point AS (
     ) AS ad_clicks_with_sap,
   FROM
     telemetry.clients_daily
+  LEFT JOIN
+    clients_with_adblocker_addons
+  USING
+    (client_id, submission_date)
 ),
 augmented AS (
   SELECT
@@ -193,6 +230,7 @@ counted AS (
     source,
     country,
     get_search_addon_version(active_addons) AS addon_version,
+    has_adblocker_addon,
     app_version,
     distribution_id,
     locale,

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/telemetry.clients_daily.schema.json

@@ -25,6 +25,21 @@
         "mode": "NULLABLE",
         "name": "version",
         "type": "STRING"
+      },
+      {
+        "mode": "NULLABLE",
+        "name": "user_disabled",
+        "type": "BOOLEAN"
+      },
+      {
+        "mode": "NULLABLE",
+        "name": "app_disabled",
+        "type": "BOOLEAN"
+      },
+      {
+        "mode": "NULLABLE",
+        "name": "blocklisted",
+        "type": "BOOLEAN"
       }
     ],
     "mode": "REPEATED",

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_access_point_probes/expect.yaml

@@ -28,6 +28,7 @@
   scalar_parent_urlbar_searchmode_touchbar_sum: []
   scalar_parent_urlbar_searchmode_typed_sum: []
   is_sap_monetizable: False
+  has_adblocker_addon: False
 - <<: *base
   client_id: b
   engine: engine1

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_access_point_probes/revenue.monetization_blocking_addons.yaml

@@ -0,0 +1,8 @@
+---
+- &base
+  addon_id: random@mozilla.com
+  addon_name: random adblocker
+  blocks_monetization: true
+- <<: *base
+  addon_id: another_addon@mozilla.com
+  addon_name: another one

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_access_point_probes/telemetry.clients_daily.yaml

@@ -18,6 +18,7 @@
   scalar_parent_urlbar_searchmode_topsites_urlbar_sum: []
   scalar_parent_urlbar_searchmode_touchbar_sum: []
   scalar_parent_urlbar_searchmode_typed_sum: []
+  active_addons: []
 - <<: *base
   client_id: b
   # should be ignored other than searchbar sap

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/expect.yaml

@@ -28,6 +28,7 @@
   scalar_parent_urlbar_searchmode_touchbar_sum: []
   scalar_parent_urlbar_searchmode_typed_sum: []
   is_sap_monetizable: False
+  has_adblocker_addon: False
 - <<: *base
   client_id: b
   engine: engine1
@@ -69,3 +70,4 @@
   engine: engine2
   source: contextmenu
   sap: 2
+  has_adblocker_addon: true

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/revenue.monetization_blocking_addons.yaml

@@ -0,0 +1,8 @@
+---
+- &base
+  addon_id: random@mozilla.com
+  addon_name: random adblocker
+  blocks_monetization: true
+- <<: *base
+  addon_id: another_addon@mozilla.com
+  addon_name: another one

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.yaml

@@ -21,6 +21,7 @@
   search_counts: []
   ad_clicks: []
   search_with_ads: []
+  active_addons: []
 - <<: *base
   client_id: b
   search_counts:
@@ -49,8 +50,14 @@
   active_addons:
     - addon_id: random@mozilla.com
       version: '0.1'
+      user_disabled: false
+      app_disabled: false
+      blocklisted: false
     - addon_id: followonsearch@mozilla.com
       version: 0.9.6
+      user_disabled: false
+      app_disabled: false
+      blocklisted: false
   search_counts:
     - source: contextmenu
       engine: engine2