DENG-4515 Add profile_group_id to search_clients_daily_v8 (#6064)

Co-authored-by: Marlene Hirose <92952117+Marlene-M-Hirose@users.noreply.github.com>

sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql

@@ -51,6 +51,15 @@ clients_with_adblocker_addons AS (
     client_id,
     submission_date
 ),
+profile_group_id AS (
+  SELECT
+    client_id,
+    profile_group_id
+  FROM
+    `moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6`
+  WHERE
+    submission_date = @submission_date
+),
 is_enterprise_policies AS (
   SELECT
     client_id,
@@ -329,19 +338,26 @@ counted AS (
         source,
         type
     )
+),
+staging AS (
+  SELECT
+    * EXCEPT (_n),
+    `moz-fx-data-shared-prod.udf.monetized_search`(
+      engine,
+      country,
+      distribution_id,
+      submission_date
+    ) AS is_sap_monetizable
+  FROM
+    counted
+  WHERE
+    _n = 1
 )
 SELECT
-  * EXCEPT (_n),
-  `moz-fx-data-shared-prod.udf.monetized_search`(
-    engine,
-    country,
-    distribution_id,
-    submission_date
-  ) AS is_sap_monetizable,
-  CAST(
-    NULL AS string
-  ) AS profile_group_id --temporary for now, will replace with logic once column added to schema successfully
+  stg.*,
+  prfl_gp_id.profile_group_id
 FROM
-  counted
-WHERE
-  _n = 1
+  staging stg
+LEFT JOIN
+  profile_group_id prfl_gp_id
+  ON stg.client_id = prfl_gp_id.client_id

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_access_point_probes/expect.yaml

@@ -2,6 +2,7 @@
 - &base
   submission_date: '2017-01-01'
   client_id: a
+  profile_group_id: prfl_a
   country: US
   sap: 0
   organic: 0
@@ -32,6 +33,7 @@
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine1
   source: searchbar
   sap: 1
@@ -39,89 +41,104 @@
 # in-content
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine3
   source: 'in-content:tagged:code1.urlbar'
   tagged_sap: 4
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine3
   source: 'in-content:organic:none.urlbar'
   organic: 3
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine4
   source: 'in-content:tagged-follow-on:code2.urlbar'
   tagged_follow_on: 5
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.urlbar_searchmode'
   tagged_sap: 1
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.contextmenu'
   tagged_sap: 2
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.about_home'
   tagged_sap: 3
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.about_newtab'
   tagged_sap: 4
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.searchbar'
   tagged_sap: 5
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.system'
   tagged_sap: 6
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.webextension'
   tagged_sap: 7
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.tabhistory'
   tagged_sap: 8
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.reload'
   tagged_sap: 9
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.unknown'
   tagged_sap: 10
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.urlbar_handoff'
   tagged_sap: 11
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine5
   source: 'in-content:tagged:code1.urlbar_persisted'
   tagged_sap: 12
@@ -129,78 +146,91 @@
 # search with ads
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:organic.urlbar'
   search_with_ads_organic: 2
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.urlbar'
   search_with_ads: 1
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.urlbar_searchmode'
   search_with_ads: 12
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.contextmenu'
   search_with_ads: 13
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.about_home'
   search_with_ads: 14
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.about_newtab'
   search_with_ads: 15
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.searchbar'
   search_with_ads: 16
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.system'
   search_with_ads: 17
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.webextension'
   search_with_ads: 18
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.tabhistory'
   search_with_ads: 19
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.reload'
   search_with_ads: 20
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.unknown'
   search_with_ads: 21
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine6
   source: 'search-with-ads:tagged.urlbar_handoff'
   search_with_ads: 22
@@ -208,78 +238,91 @@
 # ad click
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:organic.urlbar'
   ad_click_organic: 4
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.urlbar'
   ad_click: 3
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.urlbar_searchmode'
   ad_click: 22
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.contextmenu'
   ad_click: 23
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.about_home'
   ad_click: 24
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.about_newtab'
   ad_click: 25
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.searchbar'
   ad_click: 26
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.system'
   ad_click: 27
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.webextension'
   ad_click: 28
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.tabhistory'
   ad_click: 29
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.reload'
   ad_click: 30
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.unknown'
   ad_click: 31
   policies_is_enterprise: True
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine7
   source: 'ad-click:tagged.urlbar_handoff'
   ad_click: 32

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_access_point_probes/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.yaml

@@ -0,0 +1,9 @@
+- client_id: a
+  submission_date: '2017-01-01'
+  profile_group_id: prfl_a
+- client_id: b
+  submission_date: '2017-01-01'
+  profile_group_id: prfl_b
+- client_id: c
+  submission_date: '2017-01-01'
+  prfl_group_id: prfl_c

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/expect.yaml

@@ -2,6 +2,7 @@
 - &base
   submission_date: '2017-01-01'
   client_id: a
+  profile_group_id: prfl_a
   country: US
   sap: 0
   organic: 0
@@ -36,6 +37,7 @@
   os_version_minor: '23.4'
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine1
   source: searchbar
   sap: 1
@@ -46,6 +48,7 @@
   os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine2
   source: 'in-content:sap:code'
   tagged_sap: 2
@@ -56,6 +59,7 @@
   os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine2
   source: 'in-content:sap-follow-on:code'
   tagged_follow_on: 3
@@ -66,6 +70,7 @@
   os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine2
   source: 'in-content:organic:none'
   organic: 4
@@ -76,6 +81,7 @@
   os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine1
   source: 'ad-click:'
   ad_click: 2
@@ -86,6 +92,7 @@
   os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine1
   source: 'ad-click:organic.none'
   ad_click_organic: 3
@@ -96,6 +103,7 @@
   os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
+  profile_group_id: prfl_b
   engine: engine1
   source: 'search-with-ads:sap.code'
   search_with_ads: 3
@@ -106,6 +114,7 @@
   os_version_minor: 'Windows 11'
 - <<: *base
   client_id: c
+  profile_group_id: prfl_c
   addon_version: 0.9.6
   engine: engine2
   source: contextmenu

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.yaml

@@ -0,0 +1,9 @@
+- client_id: a
+  submission_date: '2017-01-01'
+  profile_group_id: prfl_a
+- client_id: b
+  submission_date: '2017-01-01'
+  profile_group_id: prfl_b
+- client_id: c
+  submission_date: '2017-01-01'
+  profile_group_id: prfl_c