RS 959 - Add OS breakouts (#5767)

* RS 959 - Add OS breakouts

* Add mobile OS breakouts

Add mobile OS breakouts

* Add mobile OS breakouts

* Add tests for desktop

* Add mobile tests

* Fix tests

* Fix CI issue

sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql

@@ -260,6 +260,16 @@ counted AS (
     search_cohort,
     os,
     os_version,
+    CASE
+      WHEN mozfun.norm.os(os) = "Windows"
+        THEN mozfun.norm.windows_version_info(os, os_version, windows_build_number)
+      ELSE CAST(mozfun.norm.truncate_version(os_version, "major") AS STRING)
+    END AS os_version_major,
+    CASE
+      WHEN mozfun.norm.os(os) = "Windows"
+        THEN mozfun.norm.windows_version_info(os, os_version, windows_build_number)
+      ELSE CAST(mozfun.norm.truncate_version(os_version, "minor") AS STRING)
+    END AS os_version_minor,
     channel,
     is_default_browser,
     UNIX_DATE(DATE(profile_creation_date)) AS profile_creation_date,

sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml

@@ -38,6 +38,12 @@ fields:
 - mode: NULLABLE
   name: os_version
   type: STRING
+- mode: NULLABLE
+  name: os_version_major
+  type: STRING
+- mode: NULLABLE
+  name: os_version_minor
+  type: STRING
 - mode: NULLABLE
   name: channel
   type: STRING

sql_generators/search/templates/mobile_search_clients_daily.template.sql

@@ -394,6 +394,14 @@ unfiltered_search_clients AS (
     channel,
     udf.mode_last(ARRAY_AGG(os)) AS os,
     udf.mode_last(ARRAY_AGG(os_version)) AS os_version,
+    COALESCE(
+      SAFE_CAST(NULLIF(SPLIT(udf.mode_last(ARRAY_AGG(os_version)), ".")[SAFE_OFFSET(0)], "") AS INTEGER),
+      0
+    ) AS os_version_major,
+    COALESCE(
+      SAFE_CAST(NULLIF(SPLIT(udf.mode_last(ARRAY_AGG(os_version)), ".")[SAFE_OFFSET(1)], "") AS INTEGER),
+      0
+    ) AS os_version_minor,
     udf.mode_last(ARRAY_AGG(default_search_engine)) AS default_search_engine,
     udf.mode_last(
       ARRAY_AGG(default_search_engine_submission_url)

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml

@@ -18,6 +18,8 @@
   normalized_app_name: Fennec
   country: FR
   os_version: '23'
+  os_version_major: 23
+  os_version_minor: 0
   profile_creation_date: 16606
   profile_age_in_days: 1625
   experiments: []
@@ -33,6 +35,8 @@
   normalized_app_name: Fennec
   country: CA
   os_version: '26'
+  os_version_major: 26
+  os_version_minor: 0
   profile_creation_date: 17332
   profile_age_in_days: 899
 - <<: *base
@@ -47,6 +51,8 @@
   normalized_app_name: Fennec
   country: CA
   os_version: '26'
+  os_version_major: 26
+  os_version_minor: 0
   profile_creation_date: 17332
   profile_age_in_days: 899
 - <<: *base
@@ -59,6 +65,8 @@
   normalized_app_name: Fennec
   country: CA
   os_version: '26'
+  os_version_major: 26
+  os_version_minor: 0
   profile_creation_date: 17653
   profile_age_in_days: 578
   experiments:
@@ -73,6 +81,8 @@
   normalized_app_name: Fenix
   country: US
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
   default_search_engine: engine1
@@ -89,6 +99,8 @@
   normalized_app_name: Fenix
   country: CA
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
 - <<: *base
@@ -103,6 +115,8 @@
   normalized_app_name: Fenix
   country: CA
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
 - <<: *base
@@ -117,6 +131,8 @@
   normalized_app_name: Fenix
   country: CA
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
 - <<: *base
@@ -129,6 +145,8 @@
   normalized_app_name: Fenix
   country: EG
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
 - <<: *base
@@ -142,6 +160,8 @@
   normalized_app_name: Fenix
   country: US
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
 - &new_field_base
@@ -156,6 +176,8 @@
   normalized_app_name: Fenix
   country: US
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
   total_uri_count: 22
@@ -222,6 +244,8 @@
   normalized_app_name: Fenix
   country: US
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
   engine: __other__
@@ -236,6 +260,8 @@
   client_id: release_ios
   os: iOS
   os_version: "13.0"
+  os_version_major: 13
+  os_version_minor: 0
   app_version: "35.0"
   app_name: Fennec
   normalized_app_name: Fennec

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/expect.yaml

@@ -19,6 +19,8 @@
   country: US
   os: iOS
   os_version: "13"
+  os_version_major: 13
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
   sample_id: 1

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml

@@ -13,6 +13,8 @@
   unknown: 0
   country: US
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   engine: engine1
   source: actionbar
   search_count: 10
@@ -37,6 +39,8 @@
 - <<: *base
   os: iOS
   os_version: "13.0"
+  os_version_major: 13
+  os_version_minor: 0
   client_id: b
   app_version: "31.0"
   app_name: Fennec
@@ -45,6 +49,8 @@
 - <<: *base
   os: iOS
   os_version: "13.0"
+  os_version_major: 13
+  os_version_minor: 0
   client_id: b
   app_version: "31.0"
   app_name: Fennec
@@ -53,6 +59,8 @@
 - <<: *base
   os: iOS
   os_version: "13.0"
+  os_version_major: 13
+  os_version_minor: 0
   client_id: b
   app_version: "31.0"
   app_name: Fennec

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml

@@ -18,6 +18,8 @@
   country: FR
   os: Android
   os_version: '23'
+  os_version_major: 23
+  os_version_minor: 0
   profile_creation_date: 16606
   profile_age_in_days: 1625
   sample_id: 1
@@ -34,6 +36,8 @@
   normalized_app_name: Fennec
   country: CA
   os_version: '26'
+  os_version_major: 26
+  os_version_minor: 0
   profile_creation_date: 17332
   profile_age_in_days: 899
 - <<: *base
@@ -46,6 +50,8 @@
   normalized_app_name: Fennec
   country: CA
   os_version: '26'
+  os_version_major: 26
+  os_version_minor: 0
   profile_creation_date: 17332
   profile_age_in_days: 899
 - <<: *base
@@ -58,6 +64,8 @@
   normalized_app_name: Fenix
   country: US
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18207
   profile_age_in_days: 24
 - <<: *base
@@ -72,6 +80,8 @@
   normalized_app_name: Fenix
   country: CA
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
 - <<: *base
@@ -84,6 +94,8 @@
   normalized_app_name: Fenix
   country: CA
   os_version: '29'
+  os_version_major: 29
+  os_version_minor: 0
   profile_creation_date: 18201
   profile_age_in_days: 30
 - <<: *base
@@ -93,6 +105,8 @@
   country: US
   os: iOS
   os_version: "35.0"
+  os_version_major: 35
+  os_version_minor: 0
   default_search_engine: engine2
 - <<: *base
   client_id: release_ios
@@ -101,6 +115,8 @@
   country: US
   os: iOS
   os_version: "35.0"
+  os_version_major: 35
+  os_version_minor: 0
   default_search_engine: engine2
 - <<: *base
   client_id: release_ios
@@ -109,6 +125,8 @@
   country: US
   os: iOS
   os_version: "35.0"
+  os_version_major: 35
+  os_version_minor: 0
   default_search_engine: engine2
 - <<: *base
   client_id: release_ios
@@ -117,4 +135,6 @@
   country: US
   os: iOS
   os_version: "35.0"
+  os_version_major: 35
+  os_version_minor: 0
   default_search_engine: engine2

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/telemetry.clients_daily.schema.json

@@ -144,6 +144,11 @@
     "name": "os_version",
     "type": "STRING"
   },
+  {
+    "mode": "NULLABLE",
+    "name": "windows_build_number",
+    "type": "INTEGER"
+  },
   {
     "mode": "NULLABLE",
     "name": "profile_age_in_days",

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_access_point_probes/telemetry.clients_daily.yaml

@@ -19,6 +19,7 @@
   scalar_parent_urlbar_searchmode_touchbar_sum: []
   scalar_parent_urlbar_searchmode_typed_sum: []
   active_addons: []
+  windows_build_number: 16288
 - <<: *base
   client_id: b
   # should be ignored other than searchbar sap

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/expect.yaml

@@ -30,48 +30,80 @@
   is_sap_monetizable: False
   has_adblocker_addon: False
   policies_is_enterprise: True
+  os: 'Darwin'
+  os_version: '23.4.0'
+  os_version_major: '23'
+  os_version_minor: '23.4'
 - <<: *base
   client_id: b
   engine: engine1
   source: searchbar
   sap: 1
   policies_is_enterprise: True
+  os: 'Windows_NT'
+  os_version: '10.0'
+  os_version_major: 'Windows 11'
+  os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
   engine: engine2
   source: 'in-content:sap:code'
   tagged_sap: 2
   policies_is_enterprise: True
+  os: 'Windows_NT'
+  os_version: '10.0'
+  os_version_major: 'Windows 11'
+  os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
   engine: engine2
   source: 'in-content:sap-follow-on:code'
   tagged_follow_on: 3
   policies_is_enterprise: True
+  os: 'Windows_NT'
+  os_version: '10.0'
+  os_version_major: 'Windows 11'
+  os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
   engine: engine2
   source: 'in-content:organic:none'
   organic: 4
   policies_is_enterprise: True
+  os: 'Windows_NT'
+  os_version: '10.0'
+  os_version_major: 'Windows 11'
+  os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
   engine: engine1
   source: 'ad-click:'
   ad_click: 2
   policies_is_enterprise: True
+  os: 'Windows_NT'
+  os_version: '10.0'
+  os_version_major: 'Windows 11'
+  os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
   engine: engine1
   source: 'ad-click:organic.none'
   ad_click_organic: 3
   policies_is_enterprise: True
+  os: 'Windows_NT'
+  os_version: '10.0'
+  os_version_major: 'Windows 11'
+  os_version_minor: 'Windows 11'
 - <<: *base
   client_id: b
   engine: engine1
   source: 'search-with-ads:sap.code'
   search_with_ads: 3
   policies_is_enterprise: True
+  os: 'Windows_NT'
+  os_version: '10.0'
+  os_version_major: 'Windows 11'
+  os_version_minor: 'Windows 11'
 - <<: *base
   client_id: c
   addon_version: 0.9.6

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.yaml

@@ -22,8 +22,14 @@
   ad_clicks: []
   search_with_ads: []
   active_addons: []
+  windows_build_number: 22635
+  os: 'Darwin'
+  os_version: '23.4.0'
 - <<: *base
   client_id: b
+  windows_build_number: 22635
+  os: 'Windows_NT'
+  os_version: '10.0'
   search_counts:
     - source: searchbar
       engine: engine1