Fix mobile os version for search related tables (#6111)

* Fix mobile os version

* Fix tests

sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/backfill.yaml

@@ -1,9 +0,0 @@
-2024-07-02:
-  start_date: 2022-01-01
-  end_date: 2024-06-25
-  reason: https://mozilla-hub.atlassian.net/browse/RS-1247
-  watchers:
-  - akommasani@mozilla.com
-  - skahmann@mozilla.com
-  - pissac@mozilla.com
-  status: Initiate

sql_generators/search/templates/android_focus.template.sql

@@ -9,7 +9,7 @@ metrics_{{ namespace }} AS (
         client_info.app_display_version AS app_version,
         '{{ channel }}' AS channel,
         normalized_os AS os,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.browser_default_search_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

sql_generators/search/templates/android_klar.template.sql

@@ -9,7 +9,7 @@ metrics_{{ namespace }} AS (
         client_info.app_display_version AS app_version,
         '{{ channel }}' AS channel,
         normalized_os AS os,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.browser_default_search_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

sql_generators/search/templates/fenix_metrics.template.sql

@@ -18,7 +18,7 @@ metrics_{{ namespace }} AS (
     client_info.app_display_version AS app_version,
     '{{ channel }}' AS channel,
     normalized_os AS os,
-    client_info.android_sdk_version AS os_version,
+    normalized_os_version AS os_version,
     metrics.string.search_default_engine_code AS default_search_engine,
     metrics.string.search_default_engine_submission_url AS default_search_engine_submission_url,
     sample_id,

sql_generators/search/templates/ios_focus.template.sql

@@ -9,7 +9,7 @@ metrics_{{ namespace }} AS (
         client_info.app_display_version AS app_version,
         '{{ channel }}' AS channel,
         normalized_os AS os,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.search_default_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

sql_generators/search/templates/ios_klar.template.sql

@@ -9,7 +9,7 @@ metrics_{{ namespace }} AS (
         client_info.app_display_version AS app_version,
         '{{ channel }}' AS channel,
         normalized_os AS os,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.search_default_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

sql_generators/search/templates/ios_metrics.template.sql

@@ -10,7 +10,7 @@ metrics_{{ namespace }} AS (
     client_info.app_display_version AS app_version,
     '{{ channel }}' AS channel,
     normalized_os AS os,
-    client_info.os_version AS os_version,
+    normalized_os_version AS os_version,
     metrics.string.search_default_engine AS default_search_engine,
     CAST(NULL AS STRING) AS default_search_engine_submission_url,
     sample_id,

sql_generators/search_v2/templates/android_focus.template.sql

@@ -10,7 +10,7 @@
         '{{ channel }}' AS channel,
         normalized_os AS os,
         client_info.locale,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.browser_default_search_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

sql_generators/search_v2/templates/android_klar.template.sql

@@ -10,7 +10,7 @@ baseline_{{ namespace }} AS (
         '{{ channel }}' AS channel,
         normalized_os AS os,
         client_info.locale,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.browser_default_search_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

sql_generators/search_v2/templates/fenix.template.sql

@@ -10,7 +10,7 @@ baseline_{{ namespace }} AS (
     client_info.app_display_version AS app_version,
     '{{ channel }}' AS channel,
     normalized_os AS os,
-    client_info.android_sdk_version AS os_version,
+    normalized_os_version AS os_version,
     metrics.string.search_default_engine_code AS default_search_engine,
     CAST(NULL AS STRING) AS default_search_engine_submission_url,
     sample_id,

sql_generators/search_v2/templates/ios.template.sql

@@ -11,7 +11,7 @@ baseline_{{ namespace }} AS (
     client_info.app_display_version AS app_version,
     '{{ channel }}' AS channel,
     normalized_os AS os,
-    client_info.android_sdk_version AS os_version,
+    normalized_os_version AS os_version,
     metrics.string.search_default_engine AS default_search_engine,
     CAST(NULL AS STRING) AS default_search_engine_submission_url,
     sample_id,

sql_generators/search_v2/templates/ios_focus.template.sql

@@ -10,7 +10,7 @@ baseline_{{ namespace }} AS (
         client_info.app_display_version AS app_version,
         '{{ channel }}' AS channel,
         normalized_os AS os,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.search_default_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

sql_generators/search_v2/templates/ios_klar.template.sql

@@ -10,7 +10,7 @@ baseline_{{ namespace }} AS (
         client_info.app_display_version AS app_version,
         '{{ channel }}' AS channel,
         normalized_os AS os,
-        client_info.android_sdk_version AS os_version,
+        normalized_os_version AS os_version,
         metrics.string.search_default_engine AS default_search_engine,
         CAST(NULL AS STRING) AS default_search_engine_submission_url,
         sample_id,

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_fenix.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_fenix_nightly.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_fennec_aurora.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_firefox.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_firefox_beta.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_focus.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_focus_beta.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_focus_nightly.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_ios_fennec.metrics.schema.json

@@ -125,6 +125,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_ios_firefox.metrics.schema.json

@@ -125,6 +125,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta.metrics.schema.json

@@ -125,6 +125,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_ios_focus.metrics.schema.json

@@ -138,6 +138,10 @@
       "type": "STRING",
       "name": "normalized_os"
     },
+    {
+      "type": "STRING",
+      "name": "normalized_os_version"
+    },
     {
       "fields": [
         {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_ios_klar.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/moz-fx-data-shared-prod.org_mozilla_klar.metrics.schema.json

@@ -138,6 +138,10 @@
     "type": "STRING",
     "name": "normalized_os"
   },
+  {
+    "type": "STRING",
+    "name": "normalized_os_version"
+  },
   {
     "fields": [
       {

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml

@@ -80,7 +80,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: US
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -98,7 +98,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: CA
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -114,7 +114,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: CA
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -130,7 +130,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: CA
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -144,7 +144,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: EG
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -159,7 +159,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: US
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -175,7 +175,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: US
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -243,7 +243,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: US
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/moz-fx-data-shared-prod.org_mozilla_fenix.metrics.yaml

@@ -5,6 +5,7 @@
     end_time: 2019-12-01T04:13-05:00
   normalized_channel: beta
   normalized_os: Android
+  normalized_os_version: "29.0"
   sample_id: 1
   client_info:
     android_sdk_version: "29"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/moz-fx-data-shared-prod.org_mozilla_ios_fennec.metrics.yaml

@@ -4,6 +4,7 @@
     end_time: 2019-12-01T04:13-05:00
   normalized_channel: nightly
   normalized_os: iOS
+  normalized_os_version: "13.0"
   sample_id: 1
   client_info:
     app_display_version: "35.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/moz-fx-data-shared-prod.org_mozilla_ios_firefox.metrics.yaml

@@ -5,6 +5,7 @@
     end_time: 2019-12-01T04:13-05:00
   normalized_channel: release
   normalized_os: iOS
+  normalized_os_version: "13.0"
   sample_id: 1
   client_info:
     app_display_version: "35.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta.metrics.yaml

@@ -4,6 +4,7 @@
     end_time: 2019-12-01T04:13-05:00
   normalized_channel: beta
   normalized_os: iOS
+  normalized_os_version: "13.0"
   sample_id: 1
   client_info:
     app_display_version: "35.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/expect.yaml

@@ -18,7 +18,7 @@
   channel: release
   country: US
   os: iOS
-  os_version: "13"
+  os_version: "13.0"
   os_version_major: 13
   os_version_minor: 0
   profile_creation_date: 18201
@@ -37,3 +37,4 @@
   source: actionbar
   engine: engine1
   search_count: 1
+  os_version: "13.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/moz-fx-data-shared-prod.org_mozilla_ios_firefox.metrics.yaml

@@ -5,6 +5,7 @@
     end_time: 2019-12-01T04:13-05:00
   normalized_channel: release
   normalized_os: iOS
+  normalized_os_version: "13.0"
   sample_id: 1
   client_info:
     app_display_version: "30.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/moz-fx-data-shared-prod.telemetry.core.yaml

@@ -12,7 +12,7 @@
   normalized_app_name: Fennec
   normalized_country_code: US
   normalized_os: iOS
-  normalized_os_version: "13"
+  normalized_os_version: "13.0"
   profile_date: "18201"
   sample_id: "1"
   searches:

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml

@@ -12,7 +12,7 @@
   search_with_ads_organic: 0
   unknown: 0
   country: US
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   engine: engine1
@@ -38,7 +38,7 @@
   channel: release
 - <<: *base
   os: iOS
-  os_version: "13.0"
+  os_version: '13.0'
   os_version_major: 13
   os_version_minor: 0
   client_id: b
@@ -48,7 +48,7 @@
   channel: release
 - <<: *base
   os: iOS
-  os_version: "13.0"
+  os_version: '13.0'
   os_version_major: 13
   os_version_minor: 0
   client_id: b

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/moz-fx-data-shared-prod.org_mozilla_fenix.metrics.yaml

@@ -2,6 +2,7 @@
 - submission_timestamp: 2020-05-01 09:13:09.842719 UTC
   normalized_channel: Other
   normalized_os: Android
+  normalized_os_version: "29.0"
   sample_id: 1
   client_info:
     android_sdk_version: "29"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/moz-fx-data-shared-prod.org_mozilla_firefox.metrics.yaml

@@ -3,6 +3,7 @@
   normalized_channel: Other
   normalized_os: Android
   sample_id: 1
+  normalized_os_version: "29.0"
   client_info:
     android_sdk_version: "29"
     app_display_version: Unknown

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/moz-fx-data-shared-prod.org_mozilla_ios_fennec.metrics.yaml

@@ -2,6 +2,7 @@
 - submission_timestamp: 2020-05-01 09:13:09.842719 UTC
   normalized_channel: Other
   normalized_os: iOS
+  normalized_os_version: "13.0"
   sample_id: 1
   client_info:
     app_display_version: "31.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/moz-fx-data-shared-prod.org_mozilla_ios_firefox.metrics.yaml

@@ -2,6 +2,7 @@
 - submission_timestamp: 2020-05-01 09:13:09.842719 UTC
   normalized_channel: Other
   normalized_os: iOS
+  normalized_os_version: "13.0"
   sample_id: 1
   client_info:
     app_display_version: "31.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta.metrics.yaml

@@ -2,6 +2,7 @@
 - submission_timestamp: 2020-05-01 09:13:09.842719 UTC
   normalized_channel: Other
   normalized_os: iOS
+  normalized_os_version: "13.0"
   sample_id: 1
   client_info:
     app_display_version: "31.0"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/moz-fx-data-shared-prod.telemetry.core.yaml

@@ -8,7 +8,7 @@
   normalized_channel: release
   normalized_country_code: US
   normalized_os: Android
-  normalized_os_version: "29"
+  normalized_os_version: "29.0"
   sample_id: 1
   searches:
     - key: engine1.actionbar

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml

@@ -63,7 +63,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: US
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18207
@@ -79,7 +79,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: CA
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201
@@ -93,7 +93,7 @@
   app_name: Firefox Preview
   normalized_app_name: Fenix
   country: CA
-  os_version: '29'
+  os_version: '29.0'
   os_version_major: 29
   os_version_minor: 0
   profile_creation_date: 18201

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/moz-fx-data-shared-prod.org_mozilla_fenix.metrics.yaml

@@ -8,6 +8,7 @@
   normalized_channel: release
   normalized_country_code: US
   normalized_os: Android
+  normalized_os_version: "29.0"
   ping_info:
     end_time: 2019-12-01T04:13-05:00
   sample_id: "1"
@@ -28,6 +29,7 @@
   normalized_channel: release
   normalized_country_code: CA
   normalized_os: Android
+  normalized_os_version: "29.0"
   ping_info:
     end_time: 2019-12-01T04:13-05:00
   sample_id: "1"
@@ -50,6 +52,7 @@
   normalized_channel: release
   normalized_country_code: CA
   normalized_os: Android
+  normalized_os_version: "29.0"
   ping_info:
     end_time: 2019-12-01T04:13-05:00
   sample_id: "1"

tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/moz-fx-data-shared-prod.org_mozilla_ios_firefox.metrics.yaml

@@ -5,6 +5,7 @@
   normalized_channel: release
   normalized_os: iOS
   sample_id: 1
+  normalized_os_version: "35.0"
   client_info:
     app_display_version: "68.2.1"
     client_id: release_ios