Add Glean iOS Focus and Klar to search metrics (#3285)

correct the column for default search engine add tests and ios to views
2022-10-18 13:52:53 -04:00 · 2022-10-18 13:52:53 -04:00 · 56983f3e8d
--- a/sql/moz-fx-data-shared-prod/search/mobile_search_aggregates/view.sql
+++ b/sql/moz-fx-data-shared-prod/search/mobile_search_aggregates/view.sql
@ -9,4 +9,4 @@ FROM
  `moz-fx-data-shared-prod.search_derived.mobile_search_aggregates_v1`
 WHERE
    -- TODO DENG-245 this will be replaced after a cutover date is determined:
-  app_name NOT IN ('Focus Android Glean', 'Klar Android Glean')
+  app_name NOT IN ('Focus Android Glean', 'Klar Android Glean', 'Focus iOS Glean', 'Klar iOS Glean')
--- a/sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/view.sql
+++ b/sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/view.sql
@ -10,4 +10,4 @@ FROM
  `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
 WHERE
    -- TODO DENG-245 this will be replaced after a cutover date is determined:
-  app_name NOT IN ('Focus Android Glean', 'Klar Android Glean')
+  app_name NOT IN ('Focus Android Glean', 'Klar Android Glean', 'Focus iOS Glean', 'Klar iOS Glean')
--- a/sql/moz-fx-data-shared-prod/search/mobile_search_clients_last_seen/view.sql
+++ b/sql/moz-fx-data-shared-prod/search/mobile_search_clients_last_seen/view.sql
@ -8,4 +8,4 @@ FROM
  `moz-fx-data-shared-prod.search_derived.mobile_search_clients_last_seen_v1`
 WHERE
    -- TODO DENG-245 this will be replaced after a cutover date is determined:
-  app_name NOT IN ('Focus Android Glean', 'Klar Android Glean')
+  app_name NOT IN ('Focus Android Glean', 'Klar Android Glean', 'Focus iOS Glean', 'Klar iOS Glean')
--- a/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql
+++ b/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql
@ -452,6 +452,60 @@ metrics_org_mozilla_klar AS (
  FROM
    org_mozilla_klar.metrics AS org_mozilla_klar_metrics
 ),
+-- metrics for Focus iOS Glean release
+metrics_org_mozilla_ios_focus AS (
+  SELECT
+    DATE(submission_timestamp) AS submission_date,
+    client_info.client_id,
+    normalized_country_code AS country,
+    'Focus iOS Glean' AS app_name,
+    'Focus' AS normalized_app_name,
+    client_info.app_display_version AS app_version,
+    'release' AS channel,
+    normalized_os AS os,
+    client_info.android_sdk_version AS os_version,
+    metrics.string.search_default_engine AS default_search_engine,
+    CAST(NULL AS STRING) AS default_search_engine_submission_url,
+    sample_id,
+    metrics.labeled_counter.browser_search_search_count AS search_count,
+    metrics.labeled_counter.browser_search_ad_clicks AS search_ad_clicks,
+    metrics.labeled_counter.browser_search_in_content AS search_in_content,
+    metrics.labeled_counter.browser_search_with_ads AS search_with_ads,
+    client_info.first_run_date,
+    ping_info.end_time,
+    ping_info.experiments,
+    metrics.counter.browser_total_uri_count,
+    client_info.locale,
+  FROM
+    org_mozilla_ios_focus.metrics AS org_mozilla_ios_focus_metrics
+),
+-- metrics for Klar iOS Glean release
+metrics_org_mozilla_ios_klar AS (
+  SELECT
+    DATE(submission_timestamp) AS submission_date,
+    client_info.client_id,
+    normalized_country_code AS country,
+    'Klar iOS Glean' AS app_name,
+    'Klar' AS normalized_app_name,
+    client_info.app_display_version AS app_version,
+    'release' AS channel,
+    normalized_os AS os,
+    client_info.android_sdk_version AS os_version,
+    metrics.string.search_default_engine AS default_search_engine,
+    CAST(NULL AS STRING) AS default_search_engine_submission_url,
+    sample_id,
+    metrics.labeled_counter.browser_search_search_count AS search_count,
+    metrics.labeled_counter.browser_search_ad_clicks AS search_ad_clicks,
+    metrics.labeled_counter.browser_search_in_content AS search_in_content,
+    metrics.labeled_counter.browser_search_with_ads AS search_with_ads,
+    client_info.first_run_date,
+    ping_info.end_time,
+    ping_info.experiments,
+    metrics.counter.browser_total_uri_count,
+    client_info.locale,
+  FROM
+    org_mozilla_ios_klar.metrics AS org_mozilla_klar_metrics
+),
 fenix_baseline AS (
  SELECT
    *
@ -542,6 +596,18 @@ android_klar_metrics AS (
  FROM
    metrics_org_mozilla_klar
 ),
+ios_focus_metrics AS (
+  SELECT
+    *
+  FROM
+    metrics_org_mozilla_ios_focus
+),
+ios_klar_metrics AS (
+  SELECT
+    *
+  FROM
+    metrics_org_mozilla_ios_klar
+),
 -- iOS organic counts are incorrect until version 34.0
 -- https://github.com/mozilla-mobile/firefox-ios/issues/8412
 ios_organic_filtered AS (
@ -606,6 +672,16 @@ glean_metrics AS (
    *
  FROM
    android_klar_metrics
+  UNION ALL
+  SELECT
+    *
+  FROM
+    ios_focus_metrics
+  UNION ALL
+  SELECT
+    *
+  FROM
+    ios_klar_metrics
 ),
 glean_combined_searches AS (
  SELECT
--- a/sql_generators/search/init.py
+++ b/sql_generators/search/init.py
@ -41,6 +41,14 @@ FOCUS_ANDROID_TUPLES = [
 KLAR_ANDROID_TUPLES = [
    ("org_mozilla_klar",            "Klar Android Glean",     "release"),  # noqa E241 E501
 ]
+
+FOCUS_iOS_TUPLES = [
+    ("org_mozilla_ios_focus",           "Focus iOS Glean",    "release"), # noqa E241 E501
+]
+
+KLAR_iOS_TUPLES = [
+    ("org_mozilla_ios_klar",            "Klar iOS Glean",     "release"),  # noqa E241 E501
+]
 # fmt: on


@ -73,6 +81,8 @@ def generate(output_dir, target_project):
    ios_query_template = env.get_template("ios_metrics.template.sql")
    android_focus_template = env.get_template("android_focus.template.sql")
    android_klar_template = env.get_template("android_klar.template.sql")
+    ios_focus_template = env.get_template("ios_focus.template.sql")
+    ios_klar_template = env.get_template("ios_klar.template.sql")

    firefox_android_queries = [
        android_query_template.render(
@ -102,11 +112,27 @@ def generate(output_dir, target_project):
        for namespace, app_name, channel in KLAR_ANDROID_TUPLES
    ]

+    focus_ios_queries = [
+        ios_focus_template.render(
+            namespace=namespace, app_name=app_name, channel=channel
+        )
+        for namespace, app_name, channel in FOCUS_iOS_TUPLES
+    ]
+
+    klar_ios_queries = [
+        ios_klar_template.render(
+            namespace=namespace, app_name=app_name, channel=channel
+        )
+        for namespace, app_name, channel in KLAR_iOS_TUPLES
+    ]
+
    queries = (
        firefox_android_queries
        + firefox_ios_queries
        + focus_android_queries
        + klar_android_queries
+        + focus_ios_queries
+        + klar_ios_queries
    )

    search_query_template = env.get_template("mobile_search_clients_daily.template.sql")
@ -139,6 +165,19 @@ def generate(output_dir, target_project):
        ]
    )

+    ios_focus_combined_metrics = union_statements(
+        [
+            f"SELECT * FROM metrics_{namespace}"
+            for namespace, _, _ in FOCUS_iOS_TUPLES
+        ]
+    )
+    ios_klar_combined_metrics = union_statements(
+        [
+            f"SELECT * FROM metrics_{namespace}"
+            for namespace, _, _ in KLAR_iOS_TUPLES
+        ]
+    )
+
    search_query = search_query_template.render(
        baseline_and_metrics_by_namespace="\n".join(queries),
        fenix_baseline=fenix_combined_baseline,
@ -146,6 +185,8 @@ def generate(output_dir, target_project):
        ios_metrics=ios_combined_metrics,
        android_focus_metrics=android_focus_combined_metrics,
        android_klar_metrics=android_klar_combined_metrics,
+        ios_focus_metrics=ios_focus_combined_metrics,
+        ios_klar_metrics=ios_klar_combined_metrics,
    )

    print(reformat(search_query))
--- a/sql_generators/search/templates/ios_focus.template.sql
+++ b/sql_generators/search/templates/ios_focus.template.sql
@ -0,0 +1,27 @@
+-- metrics for {{ app_name }} {{ channel }}
+metrics_{{ namespace }} AS (
+    SELECT
+        DATE(submission_timestamp) AS submission_date,
+        client_info.client_id,
+        normalized_country_code AS country,
+        '{{ app_name }}' AS app_name,
+        'Focus' AS normalized_app_name,
+        client_info.app_display_version AS app_version,
+        '{{ channel }}' AS channel,
+        normalized_os AS os,
+        client_info.android_sdk_version AS os_version,
+        metrics.string.search_default_engine AS default_search_engine,
+        CAST(NULL AS STRING) AS default_search_engine_submission_url,
+        sample_id,
+        metrics.labeled_counter.browser_search_search_count AS search_count,
+        metrics.labeled_counter.browser_search_ad_clicks as search_ad_clicks,
+        metrics.labeled_counter.browser_search_in_content AS search_in_content,
+        metrics.labeled_counter.browser_search_with_ads as search_with_ads,
+        client_info.first_run_date,
+        ping_info.end_time,
+        ping_info.experiments,
+        metrics.counter.browser_total_uri_count,
+        client_info.locale,
+    FROM
+        {{ namespace }}.metrics AS {{ namespace }}_metrics
+),
--- a/sql_generators/search/templates/ios_klar.template.sql
+++ b/sql_generators/search/templates/ios_klar.template.sql
@ -0,0 +1,27 @@
+-- metrics for {{ app_name }} {{ channel }}
+metrics_{{ namespace }} AS (
+    SELECT
+        DATE(submission_timestamp) AS submission_date,
+        client_info.client_id,
+        normalized_country_code AS country,
+        '{{ app_name }}' AS app_name,
+        'Klar' AS normalized_app_name,
+        client_info.app_display_version AS app_version,
+        '{{ channel }}' AS channel,
+        normalized_os AS os,
+        client_info.android_sdk_version AS os_version,
+        metrics.string.search_default_engine AS default_search_engine,
+        CAST(NULL AS STRING) AS default_search_engine_submission_url,
+        sample_id,
+        metrics.labeled_counter.browser_search_search_count AS search_count,
+        metrics.labeled_counter.browser_search_ad_clicks as search_ad_clicks,
+        metrics.labeled_counter.browser_search_in_content AS search_in_content,
+        metrics.labeled_counter.browser_search_with_ads as search_with_ads,
+        client_info.first_run_date,
+        ping_info.end_time,
+        ping_info.experiments,
+        metrics.counter.browser_total_uri_count,
+        client_info.locale,
+    FROM
+        {{ namespace }}.metrics AS {{ namespace }}_metrics
+),
--- a/sql_generators/search/templates/mobile_search_clients_daily.template.sql
+++ b/sql_generators/search/templates/mobile_search_clients_daily.template.sql
@ -124,6 +124,12 @@ android_focus_metrics AS (
 android_klar_metrics AS (
  {{ android_klar_metrics }}
 ),
+ios_focus_metrics AS (
+  {{ ios_focus_metrics }}
+),
+ios_klar_metrics AS (
+  {{ ios_klar_metrics }}
+),
 -- iOS organic counts are incorrect until version 34.0
 -- https://github.com/mozilla-mobile/firefox-ios/issues/8412
 ios_organic_filtered AS (
@ -188,6 +194,16 @@ glean_metrics AS (
    *
  FROM
    android_klar_metrics
+  UNION ALL
+  SELECT
+    *
+  FROM
+    ios_focus_metrics
+  UNION ALL
+  SELECT
+    *
+  FROM
+    ios_klar_metrics
 ),
 glean_combined_searches AS (
  SELECT
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/org_mozilla_ios_focus.metrics.schema.json
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/org_mozilla_ios_focus.metrics.schema.json
@ -0,0 +1,191 @@
+[
+    {
+      "fields": [
+        {
+          "type": "STRING",
+          "name": "android_sdk_version"
+        },
+        {
+          "type": "STRING",
+          "name": "locale"
+        },
+        {
+          "type": "STRING",
+          "name": "app_display_version"
+        },
+        {
+          "type": "STRING",
+          "name": "client_id"
+        },
+        {
+          "type": "STRING",
+          "name": "first_run_date"
+        }
+      ],
+      "type": "RECORD",
+      "name": "client_info"
+    },
+    {
+      "fields": [
+        {
+          "fields": [
+            {
+              "fields": [
+                {
+                  "type": "STRING",
+                  "name": "key"
+                },
+                {
+                  "type": "INTEGER",
+                  "name": "value"
+                }
+              ],
+              "type": "RECORD",
+              "name": "browser_search_search_count",
+              "mode": "REPEATED"
+            },
+            {
+              "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018<provider-name>\u2019.\n",
+              "fields": [
+                {
+                  "mode": "NULLABLE",
+                  "name": "key",
+                  "type": "STRING"
+                },
+                {
+                  "mode": "NULLABLE",
+                  "name": "value",
+                  "type": "INTEGER"
+                }
+              ],
+              "mode": "REPEATED",
+              "name": "browser_search_ad_clicks",
+              "type": "RECORD"
+            },
+            {
+              "description": "Records the type of interaction a user has on SERP pages.\n",
+              "fields": [
+                {
+                  "mode": "NULLABLE",
+                  "name": "key",
+                  "type": "STRING"
+                },
+                {
+                  "mode": "NULLABLE",
+                  "name": "value",
+                  "type": "INTEGER"
+                }
+              ],
+              "mode": "REPEATED",
+              "name": "browser_search_in_content",
+              "type": "RECORD"
+            },
+            {
+              "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018<provider-name>\u2019.\n",
+              "fields": [
+                {
+                  "mode": "NULLABLE",
+                  "name": "key",
+                  "type": "STRING"
+                },
+                {
+                  "mode": "NULLABLE",
+                  "name": "value",
+                  "type": "INTEGER"
+                }
+              ],
+              "mode": "REPEATED",
+              "name": "browser_search_with_ads",
+              "type": "RECORD"
+            }
+          ],
+          "type": "RECORD",
+          "name": "labeled_counter"
+        },
+        {
+          "fields": [
+            {
+              "type": "STRING",
+              "name": "search_default_engine"
+            }
+          ],
+          "type": "RECORD",
+          "name": "string"
+        },
+        {
+          "fields": [
+            {
+              "type": "INTEGER",
+              "name": "browser_total_uri_count"
+            }
+          ],
+          "type": "RECORD",
+          "name": "counter"
+        }
+      ],
+      "type": "RECORD",
+      "name": "metrics"
+    },
+    {
+      "type": "STRING",
+      "name": "normalized_channel"
+    },
+    {
+      "type": "STRING",
+      "name": "normalized_country_code"
+    },
+    {
+      "type": "STRING",
+      "name": "normalized_os"
+    },
+    {
+      "fields": [
+        {
+          "type": "STRING",
+          "name": "end_time"
+        },
+        {
+          "fields": [
+            {
+              "name": "key",
+              "type": "STRING"
+            },
+            {
+              "fields": [
+                {
+                  "name": "branch",
+                  "type": "STRING"
+                },
+                {
+                  "fields": [
+                    {
+                      "name": "type",
+                      "type": "STRING"
+                    }
+                  ],
+                  "name": "extra",
+                  "type": "RECORD"
+                }
+              ],
+              "name": "value",
+              "type": "RECORD"
+            }
+          ],
+          "mode": "REPEATED",
+          "name": "experiments",
+          "type": "RECORD"
+        }
+      ],
+      "type": "RECORD",
+      "name": "ping_info"
+    },
+    {
+      "type": "INTEGER",
+      "name": "sample_id"
+    },
+    {
+      "type": "TIMESTAMP",
+      "name": "submission_timestamp"
+    }
+  ]
+  
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/org_mozilla_ios_klar.metrics.schema.json
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/org_mozilla_ios_klar.metrics.schema.json
@ -0,0 +1,190 @@
+[
+  {
+    "fields": [
+      {
+        "type": "STRING",
+        "name": "android_sdk_version"
+      },
+      {
+        "type": "STRING",
+        "name": "locale"
+      },
+      {
+        "type": "STRING",
+        "name": "app_display_version"
+      },
+      {
+        "type": "STRING",
+        "name": "client_id"
+      },
+      {
+        "type": "STRING",
+        "name": "first_run_date"
+      }
+    ],
+    "type": "RECORD",
+    "name": "client_info"
+  },
+  {
+    "fields": [
+      {
+        "fields": [
+          {
+            "fields": [
+              {
+                "type": "STRING",
+                "name": "key"
+              },
+              {
+                "type": "INTEGER",
+                "name": "value"
+              }
+            ],
+            "type": "RECORD",
+            "name": "browser_search_search_count",
+            "mode": "REPEATED"
+          },
+          {
+            "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018<provider-name>\u2019.\n",
+            "fields": [
+              {
+                "mode": "NULLABLE",
+                "name": "key",
+                "type": "STRING"
+              },
+              {
+                "mode": "NULLABLE",
+                "name": "value",
+                "type": "INTEGER"
+              }
+            ],
+            "mode": "REPEATED",
+            "name": "browser_search_ad_clicks",
+            "type": "RECORD"
+          },
+          {
+            "description": "Records the type of interaction a user has on SERP pages.\n",
+            "fields": [
+              {
+                "mode": "NULLABLE",
+                "name": "key",
+                "type": "STRING"
+              },
+              {
+                "mode": "NULLABLE",
+                "name": "value",
+                "type": "INTEGER"
+              }
+            ],
+            "mode": "REPEATED",
+            "name": "browser_search_in_content",
+            "type": "RECORD"
+          },
+          {
+            "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018<provider-name>\u2019.\n",
+            "fields": [
+              {
+                "mode": "NULLABLE",
+                "name": "key",
+                "type": "STRING"
+              },
+              {
+                "mode": "NULLABLE",
+                "name": "value",
+                "type": "INTEGER"
+              }
+            ],
+            "mode": "REPEATED",
+            "name": "browser_search_with_ads",
+            "type": "RECORD"
+          }
+        ],
+        "type": "RECORD",
+        "name": "labeled_counter"
+      },
+      {
+        "fields": [
+          {
+            "type": "STRING",
+            "name": "search_default_engine"
+          }
+        ],
+        "type": "RECORD",
+        "name": "string"
+      },
+      {
+        "fields": [
+          {
+            "type": "INTEGER",
+            "name": "browser_total_uri_count"
+          }
+        ],
+        "type": "RECORD",
+        "name": "counter"
+      }
+    ],
+    "type": "RECORD",
+    "name": "metrics"
+  },
+  {
+    "type": "STRING",
+    "name": "normalized_channel"
+  },
+  {
+    "type": "STRING",
+    "name": "normalized_country_code"
+  },
+  {
+    "type": "STRING",
+    "name": "normalized_os"
+  },
+  {
+    "fields": [
+      {
+        "type": "STRING",
+        "name": "end_time"
+      },
+      {
+        "fields": [
+          {
+            "name": "key",
+            "type": "STRING"
+          },
+          {
+            "fields": [
+              {
+                "name": "branch",
+                "type": "STRING"
+              },
+              {
+                "fields": [
+                  {
+                    "name": "type",
+                    "type": "STRING"
+                  }
+                ],
+                "name": "extra",
+                "type": "RECORD"
+              }
+            ],
+            "name": "value",
+            "type": "RECORD"
+          }
+        ],
+        "mode": "REPEATED",
+        "name": "experiments",
+        "type": "RECORD"
+      }
+    ],
+    "type": "RECORD",
+    "name": "ping_info"
+  },
+  {
+    "type": "INTEGER",
+    "name": "sample_id"
+  },
+  {
+    "type": "TIMESTAMP",
+    "name": "submission_timestamp"
+  }
+]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_focus.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_focus.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_klar.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_klar.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/org_mozilla_ios_focus.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/org_mozilla_ios_focus.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/org_mozilla_ios_klar.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/org_mozilla_ios_klar.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_ios_focus.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_ios_focus.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_ios_klar.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_ios_klar.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_focus.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_focus.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]
--- a/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_klar.metrics.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_klar.metrics.yaml
@ -0,0 +1,2 @@
+---
+[]