From 2156023e16e4a6f7b616da9d93a67e78419dff06 Mon Sep 17 00:00:00 2001 From: Ben Wu Date: Tue, 14 Jul 2020 19:00:37 -0400 Subject: [PATCH] Add new search types, experiments, and uri count to mobile search (#1167) --- sql/search/mobile_search_aggregates/view.sql | 1 + .../mobile_search_clients_daily/view.sql | 1 + .../mobile_search_aggregates_v1/query.sql | 8 +- .../fenix_metrics.template.sql | 20 +- .../mobile_search_clients_daily.template.sql | 178 ++++++++++-- .../mobile_search_clients_daily_v1/query.sql | 267 ++++++++++++++---- .../test_aggregation/expect.ndjson | 6 - .../test_aggregation/expect.yaml | 110 ++++++++ .../mobile_search_clients_daily_v1.ndjson | 10 - ...mobile_search_clients_daily_v1.schema.json | 50 ++++ .../mobile_search_clients_daily_v1.yaml | 211 ++++++++++++++ .../test_aggregation/expect.yaml | 84 ++++++ .../org_mozilla_fenix.metrics.schema.json | 95 +++++++ .../org_mozilla_fenix.metrics.yaml | 76 ++++- ..._mozilla_fenix_nightly.metrics.schema.json | 95 +++++++ ..._mozilla_fennec_aurora.metrics.schema.json | 95 +++++++ .../org_mozilla_firefox.metrics.schema.json | 95 +++++++ ...g_mozilla_firefox_beta.metrics.schema.json | 95 +++++++ .../telemetry.core.schema.json | 5 + .../test_aggregation/telemetry.core.yaml | 2 + .../test_namespace_union/expect.yaml | 7 + .../org_mozilla_fenix.metrics.schema.json | 95 +++++++ ..._mozilla_fenix_nightly.metrics.schema.json | 95 +++++++ ..._mozilla_fennec_aurora.metrics.schema.json | 95 +++++++ .../org_mozilla_firefox.metrics.schema.json | 95 +++++++ ...g_mozilla_firefox_beta.metrics.schema.json | 95 +++++++ .../telemetry.core.schema.json | 5 + .../test_overactive_filter/expect.yaml | 33 +-- .../org_mozilla_fenix.baseline.yaml | 9 +- .../org_mozilla_fenix.metrics.schema.json | 95 +++++++ ..._mozilla_fenix_nightly.metrics.schema.json | 95 +++++++ ..._mozilla_fennec_aurora.metrics.schema.json | 95 +++++++ .../org_mozilla_firefox.metrics.schema.json | 95 +++++++ ...g_mozilla_firefox_beta.metrics.schema.json | 95 +++++++ .../telemetry.core.schema.json | 5 + udf/array_slice.sql | 23 ++ 36 files changed, 2411 insertions(+), 125 deletions(-) delete mode 100644 tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.ndjson create mode 100644 tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.yaml delete mode 100644 tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.ndjson create mode 100644 tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.yaml create mode 100644 udf/array_slice.sql diff --git a/sql/search/mobile_search_aggregates/view.sql b/sql/search/mobile_search_aggregates/view.sql index b0de601e41..1f589da4dc 100755 --- a/sql/search/mobile_search_aggregates/view.sql +++ b/sql/search/mobile_search_aggregates/view.sql @@ -4,5 +4,6 @@ AS SELECT * EXCEPT (normalized_engine), `moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine, + search_count AS sap, FROM `moz-fx-data-shared-prod.search_derived.mobile_search_aggregates_v1` diff --git a/sql/search/mobile_search_clients_daily/view.sql b/sql/search/mobile_search_clients_daily/view.sql index 424c270f65..567324d979 100755 --- a/sql/search/mobile_search_clients_daily/view.sql +++ b/sql/search/mobile_search_clients_daily/view.sql @@ -4,5 +4,6 @@ AS SELECT * EXCEPT (normalized_engine), `moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine, + search_count AS sap, FROM `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1` diff --git a/sql/search_derived/mobile_search_aggregates_v1/query.sql b/sql/search_derived/mobile_search_aggregates_v1/query.sql index 2c82e20d9a..72207137bc 100755 --- a/sql/search_derived/mobile_search_aggregates_v1/query.sql +++ b/sql/search_derived/mobile_search_aggregates_v1/query.sql @@ -13,7 +13,13 @@ SELECT os, os_version, COUNT(*) AS client_count, - SUM(search_count) AS search_count + SUM(search_count) AS search_count, + SUM(organic) AS organic, + SUM(tagged_sap) AS tagged_sap, + SUM(tagged_follow_on) AS tagged_follow_on, + SUM(ad_click) AS ad_click, + SUM(search_with_ads) AS search_with_ads, + SUM(unknown) AS unknown, FROM mobile_search_clients_daily_v1 WHERE diff --git a/sql/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql b/sql/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql index 71af0ba966..aab7b7dc56 100644 --- a/sql/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql +++ b/sql/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql @@ -12,20 +12,24 @@ metrics_{namespace} AS ( SELECT DATE(submission_timestamp) AS submission_date, client_info.client_id, - normalized_country_code, + normalized_country_code AS country, '{app_name}' AS app_name, 'Fenix' AS normalized_app_name, - client_info.app_display_version, + client_info.app_display_version AS app_version, '{channel}' AS channel, - normalized_os, - client_info.android_sdk_version, - metrics.string.search_default_engine_code, - metrics.string.search_default_engine_submission_url, + normalized_os AS os, + client_info.android_sdk_version AS os_version, + metrics.string.search_default_engine_code AS default_search_engine, + metrics.string.search_default_engine_submission_url AS default_search_engine_submission_url, sample_id, - CAST(NULL AS STRING) AS distribution_id, metrics.labeled_counter.metrics_search_count, + metrics.labeled_counter.browser_search_ad_clicks, + metrics.labeled_counter.browser_search_in_content, + metrics.labeled_counter.browser_search_with_ads, client_info.first_run_date, - ping_info.end_time + ping_info.end_time, + ping_info.experiments, + metrics.counter.events_total_uri_count AS total_uri_count, FROM {namespace}.metrics AS {namespace}_metrics ), diff --git a/sql/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql b/sql/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql index ebd0c3f217..fc7ca7a161 100755 --- a/sql/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql +++ b/sql/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql @@ -36,6 +36,39 @@ CREATE TEMP FUNCTION normalize_core_search_key(key STRING) AS ( END ); +CREATE TEMP FUNCTION normalize_fenix_experiments(experiments ANY TYPE) AS ( + ARRAY( + SELECT AS STRUCT + key, + value.branch AS value, + FROM + UNNEST(experiments) + ) +); + +CREATE TEMP FUNCTION normalize_core_experiments(experiments ANY TYPE) AS ( + ARRAY( + SELECT AS STRUCT + key, + CAST(NULL AS STRING) AS value, + FROM + UNNEST(experiments) AS key + ) +); + +-- Add search type value to each element of an array +CREATE TEMP FUNCTION + add_search_type(list ANY TYPE, search_type STRING) AS ( + ARRAY( + SELECT AS STRUCT + *, + search_type, + FROM + UNNEST(list) + ) +); + + CREATE TEMP FUNCTION null_search() AS ( [STRUCT(NULL, 0)] ); @@ -73,19 +106,68 @@ fenix_client_locales AS ( GROUP BY client_id ), +fenix_combined_searches AS ( + SELECT + * EXCEPT ( + metrics_search_count, browser_search_ad_clicks, + browser_search_in_content, browser_search_with_ads + ), + ARRAY_CONCAT( + add_search_type(metrics_search_count, 'sap'), + add_search_type(browser_search_in_content, 'in-content'), + add_search_type(browser_search_ad_clicks, 'ad-click'), + add_search_type(browser_search_with_ads, 'search-with-ads') + ) AS searches, + FROM + fenix_metrics +), fenix_flattened_searches AS ( SELECT *, - normalize_fenix_search_key(searches.key) AS normalized_search_key, - searches.value AS search_count, + + CASE + WHEN + search.search_type = 'sap' + THEN + normalize_fenix_search_key(search.key)[SAFE_OFFSET(0)] + WHEN + search.search_type = 'in-content' + THEN + SPLIT(search.key, '.')[SAFE_OFFSET(0)] + WHEN + search.search_type = 'ad-click' OR search.search_type = 'search-with-ads' + THEN + search.key + ELSE + NULL + END AS engine, + + CASE + WHEN + search.search_type = 'sap' + THEN + normalize_fenix_search_key(search.key)[SAFE_OFFSET(1)] + WHEN + search.search_type = 'in-content' + THEN + IF( + ARRAY_LENGTH(SPLIT(search.key, '.')) < 2, + NULL, + ARRAY_TO_STRING(udf.array_slice(SPLIT(search.key, '.'), 1, 3), '.') + ) + ELSE + search.search_type + END AS source, + + search.value AS search_count, UNIX_DATE(udf.parse_iso8601_date(first_run_date)) AS profile_creation_date, SAFE.DATE_DIFF( udf.parse_iso8601_date(end_time), udf.parse_iso8601_date(first_run_date), DAY - ) AS profile_age_in_days + ) AS profile_age_in_days, FROM - fenix_metrics + fenix_combined_searches LEFT JOIN fenix_client_locales USING @@ -94,17 +176,19 @@ fenix_flattened_searches AS ( UNNEST( -- Add a null search to pings that have no searches IF( - ARRAY_LENGTH(metrics_search_count) = 0, - null_search(), - metrics_search_count + ARRAY_LENGTH(searches) = 0, + add_search_type(null_search(), CAST(NULL AS STRING)), + searches ) - ) AS searches + ) AS search ), combined_search_clients AS ( SELECT DATE(submission_timestamp) AS submission_date, client_id, - normalized_search_key, + normalized_search_key[SAFE_OFFSET(0)] AS engine, + normalized_search_key[SAFE_OFFSET(1)] AS source, + 'sap' AS search_type, search_count, normalized_country_code AS country, locale, @@ -119,29 +203,58 @@ combined_search_clients AS ( distribution_id, profile_date AS profile_creation_date, profile_age_in_days, - sample_id + sample_id, + normalize_core_experiments(experiments) AS experiments, + CAST(NULL AS INT64) AS total_uri_count, FROM core_flattened_searches UNION ALL SELECT submission_date, client_id, - normalized_search_key, + engine, + source, + + CASE + WHEN + STARTS_WITH(source, 'in-content.sap.') + THEN + 'tagged-sap' + WHEN + STARTS_WITH(source, 'in-content.sap-follow-on.') + THEN + 'tagged-follow-on' + WHEN + STARTS_WITH(source, 'in-content.organic') + THEN + 'organic' + WHEN + search_type = 'ad-click' + OR search_type = 'search-with-ads' + OR search_type = 'sap' + THEN + search_type + ELSE + 'unknown' + END AS search_type, + search_count, - normalized_country_code AS country, + country, locale, app_name, normalized_app_name, - app_display_version AS app_version, + app_version, channel, - normalized_os AS os, - android_sdk_version AS os_version, - search_default_engine_code AS default_search_engine, - search_default_engine_submission_url AS default_search_engine_submission_url, + os, + os_version, + default_search_engine, + default_search_engine_submission_url, CAST(NULL AS STRING) AS distribution_id, profile_creation_date, profile_age_in_days, - sample_id + sample_id, + normalize_fenix_experiments(experiments) AS experiments, + total_uri_count, FROM fenix_flattened_searches ), @@ -149,13 +262,31 @@ unfiltered_search_clients AS ( SELECT submission_date, client_id, - IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(0)]) AS engine, - IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(1)]) AS source, + IF(search_count > 10000, NULL, engine) AS engine, + IF(search_count > 10000, NULL, source) AS source, app_name, normalized_app_name, SUM( - IF(ARRAY_LENGTH(normalized_search_key) = 0 OR search_count > 10000, 0, search_count) + IF(search_type != 'sap' OR engine IS NULL OR search_count > 10000, 0, search_count) ) AS search_count, + SUM( + IF(search_type != 'organic' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS organic, + SUM( + IF(search_type != 'tagged-sap' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS tagged_sap, + SUM( + IF(search_type != 'tagged-follow-on' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS tagged_follow_on, + SUM( + IF(search_type != 'ad-click' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS ad_click, + SUM( + IF(search_type != 'search-with-ads' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS search_with_ads, + SUM( + IF(search_type != 'unknown' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS unknown, udf.mode_last(ARRAY_AGG(country)) AS country, udf.mode_last(ARRAY_AGG(locale)) AS locale, udf.mode_last(ARRAY_AGG(app_version)) AS app_version, @@ -169,7 +300,9 @@ unfiltered_search_clients AS ( udf.mode_last(ARRAY_AGG(distribution_id)) AS distribution_id, udf.mode_last(ARRAY_AGG(profile_creation_date)) AS profile_creation_date, udf.mode_last(ARRAY_AGG(profile_age_in_days)) AS profile_age_in_days, - udf.mode_last(ARRAY_AGG(sample_id)) AS sample_id + ANY_VALUE(sample_id) AS sample_id, + udf.map_mode_last(ARRAY_CONCAT_AGG(experiments)) AS experiments, + SUM(total_uri_count) AS total_uri_count, FROM combined_search_clients WHERE @@ -179,6 +312,7 @@ unfiltered_search_clients AS ( client_id, engine, source, + search_type, app_name, normalized_app_name ) diff --git a/sql/search_derived/mobile_search_clients_daily_v1/query.sql b/sql/search_derived/mobile_search_clients_daily_v1/query.sql index 2013cff72a..e5b61ad9e4 100755 --- a/sql/search_derived/mobile_search_clients_daily_v1/query.sql +++ b/sql/search_derived/mobile_search_clients_daily_v1/query.sql @@ -36,6 +36,19 @@ CREATE TEMP FUNCTION normalize_core_search_key(key STRING) AS ( END ); +CREATE TEMP FUNCTION normalize_fenix_experiments(experiments ANY TYPE) AS ( + ARRAY(SELECT AS STRUCT key, value.branch AS value, FROM UNNEST(experiments)) +); + +CREATE TEMP FUNCTION normalize_core_experiments(experiments ANY TYPE) AS ( + ARRAY(SELECT AS STRUCT key, CAST(NULL AS STRING) AS value, FROM UNNEST(experiments) AS key) +); + +-- Add search type value to each element of an array +CREATE TEMP FUNCTION add_search_type(list ANY TYPE, search_type STRING) AS ( + ARRAY(SELECT AS STRUCT *, search_type, FROM UNNEST(list)) +); + CREATE TEMP FUNCTION null_search() AS ( [STRUCT(NULL, 0)] ); @@ -68,20 +81,24 @@ metrics_org_mozilla_fenix AS ( SELECT DATE(submission_timestamp) AS submission_date, client_info.client_id, - normalized_country_code, + normalized_country_code AS country, 'Firefox Preview' AS app_name, 'Fenix' AS normalized_app_name, - client_info.app_display_version, + client_info.app_display_version AS app_version, 'beta' AS channel, - normalized_os, - client_info.android_sdk_version, - metrics.string.search_default_engine_code, - metrics.string.search_default_engine_submission_url, + normalized_os AS os, + client_info.android_sdk_version AS os_version, + metrics.string.search_default_engine_code AS default_search_engine, + metrics.string.search_default_engine_submission_url AS default_search_engine_submission_url, sample_id, - CAST(NULL AS STRING) AS distribution_id, metrics.labeled_counter.metrics_search_count, + metrics.labeled_counter.browser_search_ad_clicks, + metrics.labeled_counter.browser_search_in_content, + metrics.labeled_counter.browser_search_with_ads, client_info.first_run_date, - ping_info.end_time + ping_info.end_time, + ping_info.experiments, + metrics.counter.events_total_uri_count AS total_uri_count, FROM org_mozilla_fenix.metrics AS org_mozilla_fenix_metrics ), @@ -99,20 +116,24 @@ metrics_org_mozilla_fenix_nightly AS ( SELECT DATE(submission_timestamp) AS submission_date, client_info.client_id, - normalized_country_code, + normalized_country_code AS country, 'Firefox Preview' AS app_name, 'Fenix' AS normalized_app_name, - client_info.app_display_version, + client_info.app_display_version AS app_version, 'nightly' AS channel, - normalized_os, - client_info.android_sdk_version, - metrics.string.search_default_engine_code, - metrics.string.search_default_engine_submission_url, + normalized_os AS os, + client_info.android_sdk_version AS os_version, + metrics.string.search_default_engine_code AS default_search_engine, + metrics.string.search_default_engine_submission_url AS default_search_engine_submission_url, sample_id, - CAST(NULL AS STRING) AS distribution_id, metrics.labeled_counter.metrics_search_count, + metrics.labeled_counter.browser_search_ad_clicks, + metrics.labeled_counter.browser_search_in_content, + metrics.labeled_counter.browser_search_with_ads, client_info.first_run_date, - ping_info.end_time + ping_info.end_time, + ping_info.experiments, + metrics.counter.events_total_uri_count AS total_uri_count, FROM org_mozilla_fenix_nightly.metrics AS org_mozilla_fenix_nightly_metrics ), @@ -130,20 +151,24 @@ metrics_org_mozilla_fennec_aurora AS ( SELECT DATE(submission_timestamp) AS submission_date, client_info.client_id, - normalized_country_code, + normalized_country_code AS country, 'Fenix' AS app_name, 'Fenix' AS normalized_app_name, - client_info.app_display_version, + client_info.app_display_version AS app_version, 'nightly' AS channel, - normalized_os, - client_info.android_sdk_version, - metrics.string.search_default_engine_code, - metrics.string.search_default_engine_submission_url, + normalized_os AS os, + client_info.android_sdk_version AS os_version, + metrics.string.search_default_engine_code AS default_search_engine, + metrics.string.search_default_engine_submission_url AS default_search_engine_submission_url, sample_id, - CAST(NULL AS STRING) AS distribution_id, metrics.labeled_counter.metrics_search_count, + metrics.labeled_counter.browser_search_ad_clicks, + metrics.labeled_counter.browser_search_in_content, + metrics.labeled_counter.browser_search_with_ads, client_info.first_run_date, - ping_info.end_time + ping_info.end_time, + ping_info.experiments, + metrics.counter.events_total_uri_count AS total_uri_count, FROM org_mozilla_fennec_aurora.metrics AS org_mozilla_fennec_aurora_metrics ), @@ -161,20 +186,24 @@ metrics_org_mozilla_firefox_beta AS ( SELECT DATE(submission_timestamp) AS submission_date, client_info.client_id, - normalized_country_code, + normalized_country_code AS country, 'Fenix' AS app_name, 'Fenix' AS normalized_app_name, - client_info.app_display_version, + client_info.app_display_version AS app_version, 'beta' AS channel, - normalized_os, - client_info.android_sdk_version, - metrics.string.search_default_engine_code, - metrics.string.search_default_engine_submission_url, + normalized_os AS os, + client_info.android_sdk_version AS os_version, + metrics.string.search_default_engine_code AS default_search_engine, + metrics.string.search_default_engine_submission_url AS default_search_engine_submission_url, sample_id, - CAST(NULL AS STRING) AS distribution_id, metrics.labeled_counter.metrics_search_count, + metrics.labeled_counter.browser_search_ad_clicks, + metrics.labeled_counter.browser_search_in_content, + metrics.labeled_counter.browser_search_with_ads, client_info.first_run_date, - ping_info.end_time + ping_info.end_time, + ping_info.experiments, + metrics.counter.events_total_uri_count AS total_uri_count, FROM org_mozilla_firefox_beta.metrics AS org_mozilla_firefox_beta_metrics ), @@ -192,20 +221,24 @@ metrics_org_mozilla_firefox AS ( SELECT DATE(submission_timestamp) AS submission_date, client_info.client_id, - normalized_country_code, + normalized_country_code AS country, 'Fenix' AS app_name, 'Fenix' AS normalized_app_name, - client_info.app_display_version, + client_info.app_display_version AS app_version, 'release' AS channel, - normalized_os, - client_info.android_sdk_version, - metrics.string.search_default_engine_code, - metrics.string.search_default_engine_submission_url, + normalized_os AS os, + client_info.android_sdk_version AS os_version, + metrics.string.search_default_engine_code AS default_search_engine, + metrics.string.search_default_engine_submission_url AS default_search_engine_submission_url, sample_id, - CAST(NULL AS STRING) AS distribution_id, metrics.labeled_counter.metrics_search_count, + metrics.labeled_counter.browser_search_ad_clicks, + metrics.labeled_counter.browser_search_in_content, + metrics.labeled_counter.browser_search_with_ads, client_info.first_run_date, - ping_info.end_time + ping_info.end_time, + ping_info.experiments, + metrics.counter.events_total_uri_count AS total_uri_count, FROM org_mozilla_firefox.metrics AS org_mozilla_firefox_metrics ), @@ -273,19 +306,70 @@ fenix_client_locales AS ( GROUP BY client_id ), +fenix_combined_searches AS ( + SELECT + * EXCEPT ( + metrics_search_count, + browser_search_ad_clicks, + browser_search_in_content, + browser_search_with_ads + ), + ARRAY_CONCAT( + add_search_type(metrics_search_count, 'sap'), + add_search_type(browser_search_in_content, 'in-content'), + add_search_type(browser_search_ad_clicks, 'ad-click'), + add_search_type(browser_search_with_ads, 'search-with-ads') + ) AS searches, + FROM + fenix_metrics +), fenix_flattened_searches AS ( SELECT *, - normalize_fenix_search_key(searches.key) AS normalized_search_key, - searches.value AS search_count, + CASE + WHEN + search.search_type = 'sap' + THEN + normalize_fenix_search_key(search.key)[SAFE_OFFSET(0)] + WHEN + search.search_type = 'in-content' + THEN + SPLIT(search.key, '.')[SAFE_OFFSET(0)] + WHEN + search.search_type = 'ad-click' + OR search.search_type = 'search-with-ads' + THEN + search.key + ELSE + NULL + END + AS engine, + CASE + WHEN + search.search_type = 'sap' + THEN + normalize_fenix_search_key(search.key)[SAFE_OFFSET(1)] + WHEN + search.search_type = 'in-content' + THEN + IF( + ARRAY_LENGTH(SPLIT(search.key, '.')) < 2, + NULL, + ARRAY_TO_STRING(udf.array_slice(SPLIT(search.key, '.'), 1, 3), '.') + ) + ELSE + search.search_type + END + AS source, + search.value AS search_count, UNIX_DATE(udf.parse_iso8601_date(first_run_date)) AS profile_creation_date, SAFE.DATE_DIFF( udf.parse_iso8601_date(end_time), udf.parse_iso8601_date(first_run_date), DAY - ) AS profile_age_in_days + ) AS profile_age_in_days, FROM - fenix_metrics + fenix_combined_searches LEFT JOIN fenix_client_locales USING @@ -293,14 +377,16 @@ fenix_flattened_searches AS ( CROSS JOIN UNNEST( -- Add a null search to pings that have no searches - IF(ARRAY_LENGTH(metrics_search_count) = 0, null_search(), metrics_search_count) - ) AS searches + IF(ARRAY_LENGTH(searches) = 0, add_search_type(null_search(), CAST(NULL AS STRING)), searches) + ) AS search ), combined_search_clients AS ( SELECT DATE(submission_timestamp) AS submission_date, client_id, - normalized_search_key, + normalized_search_key[SAFE_OFFSET(0)] AS engine, + normalized_search_key[SAFE_OFFSET(1)] AS source, + 'sap' AS search_type, search_count, normalized_country_code AS country, locale, @@ -315,29 +401,57 @@ combined_search_clients AS ( distribution_id, profile_date AS profile_creation_date, profile_age_in_days, - sample_id + sample_id, + normalize_core_experiments(experiments) AS experiments, + CAST(NULL AS INT64) AS total_uri_count, FROM core_flattened_searches UNION ALL SELECT submission_date, client_id, - normalized_search_key, + engine, + source, + CASE + WHEN + STARTS_WITH(source, 'in-content.sap.') + THEN + 'tagged-sap' + WHEN + STARTS_WITH(source, 'in-content.sap-follow-on.') + THEN + 'tagged-follow-on' + WHEN + STARTS_WITH(source, 'in-content.organic') + THEN + 'organic' + WHEN + search_type = 'ad-click' + OR search_type = 'search-with-ads' + OR search_type = 'sap' + THEN + search_type + ELSE + 'unknown' + END + AS search_type, search_count, - normalized_country_code AS country, + country, locale, app_name, normalized_app_name, - app_display_version AS app_version, + app_version, channel, - normalized_os AS os, - android_sdk_version AS os_version, - search_default_engine_code AS default_search_engine, - search_default_engine_submission_url AS default_search_engine_submission_url, + os, + os_version, + default_search_engine, + default_search_engine_submission_url, CAST(NULL AS STRING) AS distribution_id, profile_creation_date, profile_age_in_days, - sample_id + sample_id, + normalize_fenix_experiments(experiments) AS experiments, + total_uri_count, FROM fenix_flattened_searches ), @@ -345,13 +459,43 @@ unfiltered_search_clients AS ( SELECT submission_date, client_id, - IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(0)]) AS engine, - IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(1)]) AS source, + IF(search_count > 10000, NULL, engine) AS engine, + IF(search_count > 10000, NULL, source) AS source, app_name, normalized_app_name, SUM( - IF(ARRAY_LENGTH(normalized_search_key) = 0 OR search_count > 10000, 0, search_count) + IF(search_type != 'sap' OR engine IS NULL OR search_count > 10000, 0, search_count) ) AS search_count, + SUM( + IF(search_type != 'organic' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS organic, + SUM( + IF(search_type != 'tagged-sap' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS tagged_sap, + SUM( + IF( + search_type != 'tagged-follow-on' + OR engine IS NULL + OR search_count > 10000, + 0, + search_count + ) + ) AS tagged_follow_on, + SUM( + IF(search_type != 'ad-click' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS ad_click, + SUM( + IF( + search_type != 'search-with-ads' + OR engine IS NULL + OR search_count > 10000, + 0, + search_count + ) + ) AS search_with_ads, + SUM( + IF(search_type != 'unknown' OR engine IS NULL OR search_count > 10000, 0, search_count) + ) AS unknown, udf.mode_last(ARRAY_AGG(country)) AS country, udf.mode_last(ARRAY_AGG(locale)) AS locale, udf.mode_last(ARRAY_AGG(app_version)) AS app_version, @@ -365,7 +509,9 @@ unfiltered_search_clients AS ( udf.mode_last(ARRAY_AGG(distribution_id)) AS distribution_id, udf.mode_last(ARRAY_AGG(profile_creation_date)) AS profile_creation_date, udf.mode_last(ARRAY_AGG(profile_age_in_days)) AS profile_age_in_days, - udf.mode_last(ARRAY_AGG(sample_id)) AS sample_id + ANY_VALUE(sample_id) AS sample_id, + udf.map_mode_last(ARRAY_CONCAT_AGG(experiments)) AS experiments, + SUM(total_uri_count) AS total_uri_count, FROM combined_search_clients WHERE @@ -375,6 +521,7 @@ unfiltered_search_clients AS ( client_id, engine, source, + search_type, app_name, normalized_app_name ) diff --git a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.ndjson b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.ndjson deleted file mode 100644 index a9dc2d0b06..0000000000 --- a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.ndjson +++ /dev/null @@ -1,6 +0,0 @@ -{ "submission_date": "2019-12-01", "engine": "engine1", "source": "action", "app_name": "Fenix", "normalized_app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 5 } -{ "submission_date": "2019-12-01", "engine": "engine1", "source": "suggestion", "app_name": "Fenix", "normalized_app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 1 } -{ "submission_date": "2019-12-01", "engine": "engine2", "source": "suggestion", "app_name": "Fenix", "normalized_app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 5 } -{ "submission_date": "2019-12-01", "engine": "engine2", "source": "suggestion", "app_name": "Fenix", "normalized_app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "US", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 3 } -{ "submission_date": "2019-12-01", "engine": "engine1", "source": "actionbar", "app_name": "Fennec", "normalized_app_name": "Fennec", "app_version": "68.2.1", "country": "CA", "locale": "en-CA", "default_search_engine": "engine1", "os": "Android", "os_version": "26", "client_count": 1, "search_count": 3 } -{ "submission_date": "2019-12-01", "engine": "engine2", "source": "actionbar", "app_name": "Fennec", "normalized_app_name": "Fennec", "app_version": "68.2.1", "country": "CA", "locale": "en-CA", "default_search_engine": "engine1", "os": "Android", "os_version": "26", "client_count": 1, "search_count": 1 } diff --git a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.yaml b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.yaml new file mode 100644 index 0000000000..4ab708adad --- /dev/null +++ b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.yaml @@ -0,0 +1,110 @@ +--- +- &base + submission_date: '2019-12-01' + engine: engine1 + source: action + app_name: Fenix + normalized_app_name: Fenix + app_version: Nightly 191124 06:01 + channel: release + country: CA + locale: en-CA + os: Android + os_version: '29' + client_count: 1 + search_count: 5 + organic: 0 + tagged_sap: 0 + tagged_follow_on: 0 + ad_click: 0 + search_with_ads: 0 + unknown: 0 +- <<: *base + engine: engine1 + source: suggestion + app_name: Fenix + normalized_app_name: Fenix + app_version: Nightly 191124 06:01 + channel: release + country: CA + locale: en-CA + os_version: '29' + client_count: 1 + search_count: 1 +- <<: *base + engine: engine2 + source: suggestion + app_name: Fenix + normalized_app_name: Fenix + app_version: Nightly 191124 06:01 + channel: release + country: CA + locale: en-CA + os_version: '29' + client_count: 1 + search_count: 5 +- <<: *base + engine: engine2 + source: suggestion + app_name: Fenix + normalized_app_name: Fenix + app_version: Nightly 191124 06:01 + channel: release + country: US + os_version: '29' + client_count: 1 + search_count: 3 +- <<: *base + engine: engine1 + source: in-content.organic.none + app_name: Fennec + normalized_app_name: Fennec + app_version: 68.2.1 + country: CA + locale: en-CA + default_search_engine: engine1 + os_version: '26' + client_count: 1 + search_count: 0 + organic: 4 +- <<: *base + engine: engine2 + source: actionbar + app_name: Fennec + normalized_app_name: Fennec + app_version: 68.2.1 + country: CA + locale: en-CA + default_search_engine: engine1 + os_version: '26' + client_count: 1 + search_count: 1 +- <<: *base + engine: engine1 + source: actionbar + app_name: Fennec + normalized_app_name: Fennec + app_version: 68.2.1 + country: CA + locale: en-CA + default_search_engine: engine1 + os_version: '26' + client_count: 1 + search_count: 3 +- <<: *base + engine: engine2 + source: in-content.sap.none + app_name: Fennec + normalized_app_name: Fennec + app_version: 68.2.1 + country: CA + locale: en-CA + default_search_engine: engine1 + os_version: '26' + client_count: 2 + search_count: 0 + tagged_sap: 5 + tagged_follow_on: 7 + ad_click: 9 + search_with_ads: 11 + unknown: 13 diff --git a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.ndjson b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.ndjson deleted file mode 100644 index 1095942fed..0000000000 --- a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.ndjson +++ /dev/null @@ -1,10 +0,0 @@ -{"submission_date": "2019-12-01", "client_id": "a", "search_count": 0, "locale": "fr-FR", "app_version": "68.2.1", "app_name": "Fennec", "normalized_app_name": "Fennec", "country": "FR", "os": "Android", "os_version": "23", "profile_creation_date": 16606, "profile_age_in_days": 1625, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine1", "normalized_engine": "Other", "source": "actionbar", "search_count": 3, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "normalized_app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "c", "engine": "engine2", "normalized_engine": "Other", "source": "actionbar", "search_count": 1, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "normalized_app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "d", "search_count": 0, "default_search_engine": "engine2", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "normalized_app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17653, "profile_age_in_days": 578, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "e", "channel": "release", "search_count": 0, "locale": "en-US", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "normalized_app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18207, "profile_age_in_days": 24, "sample_id": 1, "default_search_engine": "engine1", "default_search_engine_submission_url": "engine1.url"} -{"submission_date": "2019-12-01", "client_id": "f", "channel": "release", "engine": "engine1", "normalized_engine": "Other", "source": "action", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "normalized_app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "g", "channel": "release", "engine": "engine1", "normalized_engine": "Other", "source": "suggestion", "search_count": 1, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "normalized_app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "h", "channel": "release", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "normalized_app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "i", "channel": "release", "search_count": 0, "locale": "ar-EG", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "normalized_app_name": "Fenix", "country": "EG", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1} -{"submission_date": "2019-12-01", "client_id": "h", "channel": "release", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "search_count": 3, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "normalized_app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1} diff --git a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.schema.json b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.schema.json index 98765566b4..2ad6422afc 100644 --- a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.schema.json +++ b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.schema.json @@ -34,6 +34,36 @@ "name": "search_count", "mode": "NULLABLE" }, + { + "type": "INTEGER", + "name": "organic", + "mode": "NULLABLE" + }, + { + "type": "INTEGER", + "name": "tagged_sap", + "mode": "NULLABLE" + }, + { + "type": "INTEGER", + "name": "tagged_follow_on", + "mode": "NULLABLE" + }, + { + "type": "INTEGER", + "name": "ad_click", + "mode": "NULLABLE" + }, + { + "type": "INTEGER", + "name": "search_with_ads", + "mode": "NULLABLE" + }, + { + "type": "INTEGER", + "name": "unknown", + "mode": "NULLABLE" + }, { "type": "STRING", "name": "locale", @@ -98,5 +128,25 @@ "type": "INTEGER", "name": "sample_id", "mode": "NULLABLE" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" + }, + { + "type": "INTEGER", + "name": "total_uri_count", + "mode": "NULLABLE" } ] diff --git a/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.yaml b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.yaml new file mode 100644 index 0000000000..7966e03be6 --- /dev/null +++ b/tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.yaml @@ -0,0 +1,211 @@ +--- +- &base + submission_date: '2019-12-01' + client_id: a + channel: release + search_count: 0 + organic: 0 + tagged_sap: 0 + tagged_follow_on: 0 + ad_click: 0 + search_with_ads: 0 + unknown: 0 + locale: fr-FR + app_version: 68.2.1 + app_name: Fennec + normalized_app_name: Fennec + country: FR + os: Android + os_version: '23' + profile_creation_date: 16606 + profile_age_in_days: 1625 + sample_id: 1 + experiments: [] +- <<: *base + client_id: b + engine: engine1 + normalized_engine: Other + source: actionbar + search_count: 3 + default_search_engine: engine1 + locale: en-CA + app_version: 68.2.1 + app_name: Fennec + normalized_app_name: Fennec + country: CA + os_version: '26' + profile_creation_date: 17332 + profile_age_in_days: 899 + sample_id: 1 +- <<: *base + client_id: b + engine: engine1 + normalized_engine: Other + source: in-content.organic.none + organic: 4 + default_search_engine: engine1 + locale: en-CA + app_version: 68.2.1 + app_name: Fennec + normalized_app_name: Fennec + country: CA + os_version: '26' + profile_creation_date: 17332 + profile_age_in_days: 899 + sample_id: 1 +- <<: *base + client_id: c + engine: engine2 + normalized_engine: Other + source: actionbar + search_count: 1 + default_search_engine: engine1 + locale: en-CA + app_version: 68.2.1 + app_name: Fennec + normalized_app_name: Fennec + country: CA + os_version: '26' + profile_creation_date: 17332 + profile_age_in_days: 899 + sample_id: 1 +- <<: *base + client_id: c + engine: engine2 + normalized_engine: Other + source: in-content.sap.none + tagged_sap: 2 + tagged_follow_on: 3 + ad_click: 4 + search_with_ads: 5 + unknown: 6 + default_search_engine: engine1 + locale: en-CA + app_version: 68.2.1 + app_name: Fennec + normalized_app_name: Fennec + country: CA + os_version: '26' + profile_creation_date: 17332 + profile_age_in_days: 899 + sample_id: 1 +- <<: *base + client_id: d + engine: engine2 + normalized_engine: Other + source: in-content.sap.none + tagged_sap: 3 + tagged_follow_on: 4 + ad_click: 5 + search_with_ads: 6 + unknown: 7 + default_search_engine: engine1 + locale: en-CA + app_version: 68.2.1 + app_name: Fennec + normalized_app_name: Fennec + country: CA + os_version: '26' + profile_creation_date: 17332 + profile_age_in_days: 899 + sample_id: 1 +- <<: *base + client_id: d + search_count: 0 + default_search_engine: engine2 + locale: en-CA + app_version: 68.2.1 + app_name: Fennec + normalized_app_name: Fennec + country: CA + os_version: '26' + profile_creation_date: 17653 + profile_age_in_days: 578 + sample_id: 1 +- <<: *base + client_id: e + search_count: 0 + locale: en-US + app_version: Nightly 191124 06:01 + app_name: Fenix + normalized_app_name: Fenix + country: US + os_version: '29' + profile_creation_date: 18207 + profile_age_in_days: 24 + sample_id: 1 + default_search_engine: engine1 + default_search_engine_submission_url: engine1.url +- <<: *base + client_id: f + engine: engine1 + normalized_engine: Other + source: action + search_count: 5 + locale: en-CA + app_version: Nightly 191124 06:01 + app_name: Fenix + normalized_app_name: Fenix + country: CA + os_version: '29' + profile_creation_date: 18201 + profile_age_in_days: 30 + sample_id: 1 +- <<: *base + client_id: g + engine: engine1 + normalized_engine: Other + source: suggestion + search_count: 1 + locale: en-CA + app_version: Nightly 191124 06:01 + app_name: Fenix + normalized_app_name: Fenix + country: CA + os_version: '29' + profile_creation_date: 18201 + profile_age_in_days: 30 + sample_id: 1 +- <<: *base + client_id: h + engine: engine2 + normalized_engine: Other + source: suggestion + search_count: 5 + locale: en-CA + app_version: Nightly 191124 06:01 + app_name: Fenix + normalized_app_name: Fenix + country: CA + os: Android + os_version: '29' + profile_creation_date: 18201 + profile_age_in_days: 30 + sample_id: 1 +- <<: *base + client_id: i + search_count: 0 + locale: ar-EG + app_version: Nightly 191124 06:01 + app_name: Fenix + normalized_app_name: Fenix + country: EG + os_version: '29' + profile_creation_date: 18201 + profile_age_in_days: 30 + sample_id: 1 +- <<: *base + client_id: j + engine: engine2 + normalized_engine: Other + source: suggestion + search_count: 3 + locale: en-CA + app_version: Nightly 191124 06:01 + app_name: Fenix + normalized_app_name: Fenix + country: US + os_version: '29' + profile_creation_date: 18201 + profile_age_in_days: 30 + sample_id: 1 diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml index e7391667be..ddd06cbe3a 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml @@ -5,6 +5,12 @@ sample_id: 1 client_id: a search_count: 0 + organic: 0 + tagged_sap: 0 + tagged_follow_on: 0 + ad_click: 0 + search_with_ads: 0 + unknown: 0 app_version: 68.2.1 app_name: Fennec normalized_app_name: Fennec @@ -12,6 +18,7 @@ os_version: '23' profile_creation_date: 16606 profile_age_in_days: 1625 + experiments: [] - <<: *base client_id: b engine: engine1 @@ -52,6 +59,8 @@ os_version: '26' profile_creation_date: 17653 profile_age_in_days: 578 + experiments: + - key: test_experiment - <<: *base client_id: a channel: beta @@ -133,3 +142,78 @@ os_version: '29' profile_creation_date: 18201 profile_age_in_days: 30 +- &new_field_base + <<: *base + client_id: e + channel: beta + engine: engine2 + source: suggestion + search_count: 4 + app_version: Nightly 191124 06:01 + app_name: Firefox Preview + normalized_app_name: Fenix + country: US + os_version: '29' + profile_creation_date: 18201 + profile_age_in_days: 30 + total_uri_count: 22 + experiments: + - key: test_experiment + value: test_branch + - key: test_experiment_2 + value: test_branch_2 +- <<: *new_field_base + engine: engine3 + source: ad-click + search_count: 0 + ad_click: 3 +- <<: *new_field_base + engine: engine4 + source: ad-click + search_count: 0 + ad_click: 3 + total_uri_count: 23 +- <<: *new_field_base + engine: engine3 + source: search-with-ads + search_count: 0 + search_with_ads: 1 +- <<: *new_field_base + engine: engine4 + source: search-with-ads + search_count: 0 + search_with_ads: 5 + total_uri_count: 23 +- <<: *new_field_base + engine: engine1 + source: in-content.organic.none + search_count: 0 + organic: 3 + total_uri_count: 23 +- <<: *new_field_base + engine: engine1 + source: in-content.sap.none + search_count: 0 + tagged_sap: 2 +- <<: *new_field_base + engine: engine2 + source: in-content.sap-follow-on.none + search_count: 0 + tagged_follow_on: 3 +- <<: *base + client_id: e + channel: beta + app_version: Nightly 191124 06:01 + app_name: Firefox Preview + normalized_app_name: Fenix + country: US + os_version: '29' + profile_creation_date: 18201 + profile_age_in_days: 30 + engine: __other__ + search_count: 0 + unknown: 8 + total_uri_count: 1 + experiments: + - key: test_experiment + value: test_branch diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.yaml b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.yaml index fe48ccb1a2..065bc01d55 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.yaml +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix.metrics.yaml @@ -81,4 +81,78 @@ labeled_counter: metrics_search_count: - key: engine2.suggestion - value: '3' + value: 3 +- <<: *base + client_info: + android_sdk_version: '29' + app_display_version: Nightly 191124 06:01 + first_run_date: 2019-11-01-05:00 + client_id: e + normalized_country_code: US + metrics: + string: + search_default_engine_code: + search_default_engine_submission_url: + labeled_counter: + metrics_search_count: + - key: engine2.suggestion + value: 4 + browser_search_ad_clicks: + - key: engine3 + value: 3 + - key: engine4 + value: 2 + browser_search_with_ads: + - key: engine3 + value: 1 + - key: engine4 + value: 3 + browser_search_in_content: + - key: engine1.in-content.organic.none + value: 1 + - key: engine1.in-content.sap.none + value: 2 + - key: engine2.in-content.sap-follow-on.none + value: 3 + counter: + events_total_uri_count: 22 + ping_info: + experiments: + - key: test_experiment + value: + branch: test_branch + - key: test_experiment_2 + value: + branch: test_branch_2 + end_time: 2019-12-01T04:13-05:00 +- <<: *base + client_info: + android_sdk_version: '29' + app_display_version: Nightly 191124 06:01 + first_run_date: 2019-11-01-05:00 + client_id: e + normalized_country_code: US + metrics: + string: + search_default_engine_code: + search_default_engine_submission_url: + labeled_counter: + browser_search_ad_clicks: + - key: engine4 + value: 1 + browser_search_with_ads: + - key: engine4 + value: 2 + browser_search_in_content: + - key: engine1.in-content.organic.none + value: 2 + - key: __other__ + value: 8 + counter: + events_total_uri_count: 1 + ping_info: + experiments: + - key: test_experiment + value: + branch: test_branch + end_time: 2019-12-01T04:13-05:00 diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix_nightly.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix_nightly.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix_nightly.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fenix_nightly.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fennec_aurora.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fennec_aurora.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fennec_aurora.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_fennec_aurora.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox_beta.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox_beta.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox_beta.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_firefox_beta.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.schema.json index d7e4370d27..2e9f56955c 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.schema.json @@ -77,5 +77,10 @@ { "type": "TIMESTAMP", "name": "submission_timestamp" + }, + { + "type": "STRING", + "name": "experiments", + "mode": "REPEATED" } ] diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.yaml b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.yaml index 461532d4cf..93157987a3 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.yaml +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/telemetry.core.yaml @@ -50,3 +50,5 @@ searches: - key: invalid value: '2' + experiments: + - test_experiment diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml index 5433aaab73..0304c75e98 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/expect.yaml @@ -3,6 +3,12 @@ submission_date: '2020-05-01' os: Android sample_id: 1 + organic: 0 + tagged_sap: 0 + tagged_follow_on: 0 + ad_click: 0 + search_with_ads: 0 + unknown: 0 country: US os_version: '29' engine: engine1 @@ -13,6 +19,7 @@ app_name: Fennec normalized_app_name: Fennec channel: release + experiments: [] - <<: *base client_id: a app_version: Nightly 191124 06:01 diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix_nightly.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix_nightly.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix_nightly.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fenix_nightly.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fennec_aurora.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fennec_aurora.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fennec_aurora.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_fennec_aurora.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox_beta.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox_beta.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox_beta.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_firefox_beta.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/telemetry.core.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/telemetry.core.schema.json index d7e4370d27..2e9f56955c 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/telemetry.core.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_namespace_union/telemetry.core.schema.json @@ -77,5 +77,10 @@ { "type": "TIMESTAMP", "name": "submission_timestamp" + }, + { + "type": "STRING", + "name": "experiments", + "mode": "REPEATED" } ] diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml index b1194a27dd..0716da7007 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml @@ -1,7 +1,14 @@ --- -- submission_date: '2019-12-01' +- &base + submission_date: '2019-12-01' client_id: a search_count: 0 + organic: 0 + tagged_sap: 0 + tagged_follow_on: 0 + ad_click: 0 + search_with_ads: 0 + unknown: 0 locale: fr-FR app_version: 68.2.1 app_name: Fennec @@ -12,7 +19,8 @@ profile_creation_date: 16606 profile_age_in_days: 1625 sample_id: 1 -- submission_date: '2019-12-01' + experiments: [] +- <<: *base client_id: b engine: engine1 source: actionbar @@ -23,12 +31,10 @@ app_name: Fennec normalized_app_name: Fennec country: CA - os: Android os_version: '26' profile_creation_date: 17332 profile_age_in_days: 899 - sample_id: 1 -- submission_date: '2019-12-01' +- <<: *base client_id: b search_count: 0 default_search_engine: engine1 @@ -37,49 +43,44 @@ app_name: Fennec normalized_app_name: Fennec country: CA - os: Android os_version: '26' profile_creation_date: 17332 profile_age_in_days: 899 - sample_id: 1 -- submission_date: '2019-12-01' +- <<: *base client_id: a channel: beta search_count: 0 + locale: en-CA app_version: Nightly 191124 06:01 app_name: Firefox Preview normalized_app_name: Fenix country: US - os: Android os_version: '29' profile_creation_date: 18207 profile_age_in_days: 24 - sample_id: 1 -- submission_date: '2019-12-01' +- <<: *base client_id: b channel: beta engine: engine1 source: action search_count: 3 + locale: en-CA app_version: Nightly 191124 06:01 app_name: Firefox Preview normalized_app_name: Fenix country: CA - os: Android os_version: '29' profile_creation_date: 18201 profile_age_in_days: 30 - sample_id: 1 -- submission_date: '2019-12-01' +- <<: *base client_id: b channel: beta search_count: 0 + locale: en-CA app_version: Nightly 191124 06:01 app_name: Firefox Preview normalized_app_name: Fenix country: CA - os: Android os_version: '29' profile_creation_date: 18201 profile_age_in_days: 30 - sample_id: 1 diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.baseline.yaml b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.baseline.yaml index 1e402866e3..49b2783972 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.baseline.yaml +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.baseline.yaml @@ -1,2 +1,9 @@ --- -[] \ No newline at end of file +- submission_timestamp: 2019-12-01 10:06:01.757971 UTC + client_info: + client_id: a + locale: en-CA +- submission_timestamp: 2019-12-01 08:06:01.123455 UTC + client_info: + client_id: b + locale: en-CA diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix_nightly.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix_nightly.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix_nightly.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fenix_nightly.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fennec_aurora.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fennec_aurora.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fennec_aurora.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_fennec_aurora.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox_beta.metrics.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox_beta.metrics.schema.json index 78d0dcab0b..fbd3b535c0 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox_beta.metrics.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_firefox_beta.metrics.schema.json @@ -39,6 +39,60 @@ "type": "RECORD", "name": "metrics_search_count", "mode": "REPEATED" + }, + { + "description": "Records clicks of adverts on SERP pages.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_ad_clicks", + "type": "RECORD" + }, + { + "description": "Records the type of interaction a user has on SERP pages.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_in_content", + "type": "RECORD" + }, + { + "description": "Records counts of SERP pages with adverts displayed.\nThe key format is \u2018\u2019.\n", + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "browser_search_with_ads", + "type": "RECORD" } ], "type": "RECORD", @@ -57,6 +111,16 @@ ], "type": "RECORD", "name": "string" + }, + { + "fields": [ + { + "type": "INTEGER", + "name": "events_total_uri_count" + } + ], + "type": "RECORD", + "name": "counter" } ], "type": "RECORD", @@ -79,6 +143,37 @@ { "type": "STRING", "name": "end_time" + }, + { + "fields": [ + { + "name": "key", + "type": "STRING" + }, + { + "fields": [ + { + "name": "branch", + "type": "STRING" + }, + { + "fields": [ + { + "name": "type", + "type": "STRING" + } + ], + "name": "extra", + "type": "RECORD" + } + ], + "name": "value", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" } ], "type": "RECORD", diff --git a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/telemetry.core.schema.json b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/telemetry.core.schema.json index d7e4370d27..2e9f56955c 100644 --- a/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/telemetry.core.schema.json +++ b/tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/telemetry.core.schema.json @@ -77,5 +77,10 @@ { "type": "TIMESTAMP", "name": "submission_timestamp" + }, + { + "type": "STRING", + "name": "experiments", + "mode": "REPEATED" } ] diff --git a/udf/array_slice.sql b/udf/array_slice.sql new file mode 100644 index 0000000000..e0e78421c1 --- /dev/null +++ b/udf/array_slice.sql @@ -0,0 +1,23 @@ +-- Return subset of array between start_index and end_index (inclusive) +-- This function can not be used on arrays of structs +CREATE OR REPLACE FUNCTION udf.array_slice(arr ANY TYPE, start_index INT64, end_index INT64) AS ( + ARRAY( + SELECT + * EXCEPT (offset) + FROM + UNNEST(arr) + WITH OFFSET + WHERE + offset + BETWEEN start_index + AND end_index + ORDER BY + offset + ) +); + +-- Test +SELECT + assert_array_equals([1, 2, 3], udf.array_slice([1, 2, 3], 0, 2)), + assert_array_equals([1, 2], udf.array_slice([1, 2, 3], 0, 1)), + assert_array_equals(['2'], udf.array_slice(['1', '2', '3'], 1, 1)),