diff --git a/dags/bqetl_search.py b/dags/bqetl_search.py index 4c5988404d..0e56701a63 100644 --- a/dags/bqetl_search.py +++ b/dags/bqetl_search.py @@ -92,10 +92,10 @@ with DAG( search_derived__search_clients_daily__v8 ) - wait_for_telemetry_derived__main_summary__v4 = ExternalTaskCompletedSensor( - task_id="wait_for_telemetry_derived__main_summary__v4", + wait_for_telemetry_derived__clients_daily__v6 = ExternalTaskCompletedSensor( + task_id="wait_for_telemetry_derived__clients_daily__v6", external_dag_id="bqetl_main_summary", - external_task_id="telemetry_derived__main_summary__v4", + external_task_id="telemetry_derived__clients_daily__v6", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", @@ -103,7 +103,7 @@ with DAG( ) search_derived__search_clients_daily__v8.set_upstream( - wait_for_telemetry_derived__main_summary__v4 + wait_for_telemetry_derived__clients_daily__v6 ) search_derived__search_clients_last_seen__v1.set_upstream( diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql b/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql index 97efc2792d..ed42007aaf 100755 --- a/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql +++ b/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql @@ -46,77 +46,9 @@ CREATE TEMP FUNCTION organicize_source_or_type(engine STRING, original STRING) A END ); -WITH overactive AS ( - -- find client_ids with over 200,000 pings in a day +WITH augmented AS ( SELECT - client_id - FROM - telemetry.main_summary - WHERE - submission_date = @submission_date - GROUP BY - client_id - HAVING - COUNT(*) > 200000 -), -client_map_sums AS ( - SELECT - client_id, - mozfun.map.mode_last(ARRAY_CONCAT_AGG(experiments)) AS experiments, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_bookmarkmenu) - ) AS scalar_parent_urlbar_searchmode_bookmarkmenu_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_handoff) - ) AS scalar_parent_urlbar_searchmode_handoff_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_keywordoffer) - ) AS scalar_parent_urlbar_searchmode_keywordoffer_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_oneoff) - ) AS scalar_parent_urlbar_searchmode_oneoff_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_other) - ) AS scalar_parent_urlbar_searchmode_other_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_shortcut) - ) AS scalar_parent_urlbar_searchmode_shortcut_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_tabmenu) - ) AS scalar_parent_urlbar_searchmode_tabmenu_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_tabtosearch) - ) AS scalar_parent_urlbar_searchmode_tabtosearch_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_tabtosearch_onboard) - ) AS scalar_parent_urlbar_searchmode_tabtosearch_onboard_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_topsites_newtab) - ) AS scalar_parent_urlbar_searchmode_topsites_newtab_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_topsites_urlbar) - ) AS scalar_parent_urlbar_searchmode_topsites_urlbar_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_touchbar) - ) AS scalar_parent_urlbar_searchmode_touchbar_sum, - mozfun.map.sum( - ARRAY_CONCAT_AGG(scalar_parent_urlbar_searchmode_typed) - ) AS scalar_parent_urlbar_searchmode_typed_sum, - FROM - telemetry.main_summary - LEFT JOIN - overactive - USING - (client_id) - WHERE - submission_date = @submission_date - AND overactive.client_id IS NULL - GROUP BY - client_id -), -augmented AS ( - SELECT - * EXCEPT (experiments), + *, ARRAY_CONCAT( ARRAY( SELECT AS STRUCT @@ -179,7 +111,7 @@ augmented AS ( value AS count, organicize_source_or_type(key, "ad-click") AS type FROM - UNNEST(scalar_parent_browser_search_ad_clicks) + UNNEST(ad_clicks) ), ARRAY( SELECT AS STRUCT @@ -188,38 +120,18 @@ augmented AS ( value AS count, organicize_source_or_type(key, "search-with-ads") AS type FROM - UNNEST(scalar_parent_browser_search_with_ads) + UNNEST(search_with_ads) ) ) AS _searches, - -- Aggregate numerical values before flattening engine/source array - SUM(subsession_length / 3600) OVER w1 AS subsession_hours_sum, - COUNTIF(subsession_counter = 1) OVER w1 AS sessions_started_on_this_day, - AVG(active_addons_count) OVER w1 AS active_addons_count_mean, - MAX( - scalar_parent_browser_engagement_max_concurrent_tab_count - ) OVER w1 AS max_concurrent_tab_count_max, - SUM(scalar_parent_browser_engagement_tab_open_event_count) OVER w1 AS tab_open_event_count_sum, - SUM(active_ticks / (3600 / 5)) OVER w1 AS active_hours_sum, - SUM(scalar_parent_browser_engagement_total_uri_count) OVER w1 AS total_uri_count, - client_map_sums.experiments, FROM - telemetry.main_summary - LEFT JOIN - client_map_sums - USING - (client_id) - WINDOW - w1 AS ( - PARTITION BY - client_id, - submission_date - ) + telemetry.clients_daily ), flattened AS ( SELECT * FROM - augmented, + augmented + CROSS JOIN UNNEST( IF -- Provide replacement empty _searches with one null search, to ensure all @@ -231,54 +143,41 @@ flattened AS ( ) ) ), - -- Aggregate by client_id using windows -windowed AS ( +-- Get count based on search type +counted AS ( SELECT - ROW_NUMBER() OVER w1_unframed AS _n, + -- use row number to dedupe over window + ROW_NUMBER() OVER w1 AS _n, submission_date, client_id, engine, source, - mozfun.stats.mode_last(ARRAY_AGG(country) OVER w1) AS country, - mozfun.stats.mode_last( - ARRAY_AGG(get_search_addon_version(active_addons)) OVER w1 - ) AS addon_version, - mozfun.stats.mode_last(ARRAY_AGG(app_version) OVER w1) AS app_version, - mozfun.stats.mode_last(ARRAY_AGG(distribution_id) OVER w1) AS distribution_id, - mozfun.stats.mode_last(ARRAY_AGG(locale) OVER w1) AS locale, - mozfun.stats.mode_last( - ARRAY_AGG(user_pref_browser_search_region) OVER w1 - ) AS user_pref_browser_search_region, - mozfun.stats.mode_last(ARRAY_AGG(search_cohort) OVER w1) AS search_cohort, - mozfun.stats.mode_last(ARRAY_AGG(os) OVER w1) AS os, - mozfun.stats.mode_last(ARRAY_AGG(os_version) OVER w1) AS os_version, - mozfun.stats.mode_last(ARRAY_AGG(channel) OVER w1) AS channel, - mozfun.stats.mode_last(ARRAY_AGG(is_default_browser) OVER w1) AS is_default_browser, - mozfun.stats.mode_last(ARRAY_AGG(profile_creation_date) OVER w1) AS profile_creation_date, - mozfun.stats.mode_last(ARRAY_AGG(default_search_engine) OVER w1) AS default_search_engine, - mozfun.stats.mode_last( - ARRAY_AGG(default_search_engine_data_load_path) OVER w1 - ) AS default_search_engine_data_load_path, - mozfun.stats.mode_last( - ARRAY_AGG(default_search_engine_data_submission_url) OVER w1 - ) AS default_search_engine_data_submission_url, - mozfun.stats.mode_last( - ARRAY_AGG(default_private_search_engine) OVER w1 - ) AS default_private_search_engine, - mozfun.stats.mode_last( - ARRAY_AGG(default_private_search_engine_data_load_path) OVER w1 - ) AS default_private_search_engine_data_load_path, - mozfun.stats.mode_last( - ARRAY_AGG(default_private_search_engine_data_submission_url) OVER w1 - ) AS default_private_search_engine_data_submission_url, - mozfun.stats.mode_last(ARRAY_AGG(sample_id) OVER w1) AS sample_id, + country, + get_search_addon_version(active_addons) AS addon_version, + app_version, + distribution_id, + locale, + user_pref_browser_search_region, + search_cohort, + os, + os_version, + channel, + is_default_browser, + UNIX_DATE(DATE(profile_creation_date)) AS profile_creation_date, + default_search_engine, + default_search_engine_data_load_path, + default_search_engine_data_submission_url, + default_private_search_engine, + default_private_search_engine_data_load_path, + default_private_search_engine_data_submission_url, + sample_id, subsession_hours_sum, sessions_started_on_this_day, active_addons_count_mean, - max_concurrent_tab_count_max, - tab_open_event_count_sum, + scalar_parent_browser_engagement_max_concurrent_tab_count_max AS max_concurrent_tab_count_max, + scalar_parent_browser_engagement_tab_open_event_count_sum AS tab_open_event_count_sum, active_hours_sum, - total_uri_count, + scalar_parent_browser_engagement_total_uri_count_sum AS total_uri_count, experiments, scalar_parent_urlbar_searchmode_bookmarkmenu_sum, scalar_parent_urlbar_searchmode_handoff_sum, @@ -293,10 +192,7 @@ windowed AS ( scalar_parent_urlbar_searchmode_topsites_urlbar_sum, scalar_parent_urlbar_searchmode_touchbar_sum, scalar_parent_urlbar_searchmode_typed_sum, - SAFE_SUBTRACT( - UNIX_DATE(DATE(SAFE.TIMESTAMP(subsession_start_date))), - profile_creation_date - ) AS profile_age_in_days, + profile_age_in_days, SUM(IF(type = 'organic', count, 0)) OVER w1 AS organic, SUM(IF(type = 'tagged-sap', count, 0)) OVER w1 AS tagged_sap, SUM(IF(type = 'tagged-follow-on', count, 0)) OVER w1 AS tagged_follow_on, @@ -313,7 +209,6 @@ windowed AS ( AND client_id IS NOT NULL AND (count < 10000 OR count IS NULL) WINDOW - -- Aggregations require a framed window w1 AS ( PARTITION BY client_id, @@ -321,27 +216,11 @@ windowed AS ( engine, source, type - ORDER BY - `timestamp` ASC - ROWS BETWEEN - UNBOUNDED PRECEDING - AND UNBOUNDED FOLLOWING - ), - -- ROW_NUMBER does not work on a framed window - w1_unframed AS ( - PARTITION BY - client_id, - submission_date, - engine, - source, - type - ORDER BY - `timestamp` ASC ) ) SELECT * EXCEPT (_n) FROM - windowed + counted WHERE _n = 1 diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/expect.yaml b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/expect.yaml index b74e3950ae..357e8732df 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/expect.yaml @@ -1,20 +1,9 @@ --- - &base - locale: de - default_search_engine_data_submission_url: 'https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b' - sample_id: 42 - default_search_engine_data_load_path: 'jar:[app]/omni.ja!browser/google.xml' - os_version: '10.0' - default_search_engine: google - app_version: 54.0.1 - channel: release submission_date: '2017-01-01' - country: DE - client_id: c + client_id: a + country: US sap: 0 - profile_age_in_days: 366 - profile_creation_date: 16801 - os: windows organic: 0 tagged_sap: 0 tagged_follow_on: 0 @@ -23,13 +12,7 @@ search_with_ads: 0 search_with_ads_organic: 0 unknown: 0 - subsession_hours_sum: 1 - sessions_started_on_this_day: 1 - active_addons_count_mean: 2 - max_concurrent_tab_count_max: 10 - tab_open_event_count_sum: 5 - active_hours_sum: 0.5 - total_uri_count: 2 + profile_creation_date: 18145 experiments: [] scalar_parent_urlbar_searchmode_bookmarkmenu_sum: [] scalar_parent_urlbar_searchmode_handoff_sum: [] @@ -45,93 +28,43 @@ scalar_parent_urlbar_searchmode_touchbar_sum: [] scalar_parent_urlbar_searchmode_typed_sum: [] - <<: *base - engine: google - source: urlbar client_id: b - country: US - sap: 4 - addon_version: 0.9.6 - total_uri_count: 1 - is_default_browser: false - user_pref_browser_search_region: DE -- &client_a - <<: *base - engine: bing - source: urlbar - sap: 4 - client_id: a - addon_version: 0.9.5 - subsession_hours_sum: 8 - sessions_started_on_this_day: 8 - active_addons_count_mean: 2 - max_concurrent_tab_count_max: 10 - tab_open_event_count_sum: 40 - active_hours_sum: 4 - total_uri_count: 16 - user_pref_browser_search_region: DE -- <<: *client_a - engine: google - source: urlbar - sap: 28 - distribution_id: totally not null -- <<: *client_a - engine: google - source: 'ad-click:' - ad_click: 10 - sap: 0 - distribution_id: totally not null -- <<: *client_a - engine: google - source: 'search-with-ads:organic' - search_with_ads_organic: 4 - sap: 0 - distribution_id: totally not null -- <<: *client_a - engine: google - source: 'search-with-ads:' - search_with_ads: 7 - sap: 0 - distribution_id: totally not null -- <<: *client_a - engine: google - source: 'ad-click:organic' - ad_click_organic: 6 - sap: 0 - distribution_id: totally not null -- <<: *client_a - engine: yahoo - source: urlbar - sap: 4 -- &client_d - <<: *base - engine: google - source: urlbar - sap: 8 - client_id: d - addon_version: 0.9.4 - subsession_hours_sum: 3 - sessions_started_on_this_day: 3 - max_concurrent_tab_count_max: 10 - tab_open_event_count_sum: 15 - active_hours_sum: 1.5 - total_uri_count: 7 - user_pref_browser_search_region: DE -- <<: *client_d - engine: bing - source: urlbar - sap: 4 -- <<: *client_d - engine: yahoo - source: urlbar - sap: 4 + engine: engine1 + source: searchbar + sap: 1 - <<: *base - client_id: e - source: urlbar - engine: google - sap: 4 - is_default_browser: true - default_private_search_engine: engine1 - default_private_search_engine_data_load_path: 'engine1.load_path' - default_private_search_engine_data_submission_url: 'https://www.engine.com' - total_uri_count: 2 - user_pref_browser_search_region: DE + client_id: b + engine: engine2 + source: 'in-content:sap:code' + tagged_sap: 2 +- <<: *base + client_id: b + engine: engine2 + source: 'in-content:sap-follow-on:code' + tagged_follow_on: 3 +- <<: *base + client_id: b + engine: engine2 + source: 'in-content:organic:none' + organic: 4 +- <<: *base + client_id: b + engine: engine1 + source: 'ad-click:' + ad_click: 2 +- <<: *base + client_id: b + engine: engine1 + source: 'ad-click:organic' + ad_click_organic: 3 +- <<: *base + client_id: b + engine: engine1 + source: 'search-with-ads:' + search_with_ads: 3 +- <<: *base + client_id: c + addon_version: 0.9.6 + engine: engine2 + source: contextmenu + sap: 2 diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.schema.json b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.schema.json new file mode 100644 index 0000000000..1f2e3fafbf --- /dev/null +++ b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.schema.json @@ -0,0 +1,549 @@ +[ + { + "mode": "NULLABLE", + "name": "submission_date", + "type": "DATE" + }, + { + "mode": "NULLABLE", + "name": "client_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "active_addons_count_mean", + "type": "FLOAT" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "addon_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "version", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "active_addons", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "active_hours_sum", + "type": "FLOAT" + }, + + { + "mode": "NULLABLE", + "name": "app_build_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "app_version", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "channel", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_search_engine", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_search_engine_data_load_path", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_search_engine_data_name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_search_engine_data_origin", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_search_engine_data_submission_url", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "distribution_id", + "type": "STRING" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "experiments", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "country", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "is_default_browser", + "type": "BOOLEAN" + }, + { + "mode": "NULLABLE", + "name": "locale", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "memory_mb", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "os", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "os_version", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "profile_age_in_days", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "profile_creation_date", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "sample_id", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "scalar_parent_browser_engagement_max_concurrent_tab_count_max", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "scalar_parent_browser_engagement_tab_open_event_count_sum", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "scalar_parent_browser_engagement_total_uri_count_sum", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_cohort", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "search_count_abouthome", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_contextmenu", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_newtab", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_searchbar", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_system", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_urlbar", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_all", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_tagged_sap", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_tagged_follow_on", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_organic", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "sessions_started_on_this_day", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "subsession_hours_sum", + "type": "NUMERIC" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_bookmarkmenu_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_handoff_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_keywordoffer_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_oneoff_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_other_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_shortcut_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_tabmenu_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_tabtosearch_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_tabtosearch_onboard_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_topsites_newtab_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_topsites_urlbar_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_touchbar_sum", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "scalar_parent_urlbar_searchmode_typed_sum", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "search_count_webextension", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_alias", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "search_count_urlbar_searchmode", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "default_private_search_engine", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_private_search_engine_data_load_path", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_private_search_engine_data_name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_private_search_engine_data_origin", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "default_private_search_engine_data_submission_url", + "type": "STRING" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "engine", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "source", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "count", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "search_counts", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "user_pref_browser_search_region", + "type": "STRING" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "search_with_ads", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "key", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "INTEGER" + } + ], + "mode": "REPEATED", + "name": "ad_clicks", + "type": "RECORD" + } +] \ No newline at end of file diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.yaml b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.yaml new file mode 100644 index 0000000000..898971593b --- /dev/null +++ b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.clients_daily.yaml @@ -0,0 +1,57 @@ +--- +- &base + submission_date: '2017-01-01' + client_id: a + country: US + profile_creation_date: '2019-09-06 00:00:00' + experiments: [] + scalar_parent_urlbar_searchmode_bookmarkmenu_sum: [] + scalar_parent_urlbar_searchmode_handoff_sum: [] + scalar_parent_urlbar_searchmode_keywordoffer_sum: [] + scalar_parent_urlbar_searchmode_oneoff_sum: [] + scalar_parent_urlbar_searchmode_other_sum: [] + scalar_parent_urlbar_searchmode_shortcut_sum: [] + scalar_parent_urlbar_searchmode_tabmenu_sum: [] + scalar_parent_urlbar_searchmode_tabtosearch_sum: [] + scalar_parent_urlbar_searchmode_tabtosearch_onboard_sum: [] + scalar_parent_urlbar_searchmode_topsites_newtab_sum: [] + scalar_parent_urlbar_searchmode_topsites_urlbar_sum: [] + scalar_parent_urlbar_searchmode_touchbar_sum: [] + scalar_parent_urlbar_searchmode_typed_sum: [] + search_counts: [] + ad_clicks: [] + search_with_ads: [] +- <<: *base + client_id: b + search_counts: + - source: searchbar + engine: engine1 + count: 1 + - source: 'in-content:sap:code' + engine: engine2 + count: 2 + - source: 'in-content:sap-follow-on:code' + engine: engine2 + count: 3 + - source: 'in-content:organic:none' + engine: engine2 + count: 4 + ad_clicks: + - key: engine1 + value: 2 + - key: 'engine1:organic' + value: 3 + search_with_ads: + - key: engine1 + value: 3 +- <<: *base + client_id: c + active_addons: + - addon_id: random@mozilla.com + version: '0.1' + - addon_id: followonsearch@mozilla.com + version: 0.9.6 + search_counts: + - source: contextmenu + engine: engine2 + count: 2 diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.main_summary.schema.json b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.main_summary.schema.json deleted file mode 100644 index 9bc4030405..0000000000 --- a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.main_summary.schema.json +++ /dev/null @@ -1,533 +0,0 @@ -[ - { - "type": "DATE", - "name": "submission_date", - "mode": "REQUIRED" - }, - { - "type": "STRING", - "name": "client_id", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "sample_id", - "mode": "REQUIRED" - }, - { - "type": "STRING", - "name": "document_id", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "timestamp", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "user_pref_browser_search_region", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "os", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "os_version", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "channel", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "profile_creation_date", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "country", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "app_version", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "distribution_id", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "locale", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "is_default_browser", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_search_engine", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_search_engine_data_load_path", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_search_engine_data_submission_url", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_private_search_engine", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_private_search_engine_data_load_path", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_private_search_engine_data_submission_url", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "subsession_start_date", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "subsession_length", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "subsession_counter", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "active_addons_count", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "active_ticks", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "scalar_parent_browser_engagement_max_concurrent_tab_count", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "scalar_parent_browser_engagement_tab_open_event_count", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "search_cohort", - "mode": "NULLABLE" - }, - { - "fields": [ - { - "type": "STRING", - "name": "engine", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "source", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "count", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "search_counts", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "addon_id", - "mode": "REQUIRED" - }, - { - "type": "BOOLEAN", - "name": "blocklisted", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "name", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "user_disabled", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "app_disabled", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "version", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "scope", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "type", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "foreign_install", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "has_binary_components", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "install_day", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "update_day", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "signed_state", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "is_system", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "is_web_extension", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "multiprocess_compatible", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "active_addons", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "REQUIRED" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_browser_search_ad_clicks", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "REQUIRED" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_browser_search_with_ads", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "experiments", - "mode": "REPEATED" - }, - { - "type": "INTEGER", - "name": "scalar_parent_browser_engagement_total_uri_count", - "mode": "NULLABLE" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_bookmarkmenu", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_handoff", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_keywordoffer", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_oneoff", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_other", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_shortcut", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_tabmenu", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_tabtosearch", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_tabtosearch_onboard", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_topsites_newtab", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_topsites_urlbar", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_touchbar", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_typed", - "mode": "REPEATED" - } -] diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.main_summary.yaml b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.main_summary.yaml deleted file mode 100644 index ef27c1ed63..0000000000 --- a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_aggregation/telemetry.main_summary.yaml +++ /dev/null @@ -1,155 +0,0 @@ ---- -- &base - locale: de - default_search_engine_data_submission_url: 'https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b' - submission_date: '2017-01-01' - user_pref_browser_search_region: DE - subsession_length: 3600 - search_cohort: null - scalar_parent_browser_search_with_ads: [] - sample_id: 42 - default_search_engine_data_load_path: 'jar:[app]/omni.ja!browser/google.xml' - os_version: '10.0' - distribution_id: null - default_search_engine: google - app_version: 54.0.1 - channel: release - scalar_parent_browser_search_ad_clicks: [] - active_ticks: 360 - search_counts: - - engine: google - source: urlbar - count: 4 - scalar_parent_browser_engagement_max_concurrent_tab_count: 10 - client_id: b - document_id: x - scalar_parent_browser_engagement_tab_open_event_count: 5 - active_addons_count: 2 - country: US - subsession_start_date: '2017-01-01 10:00:00' - subsession_counter: 1 - profile_creation_date: 16801 - os: windows - active_addons: - - addon_id: random@mozilla.com - version: '0.1' - name: random - - addon_id: followonsearch@mozilla.com - version: 0.9.6 - name: Follow-on Search Telemetry - scalar_parent_browser_engagement_total_uri_count: 1 - experiments: [] - is_default_browser: false -- &client_a - <<: *base - client_id: a - country: DE - active_addons: - - addon_id: random@mozilla.com - version: '0.1' - name: random - - addon_id: followonsearch@mozilla.com - version: 0.9.5 - name: Follow-on Search Telemetry - scalar_parent_browser_engagement_total_uri_count: 2 - is_default_browser: null -- <<: *client_a - scalar_parent_browser_search_with_ads: - - key: google - value: 2 - app_version: 52.0.3 - scalar_parent_browser_search_ad_clicks: - - key: google - value: 3 -- <<: *client_a - scalar_parent_browser_search_with_ads: - - key: google:organic - value: 4 - - key: google:sap - value: 5 - distribution_id: totally not null - scalar_parent_browser_search_ad_clicks: - - key: google:organic - value: 6 - - key: google:sap-follow-on - value: 7 -- <<: *client_a - search_counts: - - engine: bing - source: urlbar - count: 4 - - engine: yahoo - source: urlbar - count: 4 -- <<: *client_a -- <<: *client_a -- <<: *client_a -- <<: *client_a -- <<: *base - search_counts: [] - client_id: c - country: DE - active_addons: - - addon_id: random@mozilla.com - version: '0.1' - name: random - scalar_parent_browser_engagement_total_uri_count: 2 - is_default_browser: null - user_pref_browser_search_region: null -- <<: *base - search_counts: - - engine: google - source: urlbar - count: 4 - - engine: bing - source: urlbar - count: 4 - client_id: d - country: DE - active_addons: - - addon_id: followonsearch@mozilla.com - version: 0.9.4 - name: Follow-on Search Telemetry - - addon_id: followonsearch@mozilla.com - version: 0.9.4 - name: Follow-on Search Telemetry - scalar_parent_browser_engagement_total_uri_count: 3 - is_default_browser: null -- <<: *base - default_search_engine_data_submission_url: 'https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b' - client_id: d - country: DE - active_addons: - - addon_id: followonsearch@mozilla.com - version: 0.9.4 - name: Follow-on Search Telemetry - - addon_id: followonsearch@mozilla.com - version: 0.9.4 - name: Follow-on Search Telemetry - scalar_parent_browser_engagement_total_uri_count: 2 - is_default_browser: null -- <<: *base - search_counts: - - engine: yahoo - source: urlbar - count: 4 - client_id: d - country: DE - active_addons: - - addon_id: followonsearch@mozilla.com - version: 0.9.4 - name: Follow-on Search Telemetry - - addon_id: followonsearch@mozilla.com - version: 0.9.4 - name: Follow-on Search Telemetry - scalar_parent_browser_engagement_total_uri_count: 2 - is_default_browser: null -- <<: *base - client_id: e - country: DE - active_addons: [] - scalar_parent_browser_engagement_total_uri_count: 2 - default_private_search_engine_data_submission_url: 'https://www.engine.com' - default_private_search_engine_data_load_path: engine1.load_path - default_private_search_engine: engine1 - is_default_browser: true diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/expect.yaml b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/expect.yaml deleted file mode 100644 index 5e0588ab68..0000000000 --- a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/expect.yaml +++ /dev/null @@ -1,59 +0,0 @@ ---- -- &base - ad_click: 0 - ad_click_organic: 0 - organic: 0 - sap: 4 - search_with_ads: 0 - search_with_ads_organic: 0 - sessions_started_on_this_day: 0 - source: urlbar - submission_date: '2017-01-01' - tagged_follow_on: 0 - tagged_sap: 0 - unknown: 0 - client_id: a - engine: bing - experiments: - - key: exp1 - value: active - - key: exp2 - value: active - scalar_parent_urlbar_searchmode_bookmarkmenu_sum: [] - scalar_parent_urlbar_searchmode_handoff_sum: [] - scalar_parent_urlbar_searchmode_keywordoffer_sum: [] - scalar_parent_urlbar_searchmode_oneoff_sum: [] - scalar_parent_urlbar_searchmode_other_sum: [] - scalar_parent_urlbar_searchmode_shortcut_sum: [] - scalar_parent_urlbar_searchmode_tabmenu_sum: [] - scalar_parent_urlbar_searchmode_tabtosearch_sum: [] - scalar_parent_urlbar_searchmode_tabtosearch_onboard_sum: [] - scalar_parent_urlbar_searchmode_topsites_newtab_sum: [] - scalar_parent_urlbar_searchmode_topsites_urlbar_sum: [] - scalar_parent_urlbar_searchmode_touchbar_sum: [] - scalar_parent_urlbar_searchmode_typed_sum: [] -- <<: *base - client_id: a - sap: 4 - engine: yahoo - experiments: - - key: exp1 - value: active - - key: exp2 - value: active -- <<: *base - client_id: a - sap: 8 - engine: google - experiments: - - key: exp1 - value: active - - key: exp2 - value: active -- <<: *base - client_id: b - sap: 4 - engine: google - experiments: - - key: exp1 - value: active diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/query_params.yaml b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/query_params.yaml deleted file mode 100644 index ff9d8f0233..0000000000 --- a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/query_params.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: submission_date - type: DATE - value: 2017-01-01 diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/telemetry.main_summary.schema.json b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/telemetry.main_summary.schema.json deleted file mode 100644 index 0d03b13953..0000000000 --- a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/telemetry.main_summary.schema.json +++ /dev/null @@ -1,533 +0,0 @@ -[ - { - "type": "DATE", - "name": "submission_date", - "mode": "REQUIRED" - }, - { - "type": "STRING", - "name": "client_id", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "sample_id", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "document_id", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "timestamp", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "user_pref_browser_search_region", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "os", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "os_version", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "channel", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "profile_creation_date", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "country", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "app_version", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "distribution_id", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "locale", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "is_default_browser", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_search_engine", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_search_engine_data_load_path", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_search_engine_data_submission_url", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_private_search_engine", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_private_search_engine_data_load_path", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "default_private_search_engine_data_submission_url", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "subsession_start_date", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "subsession_length", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "subsession_counter", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "active_addons_count", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "active_ticks", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "scalar_parent_browser_engagement_max_concurrent_tab_count", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "scalar_parent_browser_engagement_tab_open_event_count", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "search_cohort", - "mode": "NULLABLE" - }, - { - "fields": [ - { - "type": "STRING", - "name": "engine", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "source", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "count", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "search_counts", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "addon_id", - "mode": "REQUIRED" - }, - { - "type": "BOOLEAN", - "name": "blocklisted", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "name", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "user_disabled", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "app_disabled", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "version", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "scope", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "type", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "foreign_install", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "has_binary_components", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "install_day", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "update_day", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "signed_state", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "is_system", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "is_web_extension", - "mode": "NULLABLE" - }, - { - "type": "BOOLEAN", - "name": "multiprocess_compatible", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "active_addons", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "REQUIRED" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_browser_search_ad_clicks", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "REQUIRED" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_browser_search_with_ads", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "STRING", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "experiments", - "mode": "REPEATED" - }, - { - "type": "INTEGER", - "name": "scalar_parent_browser_engagement_total_uri_count", - "mode": "NULLABLE" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_bookmarkmenu", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_handoff", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_keywordoffer", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_oneoff", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_other", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_shortcut", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_tabmenu", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_tabtosearch", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_tabtosearch_onboard", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_topsites_newtab", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_topsites_urlbar", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_touchbar", - "mode": "REPEATED" - }, - { - "fields": [ - { - "type": "STRING", - "name": "key", - "mode": "NULLABLE" - }, - { - "type": "INTEGER", - "name": "value", - "mode": "NULLABLE" - } - ], - "type": "RECORD", - "name": "scalar_parent_urlbar_searchmode_typed", - "mode": "REPEATED" - } -] diff --git a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/telemetry.main_summary.yaml b/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/telemetry.main_summary.yaml deleted file mode 100644 index 8628b33130..0000000000 --- a/tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/test_experiments/telemetry.main_summary.yaml +++ /dev/null @@ -1,46 +0,0 @@ ---- -- &base - submission_date: '2017-01-01' - scalar_parent_browser_search_with_ads: [] - scalar_parent_browser_search_ad_clicks: [] - active_addons: [] - search_counts: - - engine: google - source: urlbar - count: 4 - client_id: b - experiments: - - key: exp1 - value: active -- <<: *base - search_counts: - - engine: google - source: urlbar - count: 4 - client_id: a - experiments: - - key: exp1 - value: active -- <<: *base - search_counts: - - engine: bing - source: urlbar - count: 4 - - engine: yahoo - source: urlbar - count: 4 - client_id: a - experiments: - - key: exp1 - value: active - - key: exp2 - value: active -- <<: *base - search_counts: - - engine: google - source: urlbar - count: 4 - client_id: a - experiments: - - key: exp1 - value: inactive