mozilla
/
bigquery-etl
зеркало из https://github.com/mozilla/bigquery-etl.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Use normalize engine udf in all search queries (#596)

This commit is contained in:
Ben Wu 2019-12-16 12:18:01 -05:00 коммит произвёл GitHub
Родитель ccf7824f12
Коммит d67c760725
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
25 изменённых файлов: 253 добавлений и 145 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
sql/search/search_aggregates/view.sql
Просмотреть файл

@ -22,7 +22,8 @@ SELECT
os,
os_version,
IFNULL(client_count, 0) AS client_count,
NULL AS default_private_search_engine
NULL AS default_private_search_engine,
NULL AS normalized_engine
FROM
`moz-fx-data-derived-datasets.search.search_aggregates_v6`
WHERE

2
sql/search_derived/mobile_search_aggregates_v1/query.sql
Просмотреть файл

@ -1,6 +1,7 @@
SELECT
submission_date,
engine,
normalized_engine,
source,
app_name,
app_version,
@ -21,6 +22,7 @@ WHERE
GROUP BY
submission_date,
engine,
normalized_engine,
source,
app_name,
app_version,

95
sql/search_derived/mobile_search_clients_daily_v1/query.sql
Просмотреть файл

@ -21,6 +21,26 @@ CREATE TEMP FUNCTION udf_parse_iso8601_date(date_str STRING) RETURNS DATE AS (
SAFE.PARSE_DATE('%Y%m%d', SAFE.SUBSTR(date_str, 0, 8))
)
);
CREATE TEMP FUNCTION
udf_strict_normalize_search_engine(engine STRING) AS (
CASE
WHEN engine IS NULL THEN NULL
WHEN STARTS_WITH(engine, 'google')
OR STARTS_WITH(engine, 'Google')
OR STARTS_WITH(engine, 'other-Google') THEN 'Google'
WHEN STARTS_WITH(engine, 'ddg')
OR STARTS_WITH(engine, 'duckduckgo')
OR STARTS_WITH(engine, 'DuckDuckGo')
OR STARTS_WITH(engine, 'other-DuckDuckGo') THEN 'DuckDuckGo'
WHEN STARTS_WITH(engine, 'bing')
OR STARTS_WITH(engine, 'Bing')
OR STARTS_WITH(engine, 'other-Bing') THEN 'Bing'
WHEN STARTS_WITH(engine, 'yandex')
OR STARTS_WITH(engine, 'Yandex')
OR STARTS_WITH(engine, 'other-Yandex') THEN 'Yandex'
ELSE 'Other'
END
);
--
-- Older versions separate source and engine with an underscore instead of period
-- Return array of form [source, engine] if key is valid, empty array otherwise
@ -145,41 +165,48 @@ combined_search_clients AS (
sample_id
FROM
fenix_flattened_searches
),
unfiltered_search_clients AS (
SELECT
submission_date,
client_id,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(0)]) AS engine,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(1)]) AS source,
app_name,
SUM(
IF(
ARRAY_LENGTH(normalized_search_key) = 0
OR search_count > 10000,
0,
search_count
)
) AS search_count,
udf_mode_last(ARRAY_AGG(country)) AS country,
udf_mode_last(ARRAY_AGG(locale)) AS locale,
udf_mode_last(ARRAY_AGG(app_version)) AS app_version,
udf_mode_last(ARRAY_AGG(channel)) AS channel,
udf_mode_last(ARRAY_AGG(os)) AS os,
udf_mode_last(ARRAY_AGG(os_version)) AS os_version,
udf_mode_last(ARRAY_AGG(default_search_engine)) AS default_search_engine,
udf_mode_last(ARRAY_AGG(default_search_engine_submission_url)) AS default_search_engine_submission_url,
udf_mode_last(ARRAY_AGG(distribution_id)) AS distribution_id,
udf_mode_last(ARRAY_AGG(profile_creation_date)) AS profile_creation_date,
udf_mode_last(ARRAY_AGG(profile_age_in_days)) AS profile_age_in_days,
udf_mode_last(ARRAY_AGG(sample_id)) AS sample_id
FROM
combined_search_clients
WHERE
submission_date = @submission_date
GROUP BY
submission_date,
client_id,
engine,
source,
app_name
)
SELECT
submission_date,
client_id,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(0)]) AS engine,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(1)]) AS source,
app_name,
SUM(
IF(
ARRAY_LENGTH(normalized_search_key) = 0
OR search_count > 10000,
0,
search_count
)
) AS search_count,
udf_mode_last(ARRAY_AGG(country)) AS country,
udf_mode_last(ARRAY_AGG(locale)) AS locale,
udf_mode_last(ARRAY_AGG(app_version)) AS app_version,
udf_mode_last(ARRAY_AGG(channel)) AS channel,
udf_mode_last(ARRAY_AGG(os)) AS os,
udf_mode_last(ARRAY_AGG(os_version)) AS os_version,
udf_mode_last(ARRAY_AGG(default_search_engine)) AS default_search_engine,
udf_mode_last(ARRAY_AGG(default_search_engine_submission_url)) AS default_search_engine_submission_url,
udf_mode_last(ARRAY_AGG(distribution_id)) AS distribution_id,
udf_mode_last(ARRAY_AGG(profile_creation_date)) AS profile_creation_date,
udf_mode_last(ARRAY_AGG(profile_age_in_days)) AS profile_age_in_days,
udf_mode_last(ARRAY_AGG(sample_id)) AS sample_id
*,
udf_strict_normalize_search_engine(engine) AS normalized_engine
FROM
combined_search_clients
WHERE
submission_date = @submission_date
GROUP BY
submission_date,
client_id,
engine,
source,
app_name
unfiltered_search_clients

2
sql/search_derived/search_aggregates_v8/query.sql
Просмотреть файл

@ -5,6 +5,7 @@ SELECT
country,
distribution_id,
engine,
normalized_engine,
locale,
search_cohort,
source,
@ -32,6 +33,7 @@ GROUP BY
country,
distribution_id,
engine,
normalized_engine,
locale,
search_cohort,
source,

21
sql/search_derived/search_clients_daily_v8/query.sql
Просмотреть файл

@ -15,6 +15,26 @@ CREATE TEMP FUNCTION
MAX(_offset) DESC
LIMIT
1 ));
CREATE TEMP FUNCTION
udf_strict_normalize_search_engine(engine STRING) AS (
CASE
WHEN engine IS NULL THEN NULL
WHEN STARTS_WITH(engine, 'google')
OR STARTS_WITH(engine, 'Google')
OR STARTS_WITH(engine, 'other-Google') THEN 'Google'
WHEN STARTS_WITH(engine, 'ddg')
OR STARTS_WITH(engine, 'duckduckgo')
OR STARTS_WITH(engine, 'DuckDuckGo')
OR STARTS_WITH(engine, 'other-DuckDuckGo') THEN 'DuckDuckGo'
WHEN STARTS_WITH(engine, 'bing')
OR STARTS_WITH(engine, 'Bing')
OR STARTS_WITH(engine, 'other-Bing') THEN 'Bing'
WHEN STARTS_WITH(engine, 'yandex')
OR STARTS_WITH(engine, 'Yandex')
OR STARTS_WITH(engine, 'other-Yandex') THEN 'Yandex'
ELSE 'Other'
END
);
--
-- Return the version of the search addon if it exists, null otherwise
CREATE TEMP FUNCTION get_search_addon_version(active_addons ANY type) AS (
@ -127,6 +147,7 @@ WITH
submission_date,
client_id,
engine,
udf_strict_normalize_search_engine(engine) AS normalized_engine,
source,
udf_mode_last(ARRAY_AGG(country) OVER w1) AS country,
udf_mode_last(ARRAY_AGG(get_search_addon_version(active_addons)) OVER w1) AS addon_version,

29
sql/search_derived/search_clients_last_seen_v1/query.sql
Просмотреть файл

@ -159,18 +159,31 @@ CREATE TEMP FUNCTION
MAX(_offset) DESC
LIMIT
1 ));
CREATE TEMP FUNCTION
udf_strict_normalize_search_engine(engine STRING) AS (
CASE
WHEN engine IS NULL THEN NULL
WHEN STARTS_WITH(engine, 'google')
OR STARTS_WITH(engine, 'Google')
OR STARTS_WITH(engine, 'other-Google') THEN 'Google'
WHEN STARTS_WITH(engine, 'ddg')
OR STARTS_WITH(engine, 'duckduckgo')
OR STARTS_WITH(engine, 'DuckDuckGo')
OR STARTS_WITH(engine, 'other-DuckDuckGo') THEN 'DuckDuckGo'
WHEN STARTS_WITH(engine, 'bing')
OR STARTS_WITH(engine, 'Bing')
OR STARTS_WITH(engine, 'other-Bing') THEN 'Bing'
WHEN STARTS_WITH(engine, 'yandex')
OR STARTS_WITH(engine, 'Yandex')
OR STARTS_WITH(engine, 'other-Yandex') THEN 'Yandex'
ELSE 'Other'
END
);
--
WITH
_derived_search_cols AS (
SELECT
CASE -- we only care about certain normalized engines
WHEN engine IS NULL THEN 'none'
WHEN STARTS_WITH(engine, 'google') THEN 'google'
WHEN STARTS_WITH(engine, 'ddg') OR STARTS_WITH(engine, 'duckduckgo') THEN 'ddg'
WHEN STARTS_WITH(engine, 'bing') THEN 'bing'
WHEN STARTS_WITH(engine, 'yandex') THEN 'yandex'
ELSE 'other'
END AS short_engine,
udf_strict_normalize_search_engine(engine) AS short_engine,
COALESCE(organic, 0) + COALESCE(sap, 0) + COALESCE(unknown, 0) + COALESCE(tagged_sap, 0) + COALESCE(tagged_follow_on, 0) AS total_searches,
COALESCE(tagged_sap, 0) + COALESCE(tagged_follow_on, 0) AS tagged_searches,
COALESCE(ad_click, 0) AS ad_click,

3
templates/search/search_aggregates/view.sql
Просмотреть файл

@ -22,7 +22,8 @@ SELECT
os,
os_version,
IFNULL(client_count, 0) AS client_count,
NULL AS default_private_search_engine
NULL AS default_private_search_engine,
NULL AS normalized_engine
FROM
`moz-fx-data-derived-datasets.search.search_aggregates_v6`
WHERE

2
templates/search_derived/mobile_search_aggregates_v1/query.sql
Просмотреть файл

@ -1,6 +1,7 @@
SELECT
submission_date,
engine,
normalized_engine,
source,
app_name,
app_version,
@ -21,6 +22,7 @@ WHERE
GROUP BY
submission_date,
engine,
normalized_engine,
source,
app_name,
app_version,

75
templates/search_derived/mobile_search_clients_daily_v1/query.sql
Просмотреть файл

@ -121,41 +121,48 @@ combined_search_clients AS (
sample_id
FROM
fenix_flattened_searches
),
unfiltered_search_clients AS (
SELECT
submission_date,
client_id,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(0)]) AS engine,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(1)]) AS source,
app_name,
SUM(
IF(
ARRAY_LENGTH(normalized_search_key) = 0
OR search_count > 10000,
0,
search_count
)
) AS search_count,
udf_mode_last(ARRAY_AGG(country)) AS country,
udf_mode_last(ARRAY_AGG(locale)) AS locale,
udf_mode_last(ARRAY_AGG(app_version)) AS app_version,
udf_mode_last(ARRAY_AGG(channel)) AS channel,
udf_mode_last(ARRAY_AGG(os)) AS os,
udf_mode_last(ARRAY_AGG(os_version)) AS os_version,
udf_mode_last(ARRAY_AGG(default_search_engine)) AS default_search_engine,
udf_mode_last(ARRAY_AGG(default_search_engine_submission_url)) AS default_search_engine_submission_url,
udf_mode_last(ARRAY_AGG(distribution_id)) AS distribution_id,
udf_mode_last(ARRAY_AGG(profile_creation_date)) AS profile_creation_date,
udf_mode_last(ARRAY_AGG(profile_age_in_days)) AS profile_age_in_days,
udf_mode_last(ARRAY_AGG(sample_id)) AS sample_id
FROM
combined_search_clients
WHERE
submission_date = @submission_date
GROUP BY
submission_date,
client_id,
engine,
source,
app_name
)
SELECT
submission_date,
client_id,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(0)]) AS engine,
IF(search_count > 10000, NULL, normalized_search_key[SAFE_OFFSET(1)]) AS source,
app_name,
SUM(
IF(
ARRAY_LENGTH(normalized_search_key) = 0
OR search_count > 10000,
0,
search_count
)
) AS search_count,
udf_mode_last(ARRAY_AGG(country)) AS country,
udf_mode_last(ARRAY_AGG(locale)) AS locale,
udf_mode_last(ARRAY_AGG(app_version)) AS app_version,
udf_mode_last(ARRAY_AGG(channel)) AS channel,
udf_mode_last(ARRAY_AGG(os)) AS os,
udf_mode_last(ARRAY_AGG(os_version)) AS os_version,
udf_mode_last(ARRAY_AGG(default_search_engine)) AS default_search_engine,
udf_mode_last(ARRAY_AGG(default_search_engine_submission_url)) AS default_search_engine_submission_url,
udf_mode_last(ARRAY_AGG(distribution_id)) AS distribution_id,
udf_mode_last(ARRAY_AGG(profile_creation_date)) AS profile_creation_date,
udf_mode_last(ARRAY_AGG(profile_age_in_days)) AS profile_age_in_days,
udf_mode_last(ARRAY_AGG(sample_id)) AS sample_id
*,
udf_strict_normalize_search_engine(engine) AS normalized_engine
FROM
combined_search_clients
WHERE
submission_date = @submission_date
GROUP BY
submission_date,
client_id,
engine,
source,
app_name
unfiltered_search_clients

2
templates/search_derived/search_aggregates_v8/query.sql
Просмотреть файл

@ -5,6 +5,7 @@ SELECT
country,
distribution_id,
engine,
normalized_engine,
locale,
search_cohort,
source,
@ -32,6 +33,7 @@ GROUP BY
country,
distribution_id,
engine,
normalized_engine,
locale,
search_cohort,
source,

1
templates/search_derived/search_clients_daily_v8/query.sql
Просмотреть файл

@ -109,6 +109,7 @@ WITH
submission_date,
client_id,
engine,
udf_strict_normalize_search_engine(engine) AS normalized_engine,
source,
udf_mode_last(ARRAY_AGG(country) OVER w1) AS country,
udf_mode_last(ARRAY_AGG(get_search_addon_version(active_addons)) OVER w1) AS addon_version,

9
templates/search_derived/search_clients_last_seen_v1/query.sql
Просмотреть файл

@ -1,14 +1,7 @@
WITH
_derived_search_cols AS (
SELECT
CASE -- we only care about certain normalized engines
WHEN engine IS NULL THEN 'none'
WHEN STARTS_WITH(engine, 'google') THEN 'google'
WHEN STARTS_WITH(engine, 'ddg') OR STARTS_WITH(engine, 'duckduckgo') THEN 'ddg'
WHEN STARTS_WITH(engine, 'bing') THEN 'bing'
WHEN STARTS_WITH(engine, 'yandex') THEN 'yandex'
ELSE 'other'
END AS short_engine,
udf_strict_normalize_search_engine(engine) AS short_engine,
COALESCE(organic, 0) + COALESCE(sap, 0) + COALESCE(unknown, 0) + COALESCE(tagged_sap, 0) + COALESCE(tagged_follow_on, 0) AS total_searches,
COALESCE(tagged_sap, 0) + COALESCE(tagged_follow_on, 0) AS tagged_searches,
COALESCE(ad_click, 0) AS ad_click,

12
tests/search_derived/mobile_search_aggregates_v1/test_aggregation/expect.ndjson
Просмотреть файл

@ -1,6 +1,6 @@
{ "submission_date": "2019-12-01", "engine": "engine1", "source": "action", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 5 }
{ "submission_date": "2019-12-01", "engine": "engine1", "source": "suggestion", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 1 }
{ "submission_date": "2019-12-01", "engine": "engine2", "source": "suggestion", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 5 }
{ "submission_date": "2019-12-01", "engine": "engine2", "source": "suggestion", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "US", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 3 }
{ "submission_date": "2019-12-01", "engine": "engine1", "source": "actionbar", "app_name": "Fennec", "app_version": "68.2.1", "country": "CA", "locale": "en-CA", "default_search_engine": "engine1", "os": "Android", "os_version": "26", "client_count": 1, "search_count": 3 }
{ "submission_date": "2019-12-01", "engine": "engine2", "source": "actionbar", "app_name": "Fennec", "app_version": "68.2.1", "country": "CA", "locale": "en-CA", "default_search_engine": "engine1", "os": "Android", "os_version": "26", "client_count": 1, "search_count": 1 }
{ "submission_date": "2019-12-01", "engine": "engine1", "normalized_engine": "Other", "source": "action", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 5 }
{ "submission_date": "2019-12-01", "engine": "engine1", "normalized_engine": "Other", "source": "suggestion", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 1 }
{ "submission_date": "2019-12-01", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "CA", "locale": "en-CA", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 5 }
{ "submission_date": "2019-12-01", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "app_name": "Fenix", "app_version": "Nightly 191124 06:01", "channel": "release", "country": "US", "os": "Android", "os_version": "29", "client_count": 1, "search_count": 3 }
{ "submission_date": "2019-12-01", "engine": "engine1", "normalized_engine": "Other", "source": "actionbar", "app_name": "Fennec", "app_version": "68.2.1", "country": "CA", "locale": "en-CA", "default_search_engine": "engine1", "os": "Android", "os_version": "26", "client_count": 1, "search_count": 3 }
{ "submission_date": "2019-12-01", "engine": "engine2", "normalized_engine": "Other", "source": "actionbar", "app_name": "Fennec", "app_version": "68.2.1", "country": "CA", "locale": "en-CA", "default_search_engine": "engine1", "os": "Android", "os_version": "26", "client_count": 1, "search_count": 1 }

12
tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.ndjson
Просмотреть файл

@ -1,10 +1,10 @@
{"submission_date": "2019-12-01", "client_id": "a", "search_count": 0, "locale": "fr-FR", "app_version": "68.2.1", "app_name": "Fennec", "country": "FR", "os": "Android", "os_version": "23", "profile_creation_date": 16606, "profile_age_in_days": 1625, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine1", "source": "actionbar", "search_count": 3, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "c", "engine": "engine2", "source": "actionbar", "search_count": 1, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine1", "normalized_engine": "Other", "source": "actionbar", "search_count": 3, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "c", "engine": "engine2", "normalized_engine": "Other", "source": "actionbar", "search_count": 1, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "d", "search_count": 0, "default_search_engine": "engine2", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17653, "profile_age_in_days": 578, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "e", "channel": "release", "search_count": 0, "locale": "en-US", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18207, "profile_age_in_days": 24, "sample_id": 1, "default_search_engine": "engine1", "default_search_engine_submission_url": "engine1.url"}
{"submission_date": "2019-12-01", "client_id": "f", "channel": "release", "engine": "engine1", "source": "action", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "g", "channel": "release", "engine": "engine1", "source": "suggestion", "search_count": 1, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "h", "channel": "release", "engine": "engine2", "source": "suggestion", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "f", "channel": "release", "engine": "engine1", "normalized_engine": "Other", "source": "action", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "g", "channel": "release", "engine": "engine1", "normalized_engine": "Other", "source": "suggestion", "search_count": 1, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "h", "channel": "release", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "i", "channel": "release", "search_count": 0, "locale": "ar-EG", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "EG", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "h", "channel": "release", "engine": "engine2", "source": "suggestion", "search_count": 3, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "h", "channel": "release", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "search_count": 3, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}

5
tests/search_derived/mobile_search_aggregates_v1/test_aggregation/mobile_search_clients_daily_v1.schema.json
Просмотреть файл

@ -14,6 +14,11 @@
"name": "engine",
"mode": "NULLABLE"
},
{
"type": "STRING",
"name": "normalized_engine",
"mode": "NULLABLE"
},
{
"type": "STRING",
"name": "source",

12
tests/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.ndjson
Просмотреть файл

@ -1,10 +1,10 @@
{"submission_date": "2019-12-01", "client_id": "a", "search_count": 0, "locale": "fr-FR", "app_version": "68.2.1", "app_name": "Fennec", "country": "FR", "os": "Android", "os_version": "23", "profile_creation_date": 16606, "profile_age_in_days": 1625, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine1", "source": "actionbar", "search_count": 3, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine2", "source": "actionbar", "search_count": 1, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine1", "normalized_engine": "Other", "source": "actionbar", "search_count": 3, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine2", "normalized_engine": "Other", "source": "actionbar", "search_count": 1, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "c", "search_count": 0, "default_search_engine": "engine2", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17653, "profile_age_in_days": 578, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "a", "channel": "release", "search_count": 0, "locale": "en-US", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18207, "profile_age_in_days": 24, "sample_id": 1, "default_search_engine": "engine1", "default_search_engine_submission_url": "engine1.url"}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine1", "source": "action", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine1", "source": "suggestion", "search_count": 1, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine2", "source": "suggestion", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine1", "normalized_engine": "Other", "source": "action", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine1", "normalized_engine": "Other", "source": "suggestion", "search_count": 1, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "search_count": 5, "locale": "en-CA", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "c", "channel": "release", "search_count": 0, "locale": "ar-EG", "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "EG", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "d", "channel": "release", "engine": "engine2", "source": "suggestion", "search_count": 3, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "d", "channel": "release", "engine": "engine2", "normalized_engine": "Other", "source": "suggestion", "search_count": 3, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}

4
tests/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.ndjson
Просмотреть файл

@ -1,6 +1,6 @@
{"submission_date": "2019-12-01", "client_id": "a", "search_count": 0, "locale": "fr-FR", "app_version": "68.2.1", "app_name": "Fennec", "country": "FR", "os": "Android", "os_version": "23", "profile_creation_date": 16606, "profile_age_in_days": 1625, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine1", "source": "actionbar", "search_count": 2, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "engine": "engine1", "normalized_engine": "Other", "source": "actionbar", "search_count": 2, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "search_count": 0, "default_search_engine": "engine1", "locale": "en-CA", "app_version": "68.2.1", "app_name": "Fennec", "country": "CA", "os": "Android", "os_version": "26", "profile_creation_date": 17332, "profile_age_in_days": 899, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "a", "channel": "release", "search_count": 0, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "US", "os": "Android", "os_version": "29", "profile_creation_date": 18207, "profile_age_in_days": 24, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine1", "source": "action", "search_count": 3, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "engine": "engine1", "normalized_engine": "Other", "source": "action", "search_count": 3, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}
{"submission_date": "2019-12-01", "client_id": "b", "channel": "release", "search_count": 0, "app_version": "Nightly 191124 06:01", "app_name": "Fenix", "country": "CA", "os": "Android", "os_version": "29", "profile_creation_date": 18201, "profile_age_in_days": 30, "sample_id": 1}

10
tests/search_derived/search_aggregates_v8/test_aggregation/expect.ndjson
Просмотреть файл

@ -1,5 +1,5 @@
{"submission_date": "2017-01-01", "addon_version": "0.9.5", "app_version": "54.0.1", "country": "DE", "engine": "bing", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "addon_version": "0.9.5", "app_version": "54.0.1", "country": "DE", "distribution_id": "totally not null", "engine": "google", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 28, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "addon_version": "0.9.5", "app_version": "54.0.1", "country": "DE", "engine": "yahoo", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "addon_version": "0.9.6", "app_version": "54.0.1", "country": "US", "engine": "google", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "app_version": "54.0.1", "country": "DE", "engine": "yahoo", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "default_private_search_engine": "engine1"}
{"submission_date": "2017-01-01", "addon_version": "0.9.5", "app_version": "54.0.1", "country": "DE", "engine": "bing", "normalized_engine": "Bing", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "addon_version": "0.9.5", "app_version": "54.0.1", "country": "DE", "distribution_id": "totally not null", "engine": "google", "normalized_engine": "Google", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 28, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "addon_version": "0.9.5", "app_version": "54.0.1", "country": "DE", "engine": "yahoo", "normalized_engine": "Other", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "addon_version": "0.9.6", "app_version": "54.0.1", "country": "US", "engine": "google", "normalized_engine": "Google", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0}
{"submission_date": "2017-01-01", "app_version": "54.0.1", "country": "DE", "engine": "yahoo", "normalized_engine": "Other", "locale": "de", "source": "urlbar", "default_search_engine": "google", "os": "windows", "os_version": "10.0", "client_count": 1, "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "sap": 4, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "default_private_search_engine": "engine1"}

10
tests/search_derived/search_aggregates_v8/test_aggregation/search_clients_daily_v8.ndjson
Просмотреть файл

@ -1,6 +1,6 @@
{"engine": "google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "b", "country": "US", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.6"}
{"engine": "google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 28, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "distribution_id": "totally not null", "addon_version": "0.9.5"}
{"engine": "bing", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5"}
{"engine": "yahoo", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5"}
{"engine": "google", "normalized_engine": "Google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "b", "country": "US", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.6"}
{"engine": "google", "normalized_engine": "Google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 28, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "distribution_id": "totally not null", "addon_version": "0.9.5"}
{"engine": "bing", "normalized_engine": "Bing", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5"}
{"engine": "yahoo", "normalized_engine": "Other", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5"}
{"locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "sample_id": 42, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "c", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "sap": 0}
{"engine": "yahoo", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "sample_id": 42, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "c", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "sap": 4, "default_private_search_engine": "engine1", "default_private_search_engine_data_load_path": "engine1.load_path", "default_private_search_engine_data_submission_url": "https://www.engine.com"}
{"engine": "yahoo", "normalized_engine": "Other", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "sample_id": 42, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "c", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "sap": 4, "default_private_search_engine": "engine1", "default_private_search_engine_data_load_path": "engine1.load_path", "default_private_search_engine_data_submission_url": "https://www.engine.com"}

5
tests/search_derived/search_aggregates_v8/test_aggregation/search_clients_daily_v8.schema.json
Просмотреть файл

@ -29,6 +29,11 @@
"name": "engine",
"mode": "NULLABLE"
},
{
"type": "STRING",
"name": "normalized_engine",
"mode": "NULLABLE"
},
{
"type": "STRING",
"name": "locale",

16
tests/search_derived/search_clients_daily_v8/test_aggregation/expect.ndjson
Просмотреть файл

@ -1,9 +1,9 @@
{"engine": "google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "b", "country": "US", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.6", "subsession_hours_sum": 1.0,"sessions_started_on_this_day": 1,"active_addons_count_mean": 2.0,"max_concurrent_tab_count_max": 10,"tab_open_event_count_sum": 5,"active_hours_sum": 0.5,"total_uri_count": 1}
{"engine": "google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 28, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "distribution_id": "totally not null", "addon_version": "0.9.5", "subsession_hours_sum": 8.0, "sessions_started_on_this_day": 8, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 40, "active_hours_sum": 4.0, "total_uri_count": 16}
{"engine": "bing", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5", "subsession_hours_sum": 8.0, "sessions_started_on_this_day": 8, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 40, "active_hours_sum": 4.0, "total_uri_count": 16}
{"engine": "yahoo", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5", "subsession_hours_sum": 8.0, "sessions_started_on_this_day": 8, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 40, "active_hours_sum": 4.0, "total_uri_count": 16}
{"engine": "google", "normalized_engine": "Google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "b", "country": "US", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.6", "subsession_hours_sum": 1.0,"sessions_started_on_this_day": 1,"active_addons_count_mean": 2.0,"max_concurrent_tab_count_max": 10,"tab_open_event_count_sum": 5,"active_hours_sum": 0.5,"total_uri_count": 1}
{"engine": "google", "normalized_engine": "Google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 28, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "distribution_id": "totally not null", "addon_version": "0.9.5", "subsession_hours_sum": 8.0, "sessions_started_on_this_day": 8, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 40, "active_hours_sum": 4.0, "total_uri_count": 16}
{"engine": "bing", "normalized_engine": "Bing", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5", "subsession_hours_sum": 8.0, "sessions_started_on_this_day": 8, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 40, "active_hours_sum": 4.0, "total_uri_count": 16}
{"engine": "yahoo", "normalized_engine": "Other", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "a", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.5", "subsession_hours_sum": 8.0, "sessions_started_on_this_day": 8, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 40, "active_hours_sum": 4.0, "total_uri_count": 16}
{"locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "sample_id": 42, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "c", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "sap": 0, "subsession_hours_sum": 1.0, "sessions_started_on_this_day": 1, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 5, "active_hours_sum": 0.5, "total_uri_count": 2}
{"engine": "google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 8, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "d", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.4", "subsession_hours_sum": 3.0, "sessions_started_on_this_day": 3, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 15, "active_hours_sum": 1.5, "total_uri_count": 7}
{"engine": "bing", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "d", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.4", "subsession_hours_sum": 3.0, "sessions_started_on_this_day": 3, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 15, "active_hours_sum": 1.5, "total_uri_count": 7}
{"engine": "yahoo", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "d", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.4", "subsession_hours_sum": 3.0, "sessions_started_on_this_day": 3, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 15, "active_hours_sum": 1.5, "total_uri_count": 7}
{"engine": "google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "e", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "default_private_search_engine": "engine1", "default_private_search_engine_data_load_path": "engine1.load_path", "default_private_search_engine_data_submission_url": "https://www.engine.com", "subsession_hours_sum": 1.0, "sessions_started_on_this_day": 1, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 5, "active_hours_sum": 0.5, "total_uri_count": 2}
{"engine": "google", "normalized_engine": "Google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 8, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "d", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.4", "subsession_hours_sum": 3.0, "sessions_started_on_this_day": 3, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 15, "active_hours_sum": 1.5, "total_uri_count": 7}
{"engine": "bing", "normalized_engine": "Bing", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "d", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.4", "subsession_hours_sum": 3.0, "sessions_started_on_this_day": 3, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 15, "active_hours_sum": 1.5, "total_uri_count": 7}
{"engine": "yahoo", "normalized_engine": "Other", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "d", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "addon_version": "0.9.4", "subsession_hours_sum": 3.0, "sessions_started_on_this_day": 3, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 15, "active_hours_sum": 1.5, "total_uri_count": 7}
{"engine": "google", "normalized_engine": "Google", "locale": "de", "default_search_engine_data_submission_url": "https://www.google.com/search?q=&ie=utf-8&oe=utf-8&client=firefox-b", "user_pref_browser_search_region": "DE", "sample_id": 42, "sap": 4, "default_search_engine_data_load_path": "jar:[app]/omni.ja!browser/google.xml", "os_version": "10.0", "source": "urlbar", "default_search_engine": "google", "app_version": "54.0.1", "channel": "release", "submission_date": "2017-01-01", "client_id": "e", "country": "DE", "profile_age_in_days": 366, "profile_creation_date": 16801, "os": "windows", "organic": 0, "tagged_sap": 0, "tagged_follow_on": 0, "ad_click": 0, "search_with_ads": 0, "unknown": 0, "default_private_search_engine": "engine1", "default_private_search_engine_data_load_path": "engine1.load_path", "default_private_search_engine_data_submission_url": "https://www.engine.com", "subsession_hours_sum": 1.0, "sessions_started_on_this_day": 1, "active_addons_count_mean": 2.0, "max_concurrent_tab_count_max": 10, "tab_open_event_count_sum": 5, "active_hours_sum": 0.5, "total_uri_count": 2}

6
tests/search_derived/search_clients_last_seen_v1/test_single_day/expect.ndjson
Просмотреть файл

@ -1,3 +1,3 @@
{"submission_date":"2019-10-02","client_id":"client1","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "engine_searches": [{"key": "google", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "RE", "user_pref_browser_search_region": "RE", "app_version": "69.0.1", "default_search_engine": "other-Yahoo Web", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 0, "active_addons_count_mean": 1, "max_concurrent_tab_count_max": 1, "tab_open_event_count_sum": 2, "active_hours_sum": 2, "subsession_hours_sum": 2, "sessions_started_on_this_day": 2, "total_searches":6, "tagged_searches": 0, "ad_click": 1, "search_with_ads": 0, "organic": 0, "sap": 6, "profile_creation_date": 18171, "os": "Windows_NT", "os_version": "1", "channel": "release"}
{"submission_date":"2019-10-02","client_id":"client2","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "bing", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}, {"key": "ddg", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "EE", "user_pref_browser_search_region": "EE", "app_version": "69.0", "default_search_engine": "other", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 2, "active_addons_count_mean": 2, "max_concurrent_tab_count_max": 2, "tab_open_event_count_sum": 4, "active_hours_sum": 4, "subsession_hours_sum": 4, "sessions_started_on_this_day": 4, "total_searches": 4, "tagged_searches": 2, "ad_click": 0, "search_with_ads": 0, "organic": 0, "sap": 2, "tagged_sap": 2, "profile_creation_date": 17764, "os":"Linux", "default_search_engine_data_submission_url": "https://duckduckgo.com/html/", "os_version": "2", "channel": "release"}
{"submission_date":"2019-10-02", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "google", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}
{"submission_date":"2019-10-02","client_id":"client1","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "engine_searches": [{"key": "Google", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "RE", "user_pref_browser_search_region": "RE", "app_version": "69.0.1", "default_search_engine": "other-Yahoo Web", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 0, "active_addons_count_mean": 1, "max_concurrent_tab_count_max": 1, "tab_open_event_count_sum": 2, "active_hours_sum": 2, "subsession_hours_sum": 2, "sessions_started_on_this_day": 2, "total_searches":6, "tagged_searches": 0, "ad_click": 1, "search_with_ads": 0, "organic": 0, "sap": 6, "profile_creation_date": 18171, "os": "Windows_NT", "os_version": "1", "channel": "release"}
{"submission_date":"2019-10-02","client_id":"client2","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Bing", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}, {"key": "DuckDuckGo", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "EE", "user_pref_browser_search_region": "EE", "app_version": "69.0", "default_search_engine": "other", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 2, "active_addons_count_mean": 2, "max_concurrent_tab_count_max": 2, "tab_open_event_count_sum": 4, "active_hours_sum": 4, "subsession_hours_sum": 4, "sessions_started_on_this_day": 4, "total_searches": 4, "tagged_searches": 2, "ad_click": 0, "search_with_ads": 0, "organic": 0, "sap": 2, "tagged_sap": 2, "profile_creation_date": 17764, "os":"Linux", "default_search_engine_data_submission_url": "https://duckduckgo.com/html/", "os_version": "2", "channel": "release"}
{"submission_date":"2019-10-02", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Google", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}

2
tests/search_derived/search_clients_last_seen_v1/test_single_day/search_clients_last_seen_v1.ndjson
Просмотреть файл

@ -1,2 +1,2 @@
{"submission_date":"2019-10-01", "client_id": "client2", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 2}
{"submission_date":"2019-10-01", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "google", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}
{"submission_date":"2019-10-01", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Google", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}

24
udf/normalize_search_engine.sql
Просмотреть файл

@ -6,22 +6,11 @@ Return normalized engine name for recognized engines
CREATE TEMP FUNCTION
udf_normalize_search_engine(engine STRING) AS (
CASE
WHEN STARTS_WITH(engine, 'google')
OR STARTS_WITH(engine, 'Google')
OR STARTS_WITH(engine, 'other-Google') THEN 'Google'
WHEN STARTS_WITH(engine, 'ddg')
OR STARTS_WITH(engine, 'duckduckgo')
OR STARTS_WITH(engine, 'DuckDuckGo')
OR STARTS_WITH(engine, 'other-DuckDuckGo') THEN 'DuckDuckGo'
WHEN STARTS_WITH(engine, 'bing')
OR STARTS_WITH(engine, 'Bing')
OR STARTS_WITH(engine, 'other-Bing') THEN 'Bing'
WHEN STARTS_WITH(engine, 'yandex')
OR STARTS_WITH(engine, 'Yandex')
OR STARTS_WITH(engine, 'other-Yandex') THEN 'Yandex'
ELSE engine
END
IF(
udf_strict_normalize_search_engine(engine) = 'Other',
engine,
udf_strict_normalize_search_engine(engine)
)
);
-- Test
@ -30,5 +19,6 @@ SELECT
assert_equals('Google', udf_normalize_search_engine('google')),
assert_equals('Google', udf_normalize_search_engine('Google-abc')),
assert_equals('not-bing', udf_normalize_search_engine('not-bing')),
assert_equals('engine', udf_normalize_search_engine('engine'))
assert_equals('engine', udf_normalize_search_engine('engine')),
assert_null(udf_normalize_search_engine(NULL))

36
udf/strict_normalize_search_engine.sql Normal file
Просмотреть файл

@ -0,0 +1,36 @@
/*
Return normalized engine name for recognized engines
*/
CREATE TEMP FUNCTION
udf_strict_normalize_search_engine(engine STRING) AS (
CASE
WHEN engine IS NULL THEN NULL
WHEN STARTS_WITH(engine, 'google')
OR STARTS_WITH(engine, 'Google')
OR STARTS_WITH(engine, 'other-Google') THEN 'Google'
WHEN STARTS_WITH(engine, 'ddg')
OR STARTS_WITH(engine, 'duckduckgo')
OR STARTS_WITH(engine, 'DuckDuckGo')
OR STARTS_WITH(engine, 'other-DuckDuckGo') THEN 'DuckDuckGo'
WHEN STARTS_WITH(engine, 'bing')
OR STARTS_WITH(engine, 'Bing')
OR STARTS_WITH(engine, 'other-Bing') THEN 'Bing'
WHEN STARTS_WITH(engine, 'yandex')
OR STARTS_WITH(engine, 'Yandex')
OR STARTS_WITH(engine, 'other-Yandex') THEN 'Yandex'
ELSE 'Other'
END
);
-- Test
SELECT
assert_equals('Google', udf_strict_normalize_search_engine('google')),
assert_equals('Google', udf_strict_normalize_search_engine('Google-abc')),
assert_equals('Other', udf_strict_normalize_search_engine('not-bing')),
assert_equals('Other', udf_strict_normalize_search_engine('engine')),
assert_null(udf_strict_normalize_search_engine(NULL))