mozilla
/
bigquery-etl
зеркало из https://github.com/mozilla/bigquery-etl.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1742183 Added iOS probes to mobile_search_clients_daily (#2526)

This commit is contained in:
Alexander Nicholson 2021-11-25 17:19:49 -05:00 коммит произвёл GitHub
Родитель 8bab87735f
Коммит 48d3ac3f60
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
21 изменённых файлов: 552 добавлений и 9 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
bigquery_etl/search/templates/ios_metrics.template.sql
Просмотреть файл

@ -15,9 +15,9 @@ metrics_{{ namespace }} AS (
CAST(NULL AS STRING) AS default_search_engine_submission_url,
sample_id,
metrics.labeled_counter.search_counts AS search_count,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_ad_clicks,
extract_ios_provider(metrics.labeled_counter.browser_search_ad_clicks) as search_ad_clicks,
metrics.labeled_counter.search_in_content AS search_in_content,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_with_ads,
extract_ios_provider(metrics.labeled_counter.browser_search_with_ads) as search_with_ads,
client_info.first_run_date,
ping_info.end_time,
ping_info.experiments,

13
bigquery_etl/search/templates/mobile_search_clients_daily.template.sql
Просмотреть файл

@ -75,6 +75,19 @@ CREATE TEMP FUNCTION null_search() AS (
[STRUCT<key STRING, value INT64>(NULL, 0)]
);
CREATE TEMP FUNCTION extract_ios_provider(list ARRAY<STRUCT<key STRING, value INT64>>) AS (
ARRAY(
SELECT
STRUCT(
SPLIT(key, '-')[SAFE_OFFSET(1)] as key,
value as value
)
FROM
UNNEST(list)
)
);
WITH core_flattened_searches AS (
SELECT
*,

16
sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql
Просмотреть файл

@ -55,6 +55,10 @@ CREATE TEMP FUNCTION null_search() AS (
[STRUCT<key STRING, value INT64>(NULL, 0)]
);
CREATE TEMP FUNCTION extract_ios_provider(list ARRAY<STRUCT<key STRING, value INT64>>) AS (
ARRAY(SELECT STRUCT(SPLIT(key, '-')[SAFE_OFFSET(1)] AS key, value AS value) FROM UNNEST(list))
);
WITH core_flattened_searches AS (
SELECT
*,
@ -267,9 +271,9 @@ metrics_org_mozilla_ios_firefox AS (
CAST(NULL AS STRING) AS default_search_engine_submission_url,
sample_id,
metrics.labeled_counter.search_counts AS search_count,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_ad_clicks,
extract_ios_provider(metrics.labeled_counter.browser_search_ad_clicks) AS search_ad_clicks,
metrics.labeled_counter.search_in_content AS search_in_content,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_with_ads,
extract_ios_provider(metrics.labeled_counter.browser_search_with_ads) AS search_with_ads,
client_info.first_run_date,
ping_info.end_time,
ping_info.experiments,
@ -297,9 +301,9 @@ metrics_org_mozilla_ios_firefoxbeta AS (
CAST(NULL AS STRING) AS default_search_engine_submission_url,
sample_id,
metrics.labeled_counter.search_counts AS search_count,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_ad_clicks,
extract_ios_provider(metrics.labeled_counter.browser_search_ad_clicks) AS search_ad_clicks,
metrics.labeled_counter.search_in_content AS search_in_content,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_with_ads,
extract_ios_provider(metrics.labeled_counter.browser_search_with_ads) AS search_with_ads,
client_info.first_run_date,
ping_info.end_time,
ping_info.experiments,
@ -327,9 +331,9 @@ metrics_org_mozilla_ios_fennec AS (
CAST(NULL AS STRING) AS default_search_engine_submission_url,
sample_id,
metrics.labeled_counter.search_counts AS search_count,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_ad_clicks,
extract_ios_provider(metrics.labeled_counter.browser_search_ad_clicks) AS search_ad_clicks,
metrics.labeled_counter.search_in_content AS search_in_content,
ARRAY<STRUCT<key STRING, value INT64>>[] AS search_with_ads,
extract_ios_provider(metrics.labeled_counter.browser_search_with_ads) AS search_with_ads,
client_info.first_run_date,
ping_info.end_time,
ping_info.experiments,

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/expect.yaml
Просмотреть файл

@ -258,15 +258,49 @@
engine: engine2
source: organic.none
organic: 7
- <<: *ios_base
default_search_engine: engine2
engine: engine10
source: search-with-ads
search_with_ads: 22
- <<: *ios_base
default_search_engine: engine2
engine: engine11
source: ad-click
ad_click: 5
- <<: *ios_base
client_id: beta_ios
channel: beta
engine: engine2
source: in-content.sap.code
tagged_sap: 2
- <<: *ios_base
client_id: beta_ios
channel: beta
engine: engine9
source: ad-click
ad_click: 4
- <<: *ios_base
client_id: beta_ios
channel: beta
engine: engine8
source: search-with-ads
search_with_ads: 15
- <<: *ios_base
client_id: nightly_ios
channel: nightly
engine: engine2
source: in-content.sap-follow-on.code
tagged_follow_on: 1
- <<: *ios_base
client_id: nightly_ios
channel: nightly
engine: engine6
source: search-with-ads
search_with_ads: 11
- <<: *ios_base
client_id: nightly_ios
channel: nightly
engine: engine7
source: ad-click
ad_click: 6

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_fennec.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

6
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_fennec.metrics.yaml
Просмотреть файл

@ -17,3 +17,9 @@
search_in_content:
- key: engine2.in-content.sap-follow-on.code
value: "1"
browser_search_with_ads:
- key: provider-engine6
value: 11
browser_search_ad_clicks:
- key: provider-engine7
value: 6

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_firefox.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

12
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_firefox.metrics.yaml
Просмотреть файл

@ -26,6 +26,12 @@
search_in_content:
- key: engine2.organic.none
value: "3"
browser_search_with_ads:
- key: provider-engine10
value: 10
browser_search_ad_clicks:
- key: provider-engine11
value: 2
- <<: *base
metrics:
string:
@ -34,3 +40,9 @@
search_in_content:
- key: engine2.organic.none
value: "4"
browser_search_with_ads:
- key: provider-engine10
value: 12
browser_search_ad_clicks:
- key: provider-engine11
value: 3

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_firefoxbeta.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

6
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_aggregation/org_mozilla_ios_firefoxbeta.metrics.yaml
Просмотреть файл

@ -17,3 +17,9 @@
search_in_content:
- key: engine2.in-content.sap.code
value: "2"
browser_search_with_ads:
- key: provider-engine8
value: 15
browser_search_ad_clicks:
- key: provider-engine9
value: 4

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/org_mozilla_ios_fennec.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/org_mozilla_ios_firefox.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_ios_version_filter/org_mozilla_ios_firefoxbeta.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_ios_fennec.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_ios_firefox.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_namespace_union/org_mozilla_ios_firefoxbeta.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

32
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/expect.yaml
Просмотреть файл

@ -85,3 +85,35 @@
os_version: '29'
profile_creation_date: 18201
profile_age_in_days: 30
- <<: *base
client_id: release_ios
channel: release
locale: en-US
country: US
os: iOS
os_version: "35.0"
default_search_engine: engine2
- <<: *base
client_id: release_ios
channel: release
locale: en-US
country: US
os: iOS
os_version: "35.0"
default_search_engine: engine2
- <<: *base
client_id: release_ios
channel: release
locale: en-US
country: US
os: iOS
os_version: "35.0"
default_search_engine: engine2
- <<: *base
client_id: release_ios
channel: release
locale: en-US
country: US
os: iOS
os_version: "35.0"
default_search_engine: engine2

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_fennec.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_firefox.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",

30
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_firefox.metrics.yaml
Просмотреть файл

@ -1,2 +1,30 @@
---
[]
- submission_timestamp: 2019-12-01 09:13:09.842719 UTC
ping_info:
end_time: 2019-12-01T04:13-05:00
normalized_channel: release
normalized_os: iOS
sample_id: 1
client_info:
app_display_version: "68.2.1"
client_id: release_ios
os_version: "35.0"
locale: en-US
first_run_date: 2015-06-20-05:00
normalized_country_code: US
metrics:
string:
search_default_engine: engine2
labeled_counter:
search_counts:
- key: engine1
value: 10001
search_in_content:
- key: engine2.organic.none
value: 20000
browser_search_with_ads:
- key: provider-engine10
value: 30000
browser_search_ad_clicks:
- key: provider-engine11
value: 40000

34
tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/test_overactive_filter/org_mozilla_ios_firefoxbeta.metrics.schema.json
Просмотреть файл

@ -60,6 +60,40 @@
"mode": "REPEATED",
"name": "search_in_content",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_ad_clicks",
"type": "RECORD"
},
{
"fields": [
{
"mode": "NULLABLE",
"name": "key",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "browser_search_with_ads",
"type": "RECORD"
}
],
"type": "RECORD",