mozilla
/
bigquery-etl
зеркало из https://github.com/mozilla/bigquery-etl.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Fix `verify-format-sql` CI check not reporting SQL formatting issues (#4827) 

* Verify the format of the original SQL, not the generated SQL.

The generated SQL gets reformatted by `bqetl query render`.

* Format all SQL.

* Quote column names containing Jinja expressions to prevent `bqetl format` causing invalid SQL.

* Adjust indentation of some comments to align with the formatted SQL.

* Refactor final `SELECT` in `telemetry_derived.clients_first_seen_v2` to work better with `bqetl format` SQL formatting.

* Fix trailing line comments breaking inline block formatting.

* Fix leading whitespace before Jinja comments not being preserved.

* Add `schema.yaml` for `firefox_ios_derived.baseline_clients_yearly_v1`.

So the `deploy-changes-to-stage` CI can work for the downstream `firefox_ios.baseline_clients_yearly` view.

* Add `schema.yaml` for `firefox_accounts_derived/fxa_users_services_daily_v1`.

So the `dry-run-sql` CI can work for the downstream `firefox_accounts_derived.fxa_users_services_last_seen_v1` ETL.

* Correct `schema.yaml` and `init.sql` for `firefox_accounts_derived.fxa_users_last_seen_v1`.

So the `dry-run-sql` CI can work for the downstream `firefox_accounts_derived.fxa_users_last_seen_v1` ETL.

* Fully qualify table reference in `init.sql` for `firefox_accounts_derived.fxa_users_last_seen_v1`.

So the table dependency will get detected by the `deploy-changes-to-stage` CI to deploy it so the `dry-run-sql` CI can work for the `init.sql` file.

* Improve `JinjaComment` inheritance and docstring.

* Implement `Line.ends_with_line_comment` property and refactor `inline_block_format()`.
This commit is contained in:
Sean Rose 2024-01-22 11:48:08 -08:00 коммит произвёл GitHub
Родитель befe468aea
Коммит 0530c1dc81
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
71 изменённых файлов: 1714 добавлений и 961 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

24
.circleci/config.yml
Просмотреть файл

@ -66,15 +66,6 @@ jobs:
- checkout
- *restore_venv_cache
- *build
- &attach_generated_sql
attach_workspace:
at: /tmp/workspace
- &copy_generated_sql
run:
name: Move generated-sql into place
command: |
rm -rf sql/
cp -r /tmp/workspace/generated-sql/sql sql
- run:
name: Verify that SQL is correctly formatted
command: |
@ -98,7 +89,9 @@ jobs:
- checkout
- *restore_venv_cache
- *build
- *attach_generated_sql
- &attach_generated_sql
attach_workspace:
at: /tmp/workspace
- &copy_staged_sql
run:
name: Move sql deployed on stage into place
@ -204,7 +197,12 @@ jobs:
- *restore_venv_cache
- *build
- *attach_generated_sql
- *copy_generated_sql
- &copy_generated_sql
run:
name: Move generated-sql into place
command: |
rm -rf sql/
cp -r /tmp/workspace/generated-sql/sql sql
- run:
name: Generate DAGs
command: |
@ -686,9 +684,7 @@ workflows:
- manual-trigger-required-for-fork
- build:
context: data-eng-circleci-tests
- verify-format-sql:
requires:
- generate-sql
- verify-format-sql
- deploy-changes-to-stage:
requires:
- generate-sql

26
bigquery_etl/format_sql/formatter.py
Просмотреть файл

@ -21,6 +21,7 @@ from .tokenizer import (
JinjaComment,
JinjaExpression,
JinjaStatement,
LineComment,
Literal,
NewlineKeyword,
OpeningBracket,
@ -96,7 +97,7 @@ def simple_format(tokens, indent=" "):
# no space before statement separator
# no space before first token
pass
elif isinstance(token, (Comment, JinjaComment)):
elif isinstance(token, Comment):
# blank line before comments only if they start on their own line
# and come after a statement separator
if token.value.startswith("\n") and prev_was_statement_separator:
@ -145,7 +146,6 @@ def simple_format(tokens, indent=" "):
ExpressionSeparator,
StatementSeparator,
JinjaStatement,
JinjaComment,
),
)
allow_space_before_next_token = not isinstance(token, FieldAccessOperator)
@ -188,7 +188,7 @@ class Line:
def __init__(self, indent_token=None, can_format=True):
"""Initialize."""
self.indent_token = indent_token
self.can_format = can_format
self.can_format = can_format and not isinstance(indent_token, Comment)
if indent_token is None:
self.indent_level = 0
else:
@ -197,17 +197,11 @@ class Line:
self.indent_level -= 1
self.inline_tokens = []
self.inline_length = 0
self.can_format = can_format and not isinstance(
indent_token, (Comment, JinjaComment)
)
def add(self, token):
"""Add a token to this line."""
self.inline_length += len(token.value)
self.inline_tokens.append(token)
self.can_format = self.can_format and not isinstance(
token, (Comment, JinjaComment)
)
@property
def tokens(self):
@ -247,6 +241,11 @@ class Line:
"""Determine if this line starts with a ClosingBracket."""
return self.inline_tokens and isinstance(self.inline_tokens[0], ClosingBracket)
@property
def ends_with_line_comment(self):
"""Determine if this line ends with a line comment."""
return self.inline_tokens and isinstance(self.inline_tokens[-1], LineComment)
def inline_block_format(tokens, max_line_length=100):
"""Extend simple_format to inline each bracket block if possible.
@ -304,14 +303,17 @@ def inline_block_format(tokens, max_line_length=100):
line_length = indent_level + line.inline_length
pending_lines = 0
pending = []
last_token_was_opening_bracket = line.ends_with_opening_bracket
open_brackets = 1
index += 1 # start on the next line
previous_line = line
for line in lines[index:]:
if not line.can_format:
break
# Line comments can't be moved into the middle of a line.
if previous_line.ends_with_line_comment:
break
if (
not last_token_was_opening_bracket
not previous_line.ends_with_opening_bracket
and not line.starts_with_closing_bracket
):
pending.append(Whitespace(" "))
@ -334,7 +336,7 @@ def inline_block_format(tokens, max_line_length=100):
break
if line.ends_with_opening_bracket:
open_brackets += 1
last_token_was_opening_bracket = line.ends_with_opening_bracket
previous_line = line
def reformat(query, format_=inline_block_format, trailing_newline=False):

11
bigquery_etl/format_sql/tokenizer.py
Просмотреть файл

@ -785,13 +785,10 @@ class JinjaBlockMiddle(JinjaBlockEnd, JinjaBlockStart):
pattern = re.compile(r"{%[-+]? *(elif|else)\b.*?%}", re.DOTALL)
class JinjaComment(Comment):
"""Jinja comment delimiters {# #}.
class JinjaComment(BlockComment):
"""Jinja comment that may span multiple lines."""
May be followed by no whitespace or a new line and increased indent.
"""
pattern = re.compile(r"{#.*?#}", re.DOTALL)
pattern = re.compile(r"\n?[^\S\n]*{#.*?#}", re.DOTALL)
class OpeningBracket(Token):
@ -871,8 +868,8 @@ class FieldAccessOperator(Operator):
BIGQUERY_TOKEN_PRIORITY = [
LineComment,
BlockComment,
Whitespace,
JinjaComment,
Whitespace,
JinjaExpression,
JinjaBlockStart,
JinjaBlockMiddle,

4
sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v1/query.sql
Просмотреть файл

@ -172,9 +172,7 @@ downloads_and_ga_session AS (
mozfun.stats.mode_last_retain_nulls(ARRAY_AGG(landing_page)) AS landing_page,
mozfun.stats.mode_last_retain_nulls(ARRAY_AGG(pageviews)) AS pageviews,
mozfun.stats.mode_last_retain_nulls(ARRAY_AGG(unique_pageviews)) AS unique_pageviews,
LOGICAL_OR(
has_ga_download_event
) AS has_ga_download_event, -- this will be ignored if nrows >1
LOGICAL_OR(has_ga_download_event) AS has_ga_download_event, -- this will be ignored if nrows >1
mozfun.stats.mode_last_retain_nulls(
ARRAY_AGG(count_dltoken_duplicates)
) AS count_dltoken_duplicates,

3
sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v2/checks.sql
Просмотреть файл

@ -1,5 +1,4 @@
#fail
ASSERT(
(
SELECT
@ -11,4 +10,4 @@ ASSERT(
) > 50000
)
AS
'ETL Data Check Failed: Table {{project_id}}.{{dataset_id}}.{{table_name}} contains less than 50,000 rows for date: {{ download_date }}.'
'ETL Data Check Failed: Table {{project_id}}.{{dataset_id}}.{{table_name}} contains less than 50,000 rows for date: {{ download_date }}.'

45
sql/moz-fx-data-shared-prod/adjust/adjust_cohort/view.sql
Просмотреть файл

@ -2,26 +2,25 @@ CREATE OR REPLACE VIEW
`moz-fx-data-shared-prod.adjust.adjust_cohort`
AS
SELECT
cohort_start_month,
period_length,
period,
app,
network,
network_token,
country,
os,
cohort_size,
retained_users,
retention_rate,
time_spent_per_user,
time_spent_per_session,
time_spent,
sessions_per_user,
sessions,
date
FROM `moz-fx-data-shared-prod.adjust_derived.adjust_cohort_v1`
WHERE (network != "Untrusted Devices")
AND
date in (
SELECT MAX(date) from `moz-fx-data-shared-prod.adjust_derived.adjust_cohort_v1`
)
cohort_start_month,
period_length,
period,
app,
network,
network_token,
country,
os,
cohort_size,
retained_users,
retention_rate,
time_spent_per_user,
time_spent_per_session,
time_spent,
sessions_per_user,
sessions,
date
FROM
`moz-fx-data-shared-prod.adjust_derived.adjust_cohort_v1`
WHERE
(network != "Untrusted Devices")
AND date IN (SELECT MAX(date) FROM `moz-fx-data-shared-prod.adjust_derived.adjust_cohort_v1`)

47
sql/moz-fx-data-shared-prod/adjust/adjust_kpi_deliverables/view.sql
Просмотреть файл

@ -2,26 +2,27 @@ CREATE OR REPLACE VIEW
`moz-fx-data-shared-prod.adjust.adjust_kpi_deliverables`
AS
SELECT
date,
app,
network,
network_token,
campaign,
campaign_token,
adgroup,
adgroup_token,
creative,
creative_token,
country,
os,
device,
clicks,
installs,
limit_ad_tracking_install_rate,
click_conversion_rate,
impression_conversion_rate,
sessions,
daus,
waus,
maus
FROM `moz-fx-data-shared-prod.adjust_derived.adjust_deliverables_v1`
date,
app,
network,
network_token,
campaign,
campaign_token,
adgroup,
adgroup_token,
creative,
creative_token,
country,
os,
device,
clicks,
installs,
limit_ad_tracking_install_rate,
click_conversion_rate,
impression_conversion_rate,
sessions,
daus,
waus,
maus
FROM
`moz-fx-data-shared-prod.adjust_derived.adjust_deliverables_v1`

40
sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1/query.sql
Просмотреть файл

@ -11,22 +11,19 @@ WITH client_shares AS (
COUNT(*) AS global_total,
COUNT(
CASE
-- SUGGEST DESKTOP ELIGIBILITY REQUIREMENTS
WHEN
normalized_app_name = "Firefox Desktop"
AND
-- desktop Suggest experiment start -- 12.5% exposure until 2022-09-21
(
submission_date >= "2022-06-07"
AND browser_version_info.major_version >= 92
AND browser_version_info.version NOT IN ('92', '92.', '92.0', '92.0.0')
AND country IN UNNEST(["US"])
AND locale LIKE "en%"
)
THEN
1
ELSE
NULL
-- SUGGEST DESKTOP ELIGIBILITY REQUIREMENTS
WHEN normalized_app_name = "Firefox Desktop"
AND
-- desktop Suggest experiment start -- 12.5% exposure until 2022-09-21
(
submission_date >= "2022-06-07"
AND browser_version_info.major_version >= 92
AND browser_version_info.version NOT IN ('92', '92.', '92.0', '92.0.0')
AND country IN UNNEST(["US"])
AND locale LIKE "en%"
)
THEN 1
ELSE NULL
END
) AS eligible_clients
FROM
@ -116,13 +113,10 @@ desktop_population AS (
"desktop" AS device,
COUNT(
CASE
WHEN
impression_sponsored_count > 0
OR impression_nonsponsored_count > 0
THEN
client_id
ELSE
NULL
WHEN impression_sponsored_count > 0
OR impression_nonsponsored_count > 0
THEN client_id
ELSE NULL
END
) AS suggest_exposed_clients,
SUM(impression_sponsored_count + impression_nonsponsored_count) AS total_impressions,

271
sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_events_v1/query.sql
Просмотреть файл

@ -9,88 +9,213 @@ WITH _events_ping_distinct_client_count AS (
GROUP BY
submission_date
),
client_product_feature_usage AS (
SELECT
client_info.client_id,
DATE(submission_timestamp) AS submission_date,
/*Logins*/
--autofill
COUNTIF(event_category = 'logins' AND event_name = 'password_detected') AS autofill_password_detected_logins,
COUNTIF(event_category = 'logins' AND event_name = 'autofill_prompt_shown') AS autofill_prompt_shown_logins,
COUNTIF(event_category = 'logins' AND event_name = 'autofill_prompt_dismissed') AS autofill_prompt_dismissed_logins,
COUNTIF(
event_category = 'logins'
AND event_name = 'password_detected'
) AS autofill_password_detected_logins,
COUNTIF(
event_category = 'logins'
AND event_name = 'autofill_prompt_shown'
) AS autofill_prompt_shown_logins,
COUNTIF(
event_category = 'logins'
AND event_name = 'autofill_prompt_dismissed'
) AS autofill_prompt_dismissed_logins,
COUNTIF(event_category = 'logins' AND event_name = 'autofilled') AS autofilled_logins,
--management
COUNTIF(event_category = 'logins' AND event_name = 'management_add_tapped') AS management_add_tapped_logins,
COUNTIF(event_category = 'logins' AND event_name = 'management_logins_tapped') AS management_tapped_logins,
COUNTIF(
event_category = 'logins'
AND event_name = 'management_add_tapped'
) AS management_add_tapped_logins,
COUNTIF(
event_category = 'logins'
AND event_name = 'management_logins_tapped'
) AS management_tapped_logins,
/*Credit Card*/
--autofill
COUNTIF(event_category = 'credit_cards' AND event_name = 'form_detected') AS form_detected_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'autofill_prompt_shown') AS autofill_prompt_shown_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'autofill_prompt_expanded') AS autofill_prompt_expanded_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'autofill_prompt_dismissed') AS autofill_prompt_dismissed_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'autofill_prompt_shown'
) AS autofill_prompt_shown_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'autofill_prompt_expanded'
) AS autofill_prompt_expanded_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'autofill_prompt_dismissed'
) AS autofill_prompt_dismissed_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'autofilled') AS autofilled_cc,
--save prompt
COUNTIF(event_category = 'credit_cards' AND event_name = 'save_prompt_shown') AS save_prompt_shown_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'save_prompt_create') AS save_prompt_create_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'save_prompt_update') AS save_prompt_update_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'save_prompt_shown'
) AS save_prompt_shown_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'save_prompt_create'
) AS save_prompt_create_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'save_prompt_update'
) AS save_prompt_update_cc,
--management
COUNTIF(event_category = 'credit_cards' AND event_name = 'management_add_tapped') AS management_add_tapped_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'management_card_tapped') AS management_tapped_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'management_add_tapped'
) AS management_add_tapped_cc,
COUNTIF(
event_category = 'credit_cards'
AND event_name = 'management_card_tapped'
) AS management_tapped_cc,
COUNTIF(event_category = 'credit_cards' AND event_name = 'modified') AS modified_cc,
/*Addresses*/
--autofill
COUNTIF(event_category = 'addresses' AND event_name = 'form_detected') AS form_detected_address,
COUNTIF(event_category = 'addresses' AND event_name = 'autofill_prompt_shown') AS autofill_prompt_shown_address,
COUNTIF(event_category = 'addresses' AND event_name = 'autofill_prompt_expanded') AS autofill_prompt_expanded_address,
COUNTIF(event_category = 'addresses' AND event_name = 'autofill_prompt_dismissed') AS autofill_prompt_dismissed_address,
COUNTIF(
event_category = 'addresses'
AND event_name = 'autofill_prompt_shown'
) AS autofill_prompt_shown_address,
COUNTIF(
event_category = 'addresses'
AND event_name = 'autofill_prompt_expanded'
) AS autofill_prompt_expanded_address,
COUNTIF(
event_category = 'addresses'
AND event_name = 'autofill_prompt_dismissed'
) AS autofill_prompt_dismissed_address,
COUNTIF(event_category = 'addresses' AND event_name = 'autofilled') AS autofilled_address,
--management
COUNTIF(event_category = 'addresses' AND event_name = 'management_add_tapped') AS management_add_tapped_address,
COUNTIF(event_category = 'addresses' AND event_name = 'management_address_tapped') AS management_tapped_address,
COUNTIF(
event_category = 'addresses'
AND event_name = 'management_add_tapped'
) AS management_add_tapped_address,
COUNTIF(
event_category = 'addresses'
AND event_name = 'management_address_tapped'
) AS management_tapped_address,
/*Bookmark*/
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'copied') AS bookmark_copied,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'edited') AS bookmark_edited,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'folder_add') AS bookmark_folder_add,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'folder_add'
) AS bookmark_folder_add,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'open') AS bookmark_open,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'open_all_in_new_tabs') AS bookmark_open_all_in_new_tabs,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'open_all_in_private_tabs') AS bookmark_open_all_in_private_tabs,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'open_in_new_tab') AS bookmark_open_in_new_tab,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'open_in_new_tabs') AS bookmark_open_in_new_tabs,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'open_in_private_tab') AS bookmark_open_in_private_tab,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'open_in_private_tabs') AS bookmark_open_in_private_tabs,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'open_all_in_new_tabs'
) AS bookmark_open_all_in_new_tabs,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'open_all_in_private_tabs'
) AS bookmark_open_all_in_private_tabs,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'open_in_new_tab'
) AS bookmark_open_in_new_tab,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'open_in_new_tabs'
) AS bookmark_open_in_new_tabs,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'open_in_private_tab'
) AS bookmark_open_in_private_tab,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'open_in_private_tabs'
) AS bookmark_open_in_private_tabs,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'removed') AS bookmark_removed,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'search_icon_tapped') AS bookmark_search_icon_tapped,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'search_result_tapped') AS bookmark_search_result_tapped,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'search_icon_tapped'
) AS bookmark_search_icon_tapped,
COUNTIF(
event_category = 'bookmarks_management'
AND event_name = 'search_result_tapped'
) AS bookmark_search_result_tapped,
COUNTIF(event_category = 'bookmarks_management' AND event_name = 'shared') AS bookmark_shared,
/*History*/
COUNTIF(event_category = 'history' AND event_name = 'opened') AS history_opened,
COUNTIF(event_category = 'history' AND event_name = 'opened_item') AS history_opened_item,
COUNTIF(event_category = 'history' AND event_name = 'opened_items_in_new_tabs') AS history_opened_items_in_new_tabs,
COUNTIF(event_category = 'history' AND event_name = 'opened_items_in_private_tabs') AS history_opened_items_in_private_tabs,
COUNTIF(event_category = 'history' AND event_name = 'recent_searches_tapped') AS history_recent_searches_tapped,
COUNTIF(event_category = 'history' AND event_name = 'remove_prompt_cancelled') AS history_remove_prompt_cancelled,
COUNTIF(event_category = 'history' AND event_name = 'remove_prompt_opened') AS history_remove_prompt_opened,
COUNTIF(
event_category = 'history'
AND event_name = 'opened_items_in_new_tabs'
) AS history_opened_items_in_new_tabs,
COUNTIF(
event_category = 'history'
AND event_name = 'opened_items_in_private_tabs'
) AS history_opened_items_in_private_tabs,
COUNTIF(
event_category = 'history'
AND event_name = 'recent_searches_tapped'
) AS history_recent_searches_tapped,
COUNTIF(
event_category = 'history'
AND event_name = 'remove_prompt_cancelled'
) AS history_remove_prompt_cancelled,
COUNTIF(
event_category = 'history'
AND event_name = 'remove_prompt_opened'
) AS history_remove_prompt_opened,
COUNTIF(event_category = 'history' AND event_name = 'removed') AS history_removed,
COUNTIF(event_category = 'history' AND event_name = 'removed_all') AS history_removed_all,
COUNTIF(event_category = 'history' AND event_name = 'removed_last_hour') AS history_removed_last_hour,
COUNTIF(event_category = 'history' AND event_name = 'removed_today_and_yesterday') AS history_removed_today_and_yesterday,
COUNTIF(event_category = 'history' AND event_name = 'search_icon_tapped') AS history_search_icon_tapped,
COUNTIF(event_category = 'history' AND event_name = 'search_result_tapped') AS history_search_result_tapped,
COUNTIF(event_category = 'history' AND event_name = 'search_term_group_open_tab') AS history_search_term_group_open_tab,
COUNTIF(event_category = 'history' AND event_name = 'search_term_group_remove_all') AS history_search_term_group_remove_all,
COUNTIF(event_category = 'history' AND event_name = 'search_term_group_remove_tab') AS history_search_term_group_remove_tab,
COUNTIF(event_category = 'history' AND event_name = 'search_term_group_tapped') AS history_search_term_group_tapped,
COUNTIF(
event_category = 'history'
AND event_name = 'removed_last_hour'
) AS history_removed_last_hour,
COUNTIF(
event_category = 'history'
AND event_name = 'removed_today_and_yesterday'
) AS history_removed_today_and_yesterday,
COUNTIF(
event_category = 'history'
AND event_name = 'search_icon_tapped'
) AS history_search_icon_tapped,
COUNTIF(
event_category = 'history'
AND event_name = 'search_result_tapped'
) AS history_search_result_tapped,
COUNTIF(
event_category = 'history'
AND event_name = 'search_term_group_open_tab'
) AS history_search_term_group_open_tab,
COUNTIF(
event_category = 'history'
AND event_name = 'search_term_group_remove_all'
) AS history_search_term_group_remove_all,
COUNTIF(
event_category = 'history'
AND event_name = 'search_term_group_remove_tab'
) AS history_search_term_group_remove_tab,
COUNTIF(
event_category = 'history'
AND event_name = 'search_term_group_tapped'
) AS history_search_term_group_tapped,
COUNTIF(event_category = 'history' AND event_name = 'shared') AS history_shared,
/*FxA*/
COUNTIF(event_category = 'sync' AND event_name = 'failed') AS sync_failed,
COUNTIF(event_category = 'sync_account' AND event_name = 'opened') AS sync_account_opened,
COUNTIF(event_category = 'sync_account' AND event_name = 'send_tab') AS sync_account_send_tab,
COUNTIF(event_category = 'sync_account' AND event_name = 'sign_in_to_send_tab') AS sync_account_sign_in_to_send_tab,
COUNTIF(
event_category = 'sync_account'
AND event_name = 'sign_in_to_send_tab'
) AS sync_account_sign_in_to_send_tab,
COUNTIF(event_category = 'sync_account' AND event_name = 'sync_now') AS sync_account_sync_now,
COUNTIF(event_category = 'sync_auth' AND event_name = 'closed') AS sync_auth_closed,
COUNTIF(event_category = 'sync_auth' AND event_name = 'opened') AS sync_auth_opened,
COUNTIF(event_category = 'sync_auth' AND event_name = 'other_external') AS sync_auth_other_external,
COUNTIF(
event_category = 'sync_auth'
AND event_name = 'other_external'
) AS sync_auth_other_external,
COUNTIF(event_category = 'sync_auth' AND event_name = 'paired') AS sync_auth_paired,
COUNTIF(event_category = 'sync_auth' AND event_name = 'recovered') AS sync_auth_recovered,
COUNTIF(event_category = 'sync_auth' AND event_name = 'scan_pairing') AS sync_auth_scan_pairing,
@ -98,25 +223,61 @@ client_product_feature_usage AS (
COUNTIF(event_category = 'sync_auth' AND event_name = 'sign_out') AS sync_auth_sign_out,
COUNTIF(event_category = 'sync_auth' AND event_name = 'sign_up') AS sync_auth_sign_up,
COUNTIF(event_category = 'sync_auth' AND event_name = 'use_email') AS sync_auth_use_email,
COUNTIF(event_category = 'sync_auth' AND event_name = 'use_email_problem') AS sync_auth_use_email_problem,
COUNTIF(
event_category = 'sync_auth'
AND event_name = 'use_email_problem'
) AS sync_auth_use_email_problem,
/*Privacy*/
COUNTIF(event_category = 'homepage' AND event_name = 'private_mode_icon_tapped') AS hp_private_mode_tapped,
COUNTIF(event_category = 'tabs_tray' AND event_name = 'private_mode_tapped') AS tab_tray_private_mode_switched,
COUNTIF(event_category = 'app_icon' AND event_name = 'new_private_tab_tapped') AS app_icon_private_tab_tapped,
COUNTIF(event_category = 'tabs_tray' AND event_name = 'new_private_tab_tapped') AS tab_tray_private_mode_tapped,
COUNTIF(
event_category = 'homepage'
AND event_name = 'private_mode_icon_tapped'
) AS hp_private_mode_tapped,
COUNTIF(
event_category = 'tabs_tray'
AND event_name = 'private_mode_tapped'
) AS tab_tray_private_mode_switched,
COUNTIF(
event_category = 'app_icon'
AND event_name = 'new_private_tab_tapped'
) AS app_icon_private_tab_tapped,
COUNTIF(
event_category = 'tabs_tray'
AND event_name = 'new_private_tab_tapped'
) AS tab_tray_private_mode_tapped,
--etp
COUNTIF(event_category = 'tracking_protection' AND event_name = 'etp_setting_changed') AS etp_setting_changed,
COUNTIF(
event_category = 'tracking_protection'
AND event_name = 'etp_setting_changed'
) AS etp_setting_changed,
COUNTIF(event_category = 'tracking_protection' AND event_name = 'etp_settings') AS etp_settings,
COUNTIF(event_category = 'tracking_protection' AND event_name = 'etp_shield') AS etp_shield,
COUNTIF(event_category = 'tracking_protection' AND event_name = 'etp_tracker_list') AS etp_tracker_list,
COUNTIF(
event_category = 'tracking_protection'
AND event_name = 'etp_tracker_list'
) AS etp_tracker_list,
/*Default browser*/
COUNTIF(event_category = 'events' AND event_name = 'default_browser_changed') AS default_browser_changed,
COUNTIF(
event_category = 'events'
AND event_name = 'default_browser_changed'
) AS default_browser_changed,
/*Notification*/
COUNTIF(event_category = 'events' AND event_name = 're_engagement_notif_shown') AS re_engagement_notif_shown,
COUNTIF(event_category = 'events' AND event_name = 're_engagement_notif_tapped') AS re_engagement_notif_tapped,
COUNTIF(
event_category = 'events'
AND event_name = 're_engagement_notif_shown'
) AS re_engagement_notif_shown,
COUNTIF(
event_category = 'events'
AND event_name = 're_engagement_notif_tapped'
) AS re_engagement_notif_tapped,
/*Customize Home*/
COUNTIF(event_category = 'app_menu' AND event_name = 'customize_homepage') AS app_menu_customize_homepage,
COUNTIF(event_category = 'home_screen' AND event_name = 'customize_home_clicked') AS home_page_customize_home_clicked
COUNTIF(
event_category = 'app_menu'
AND event_name = 'customize_homepage'
) AS app_menu_customize_homepage,
COUNTIF(
event_category = 'home_screen'
AND event_name = 'customize_home_clicked'
) AS home_page_customize_home_clicked
FROM
fenix.events_unnested
WHERE

215
sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v1/query.sql
Просмотреть файл

@ -9,7 +9,6 @@ WITH _metrics_ping_distinct_client_count AS (
GROUP BY
submission_date
),
client_product_feature_usage AS (
SELECT
client_info.client_id,
@ -26,22 +25,10 @@ client_product_feature_usage AS (
COALESCE(SUM(metrics.counter.addresses_updated), 0) AS addresses_modified,
COALESCE(SUM(metrics.quantity.addresses_saved_all), 0) AS currently_stored_addresses,
--Bookmark
COALESCE(
SUM(metrics_bookmarks_add_table.value),
0
) AS bookmarks_add,
COALESCE(
SUM(metrics_bookmarks_delete_table.value),
0
) AS bookmarks_delete,
COALESCE(
SUM(metrics_bookmarks_edit_table.value),
0
) AS bookmarks_edit,
COALESCE(
SUM(metrics_bookmarks_open_table.value),
0
) AS bookmarks_open,
COALESCE(SUM(metrics_bookmarks_add_table.value), 0) AS bookmarks_add,
COALESCE(SUM(metrics_bookmarks_delete_table.value), 0) AS bookmarks_delete,
COALESCE(SUM(metrics_bookmarks_edit_table.value), 0) AS bookmarks_edit,
COALESCE(SUM(metrics_bookmarks_open_table.value), 0) AS bookmarks_open,
COALESCE(
SUM(metrics.counter.metrics_desktop_bookmarks_count),
0
@ -50,41 +37,53 @@ client_product_feature_usage AS (
SUM(metrics.counter.metrics_mobile_bookmarks_count),
0
) AS metrics_mobile_bookmarks_count,
SUM(CASE WHEN metrics.boolean.metrics_has_desktop_bookmarks IS TRUE THEN 1 ELSE 0 END) AS metrics_has_desktop_bookmarks,
SUM(CASE WHEN metrics.boolean.metrics_has_mobile_bookmarks IS TRUE THEN 1 ELSE 0 END) AS metrics_has_mobile_bookmarks,
SUM(
CASE
WHEN metrics.boolean.metrics_has_desktop_bookmarks IS TRUE
THEN 1
ELSE 0
END
) AS metrics_has_desktop_bookmarks,
SUM(
CASE
WHEN metrics.boolean.metrics_has_mobile_bookmarks IS TRUE
THEN 1
ELSE 0
END
) AS metrics_has_mobile_bookmarks,
--Privacy
SUM(
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection = 'standard'
THEN 1
ELSE 0
END
) AS etp_standard,
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection = 'standard'
THEN 1
ELSE 0
END
) AS etp_standard,
SUM(
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection = 'strict'
THEN 1
ELSE 0
END
) AS etp_strict,
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection = 'strict'
THEN 1
ELSE 0
END
) AS etp_strict,
SUM(
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection = 'custom'
THEN 1
ELSE 0
END
) AS etp_custom,
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection = 'custom'
THEN 1
ELSE 0
END
) AS etp_custom,
SUM(
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection NOT IN (
'custom',
'standard',
'strict'
)
THEN 1
ELSE 0
END
) AS etp_disabled,
CASE
WHEN metrics.string.preferences_enhanced_tracking_protection NOT IN (
'custom',
'standard',
'strict'
)
THEN 1
ELSE 0
END
) AS etp_disabled,
--Tab count
COALESCE(
SUM(metrics.counter.metrics_private_tabs_open_count),
@ -92,49 +91,101 @@ client_product_feature_usage AS (
) AS metrics_private_tabs_open_count,
COALESCE(SUM(metrics.counter.metrics_tabs_open_count), 0) AS metrics_tabs_open_count,
--Default browser
SUM(CASE WHEN metrics.boolean.metrics_default_browser THEN 1 ELSE 0 END) AS metrics_default_browser,
SUM(
CASE
WHEN metrics.boolean.metrics_default_browser
THEN 1
ELSE 0
END
) AS metrics_default_browser,
--Awesomebar Location
SUM(
CASE
WHEN metrics.string.preferences_toolbar_position_setting IN ('top', 'fixed_top')
THEN 1
ELSE 0
END
) AS awesomebar_top,
CASE
WHEN metrics.string.preferences_toolbar_position_setting IN ('top', 'fixed_top')
THEN 1
ELSE 0
END
) AS awesomebar_top,
SUM(
CASE
WHEN metrics.string.preferences_toolbar_position_setting = 'bottom'
THEN 1
ELSE 0
END
) AS awesomebar_bottom,
CASE
WHEN metrics.string.preferences_toolbar_position_setting = 'bottom'
THEN 1
ELSE 0
END
) AS awesomebar_bottom,
SUM(
CASE
WHEN metrics.string.preferences_toolbar_position_setting NOT IN (
'top',
'fixed_top',
'bottom'
)
THEN 1
ELSE 0
END
) AS awesomebar_null,
CASE
WHEN metrics.string.preferences_toolbar_position_setting NOT IN (
'top',
'fixed_top',
'bottom'
)
THEN 1
ELSE 0
END
) AS awesomebar_null,
--Notification
SUM(CASE WHEN metrics.boolean.metrics_notifications_allowed THEN 1 ELSE 0 END) AS metrics_notifications_allowed,
SUM(CASE WHEN metrics.boolean.events_marketing_notification_allowed THEN 1 ELSE 0 END) AS events_marketing_notification_allowed,
SUM(
CASE
WHEN metrics.boolean.metrics_notifications_allowed
THEN 1
ELSE 0
END
) AS metrics_notifications_allowed,
SUM(
CASE
WHEN metrics.boolean.events_marketing_notification_allowed
THEN 1
ELSE 0
END
) AS events_marketing_notification_allowed,
--Customize Homepage
SUM(CASE WHEN metrics.boolean.customize_home_contile THEN 1 ELSE 0 END) AS customize_home_contile,
SUM(CASE WHEN metrics.boolean.customize_home_jump_back_in THEN 1 ELSE 0 END) AS customize_home_jump_back_in,
SUM(CASE WHEN metrics.boolean.customize_home_most_visited_sites THEN 1 ELSE 0 END) AS customize_home_most_visited_sites,
SUM(
CASE
WHEN metrics.boolean.customize_home_contile
THEN 1
ELSE 0
END
) AS customize_home_contile,
SUM(
CASE
WHEN metrics.boolean.customize_home_jump_back_in
THEN 1
ELSE 0
END
) AS customize_home_jump_back_in,
SUM(
CASE
WHEN metrics.boolean.customize_home_most_visited_sites
THEN 1
ELSE 0
END
) AS customize_home_most_visited_sites,
SUM(CASE WHEN metrics.boolean.customize_home_pocket THEN 1 ELSE 0 END) AS customize_home_pocket,
SUM(CASE WHEN metrics.boolean.customize_home_recently_saved THEN 1 ELSE 0 END) AS customize_home_recently_saved,
SUM(CASE WHEN metrics.boolean.customize_home_recently_visited THEN 1 ELSE 0 END) AS customize_home_recently_visited
SUM(
CASE
WHEN metrics.boolean.customize_home_recently_saved
THEN 1
ELSE 0
END
) AS customize_home_recently_saved,
SUM(
CASE
WHEN metrics.boolean.customize_home_recently_visited
THEN 1
ELSE 0
END
) AS customize_home_recently_visited
FROM
fenix.metrics AS metric
LEFT JOIN UNNEST(metrics.labeled_counter.metrics_bookmarks_add) AS metrics_bookmarks_add_table
LEFT JOIN UNNEST(metrics.labeled_counter.metrics_bookmarks_delete) AS metrics_bookmarks_delete_table
LEFT JOIN UNNEST(metrics.labeled_counter.metrics_bookmarks_edit) AS metrics_bookmarks_edit_table
LEFT JOIN UNNEST(metrics.labeled_counter.metrics_bookmarks_open) AS metrics_bookmarks_open_table
LEFT JOIN
UNNEST(metrics.labeled_counter.metrics_bookmarks_add) AS metrics_bookmarks_add_table
LEFT JOIN
UNNEST(metrics.labeled_counter.metrics_bookmarks_delete) AS metrics_bookmarks_delete_table
LEFT JOIN
UNNEST(metrics.labeled_counter.metrics_bookmarks_edit) AS metrics_bookmarks_edit_table
LEFT JOIN
UNNEST(metrics.labeled_counter.metrics_bookmarks_open) AS metrics_bookmarks_open_table
WHERE
DATE(submission_timestamp) = @submission_date
GROUP BY
@ -198,7 +249,7 @@ product_features_agg AS (
/*Bookmark*/
--bookmarks_add
COUNT(DISTINCT CASE WHEN bookmarks_add > 0 THEN client_id END) AS bookmarks_add_users,
SUM(bookmarks_add)AS bookmarks_add,
SUM(bookmarks_add) AS bookmarks_add,
--bookmarks_delete
COUNT(DISTINCT CASE WHEN bookmarks_delete > 0 THEN client_id END) AS bookmarks_delete_users,
SUM(bookmarks_delete) AS bookmarks_delete,

14
sql/moz-fx-data-shared-prod/firefox_accounts/docker_fxa_admin_server_sanitized/view.sql
Просмотреть файл

@ -2,7 +2,7 @@ CREATE OR REPLACE VIEW
`moz-fx-data-shared-prod.firefox_accounts.docker_fxa_admin_server_sanitized`
AS
SELECT
* REPLACE(
* REPLACE (
STRUCT(
resource.type,
STRUCT(
@ -33,10 +33,7 @@ SELECT
) AS labels,
STRUCT(
jsonPayload.timestamp,
STRUCT(
jsonPayload.fields.event,
jsonPayload.fields.search_type
) AS fields
STRUCT(jsonPayload.fields.event, jsonPayload.fields.search_type) AS fields
) AS jsonPayload
),
CAST(NULL AS ARRAY<STRUCT<id STRING>>) AS errorGroups
@ -44,7 +41,7 @@ FROM
`moz-fx-data-shared-prod.firefox_accounts_derived.docker_fxa_admin_server_sanitized_v1`
UNION ALL
SELECT
* REPLACE(
* REPLACE (
STRUCT(
resource.type,
STRUCT(
@ -75,10 +72,7 @@ SELECT
) AS labels,
STRUCT(
jsonPayload.timestamp,
STRUCT(
jsonPayload.fields.event,
jsonPayload.fields.search_type
) AS fields
STRUCT(jsonPayload.fields.event, jsonPayload.fields.search_type) AS fields
) AS jsonPayload
)
FROM

4
sql/moz-fx-data-shared-prod/firefox_accounts/fxa_all_events/view.sql
Просмотреть файл

@ -39,9 +39,7 @@ auth_bounce_events AS (
receiveTimestamp,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(jsonPayload.fields.time AS INT64)) AS event_time,
jsonPayload.fields.user_id,
CAST(
NULL AS STRING
) AS country, -- No country field in auth_bounces
CAST(NULL AS STRING) AS country, -- No country field in auth_bounces
CAST(NULL AS STRING) AS country_code,
jsonPayload.fields.language,
jsonPayload.fields.app_version,

6
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/init.sql
Просмотреть файл

@ -12,14 +12,14 @@ SELECT
0 AS days_seen_bits,
0 AS days_seen_in_tier1_country_bits,
0 AS days_registered_bits,
0 AS days_seen_no_monitor_bits,
-- We make sure to delay * until the end so that as new columns are added
-- to fxa_users_daily, we can add those columns in the same order to the end
-- of this schema, which may be necessary for the daily join query between
-- the two tables to validate.
* EXCEPT (submission_date, seen_in_tier1_country, registered)
* EXCEPT (submission_date, seen_in_tier1_country, registered),
0 AS days_seen_no_monitor_bits,
FROM
fxa_users_daily_v1
`moz-fx-data-shared-prod.firefox_accounts_derived.fxa_users_daily_v1`
WHERE
-- Output empty table and read no input rows
FALSE

6
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/query.sql
Просмотреть файл

@ -39,11 +39,7 @@ _previous AS (
)
SELECT
@submission_date AS submission_date,
IF(
_current.user_id IS NOT NULL,
_current,
_previous
).* REPLACE ( --
IF(_current.user_id IS NOT NULL, _current, _previous).* REPLACE ( --
udf.combine_adjacent_days_28_bits(
_previous.days_seen_bits,
_current.days_seen_bits

20
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/schema.yaml
Просмотреть файл

@ -1,12 +1,5 @@
fields:
- mode: NULLABLE
name: user_id
type: STRING
description: |
A 36 char long hash value representing
User ID (registered user).
- mode: NULLABLE
name: submission_date
type: DATE
@ -33,10 +26,11 @@ fields:
No. of days since registration event.
- mode: NULLABLE
name: days_seen_no_monitor_bits
type: INTEGER
name: user_id
type: STRING
description: |
No. of days since monitor flag was last True.
A 36 char long hash value representing
User ID (registered user).
- mode: NULLABLE
name: country
@ -71,3 +65,9 @@ fields:
type: STRING
description: |
Version of the OS the device was most recently using.
- mode: NULLABLE
name: days_seen_no_monitor_bits
type: INTEGER
description: |
No. of days since monitor flag was last True.

6
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v2/query.sql
Просмотреть файл

@ -31,11 +31,7 @@ _previous AS (
)
SELECT
@submission_date AS submission_date,
IF(
_current.user_id IS NOT NULL,
_current,
_previous
).* REPLACE ( --
IF(_current.user_id IS NOT NULL, _current, _previous).* REPLACE ( --
udf.combine_adjacent_days_28_bits(
_previous.days_seen_bits,
_current.days_seen_bits

8
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v1/query.sql
Просмотреть файл

@ -1,6 +1,4 @@
CREATE TEMP FUNCTION udf_contains_tier1_country(
x ANY TYPE
) AS ( --
CREATE TEMP FUNCTION udf_contains_tier1_country(x ANY TYPE) AS ( --
EXISTS(
SELECT
country
@ -19,9 +17,7 @@ CREATE TEMP FUNCTION udf_contains_tier1_country(
--
-- This UDF is also only applicable in the context of this query.
CREATE TEMP FUNCTION udf_contains_registration(
x ANY TYPE
) AS ( --
CREATE TEMP FUNCTION udf_contains_registration(x ANY TYPE) AS ( --
EXISTS(
SELECT
event_type

31
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,31 @@
fields:
- name: submission_date
type: DATE
mode: NULLABLE
- name: user_id
type: STRING
mode: NULLABLE
- name: service
type: STRING
mode: NULLABLE
- name: country
type: STRING
mode: NULLABLE
- name: language
type: STRING
mode: NULLABLE
- name: app_version
type: STRING
mode: NULLABLE
- name: os_name
type: STRING
mode: NULLABLE
- name: os_version
type: STRING
mode: NULLABLE
- name: seen_in_tier1_country
type: BOOLEAN
mode: NULLABLE
- name: registered
type: BOOLEAN
mode: NULLABLE

4
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v2/query.sql
Просмотреть файл

@ -1,6 +1,4 @@
CREATE TEMP FUNCTION udf_contains_tier1_country(
x ANY TYPE
) AS ( --
CREATE TEMP FUNCTION udf_contains_tier1_country(x ANY TYPE) AS ( --
EXISTS(
SELECT
country

6
sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/query.sql
Просмотреть файл

@ -31,11 +31,7 @@ _previous AS (
combined AS (
SELECT
@submission_date AS submission_date,
IF(
_current.user_id IS NOT NULL,
_current,
_previous
).* REPLACE ( --
IF(_current.user_id IS NOT NULL, _current, _previous).* REPLACE ( --
udf.combine_adjacent_days_28_bits(
_previous.days_seen_bits,
_current.days_seen_bits

2
sql/moz-fx-data-shared-prod/firefox_desktop/newtab_live/view.sql
Просмотреть файл

@ -14,4 +14,4 @@ SELECT
client_info.client_id,
events
FROM
`moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1`
`moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1`

12
sql/moz-fx-data-shared-prod/firefox_desktop_derived/desktop_installs_v1/query.sql
Просмотреть файл

@ -133,8 +133,16 @@ SELECT
install_ping.new_launched,
install_ping.sample_id,
install_ping.install_attempts,
CASE WHEN succeeded is true then install_ping.install_attempts else 0 end AS installs,
CASE WHEN succeeded is false then install_ping.install_attempts else 0 end as unsuccessful_installs,
CASE
WHEN succeeded IS TRUE
THEN install_ping.install_attempts
ELSE 0
END AS installs,
CASE
WHEN succeeded IS FALSE
THEN install_ping.install_attempts
ELSE 0
END AS unsuccessful_installs,
download_token_info.download_date AS attribution_dltoken_date
FROM
install_ping

13
sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_yearly/view.sql
Просмотреть файл

@ -1,13 +1,18 @@
{% set usage_types = [("seen", "TRUE"), ("active", "durations > 0")] %}
CREATE OR REPLACE VIEW
`moz-fx-data-shared-prod.firefox_ios.baseline_clients_yearly`
AS
SELECT
{% for usage_type, _ in usage_types %}
`moz-fx-data-shared-prod`.udf.bits_to_days_since_seen(days_{{ usage_type }}_bytes) AS days_since_{{ usage_type }},
`moz-fx-data-shared-prod`.udf.bits_to_days_since_seen(~days_{{ usage_type }}_bytes) AS consecutive_days_{{ usage_type }},
`moz-fx-data-shared-prod`.udf.bits_to_days_seen(days_{{ usage_type }}_bytes) AS days_{{ usage_type }}_in_past_year,
`moz-fx-data-shared-prod`.udf.bits_to_days_since_seen(
`days_{{ usage_type }}_bytes`
) AS `days_since_{{ usage_type }}`,
`moz-fx-data-shared-prod`.udf.bits_to_days_since_seen(
~`days_{{ usage_type }}_bytes`
) AS `consecutive_days_{{ usage_type }}`,
`moz-fx-data-shared-prod`.udf.bits_to_days_seen(
`days_{{ usage_type }}_bytes`
) AS `days_{{ usage_type }}_in_past_year`,
{% endfor %}
DATE_DIFF(submission_date, first_seen_date, DAY) AS days_since_first_seen,
EXTRACT(DAYOFWEEK FROM submission_date) AS day_of_week,

2
sql/moz-fx-data-shared-prod/firefox_ios/firefox_ios_clients/view.sql
Просмотреть файл

@ -23,4 +23,4 @@ FROM
`moz-fx-data-shared-prod.firefox_ios_derived.firefox_ios_clients_v1` AS clients
LEFT JOIN
`moz-fx-data-shared-prod.firefox_ios.clients_activation` AS activation
USING(client_id, first_seen_date)
USING (client_id, first_seen_date)

3
sql/moz-fx-data-shared-prod/firefox_ios/funnel_retention_week_4/view.sql
Просмотреть файл

@ -3,4 +3,5 @@ CREATE OR REPLACE VIEW
AS
SELECT
*
FROM `moz-fx-data-shared-prod.firefox_ios_derived.funnel_retention_week_4_v1`
FROM
`moz-fx-data-shared-prod.firefox_ios_derived.funnel_retention_week_4_v1`

4
sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql
Просмотреть файл

@ -67,6 +67,7 @@ SELECT
)
FROM
base;
#fail
SELECT
IF(
@ -78,4 +79,5 @@ FROM
`moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1`
WHERE
submission_date = @submission_date;
-- TODO: for this query it'd be useful to compare sum variance between each day to improve our confidence the data was complete at the execution time.
-- TODO: for this query it'd be useful to compare sum variance between each day to improve our confidence the data was complete at the execution time.

125
sql/moz-fx-data-shared-prod/firefox_ios_derived/baseline_clients_yearly_v1/query.sql
Просмотреть файл

@ -1,77 +1,72 @@
{% set usage_types = [("seen", "TRUE"), ("active", "durations > 0")] %}
{% if is_init() %}
CREATE TABLE IF NOT EXISTS
`moz-fx-data-shared-prod`.firefox_ios_derived.baseline_clients_yearly_v1
PARTITION BY
submission_date
CLUSTER BY
normalized_channel,
sample_id
OPTIONS
(require_partition_filter = TRUE)
AS
SELECT
{% for usage_type, _ in usage_types %}
CAST(NULL AS BYTES) AS days_{{ usage_type }}_bytes,
{% endfor %}
-- We make sure to delay * until the end so that as new columns are added
-- to the daily table we can add those columns in the same order to the end
-- of this schema, which may be necessary for the daily join query between
-- the two tables to validate.
*
FROM
`moz-fx-data-shared-prod`.firefox_ios.baseline_clients_daily
WHERE
-- Output empty table and read no input rows
FALSE
{% else %}
WITH _current AS (
CREATE TABLE IF NOT EXISTS
`moz-fx-data-shared-prod`.firefox_ios_derived.baseline_clients_yearly_v1
PARTITION BY
submission_date
CLUSTER BY
normalized_channel,
sample_id
OPTIONS
(require_partition_filter = TRUE)
AS
SELECT
-- In this raw table, we capture the history of activity over the past
-- 365 days for each usage criterion as an array of bytes. The
-- rightmost bit represents whether the user was active in the current day.
{% for usage_type, criterion in usage_types %}
udf.bool_to_365_bits({{ criterion }}) AS days_{{ usage_type }}_bytes,
{% for usage_type, _ in usage_types %}
CAST(NULL AS BYTES) AS `days_{{ usage_type }}_bytes`,
{% endfor %}
* EXCEPT (submission_date),
-- We make sure to delay * until the end so that as new columns are added
-- to the daily table we can add those columns in the same order to the end
-- of this schema, which may be necessary for the daily join query between
-- the two tables to validate.
*
FROM
`moz-fx-data-shared-prod`.firefox_ios.baseline_clients_daily
WHERE
submission_date = @submission_date
AND sample_id IS NOT NULL
),
-- Output empty table and read no input rows
FALSE
{% else %}
WITH _current AS (
SELECT
-- In this raw table, we capture the history of activity over the past
-- 365 days for each usage criterion as an array of bytes. The
-- rightmost bit represents whether the user was active in the current day.
{% for usage_type, criterion in usage_types %}
udf.bool_to_365_bits({{ criterion }}) AS `days_{{ usage_type }}_bytes`,
{% endfor %}
* EXCEPT (submission_date),
FROM
`moz-fx-data-shared-prod`.firefox_ios.baseline_clients_daily
WHERE
submission_date = @submission_date
AND sample_id IS NOT NULL
),
--
_previous AS (
SELECT
* EXCEPT (submission_date)
FROM
`moz-fx-data-shared-prod`.firefox_ios_derived.baseline_clients_yearly_v1
WHERE
submission_date = DATE_SUB(@submission_date, INTERVAL 1 DAY)
_previous AS (
SELECT
* EXCEPT (submission_date)
FROM
`moz-fx-data-shared-prod`.firefox_ios_derived.baseline_clients_yearly_v1
WHERE
submission_date = DATE_SUB(@submission_date, INTERVAL 1 DAY)
-- Filter out rows from yesterday that have now fallen outside the 365-day window.
AND BIT_COUNT(udf.shift_365_bits_one_day(days_seen_bytes)) > 0
AND sample_id IS NOT NULL
)
--
SELECT
@submission_date AS submission_date,
IF(_current.client_id IS NOT NULL, _current, _previous).* REPLACE (
{% for usage_type, _ in usage_types %}
udf.combine_adjacent_days_365_bits(
_previous.days_{{ usage_type }}_bytes,
_current.days_{{ usage_type }}_bytes
) AS days_{{ usage_type }}_bytes
{{ "," if not loop.last }}
{% endfor %}
AND BIT_COUNT(udf.shift_365_bits_one_day(days_seen_bytes)) > 0
AND sample_id IS NOT NULL
)
FROM
_current
FULL JOIN
_previous
USING (client_id)
--
SELECT
@submission_date AS submission_date,
IF(_current.client_id IS NOT NULL, _current, _previous).* REPLACE (
{% for usage_type, _ in usage_types %}
udf.combine_adjacent_days_365_bits(
_previous.`days_{{ usage_type }}_bytes`,
_current.`days_{{ usage_type }}_bytes`
) AS `days_{{ usage_type }}_bytes` {{ "," if not loop.last }}
{% endfor %}
)
FROM
_current
FULL JOIN
_previous
USING (client_id)
{% endif %}

82
sql/moz-fx-data-shared-prod/firefox_ios_derived/baseline_clients_yearly_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,82 @@
fields:
- name: submission_date
type: DATE
mode: NULLABLE
- name: days_seen_bytes
type: BYTES
mode: NULLABLE
- name: days_active_bytes
type: BYTES
mode: NULLABLE
- name: normalized_app_id
type: STRING
mode: NULLABLE
- name: client_id
type: STRING
mode: NULLABLE
- name: sample_id
type: INTEGER
mode: NULLABLE
- name: first_run_date
type: DATE
mode: NULLABLE
- name: durations
type: INTEGER
mode: NULLABLE
- name: days_seen_session_start_bits
type: INTEGER
mode: NULLABLE
- name: days_seen_session_end_bits
type: INTEGER
mode: NULLABLE
- name: normalized_channel
type: STRING
mode: NULLABLE
- name: normalized_os
type: STRING
mode: NULLABLE
- name: normalized_os_version
type: STRING
mode: NULLABLE
- name: android_sdk_version
type: STRING
mode: NULLABLE
- name: locale
type: STRING
mode: NULLABLE
- name: city
type: STRING
mode: NULLABLE
- name: country
type: STRING
mode: NULLABLE
- name: app_build
type: STRING
mode: NULLABLE
- name: app_channel
type: STRING
mode: NULLABLE
- name: app_display_version
type: STRING
mode: NULLABLE
- name: architecture
type: STRING
mode: NULLABLE
- name: device_manufacturer
type: STRING
mode: NULLABLE
- name: device_model
type: STRING
mode: NULLABLE
- name: telemetry_sdk_build
type: STRING
mode: NULLABLE
- name: first_seen_date
type: DATE
mode: NULLABLE
- name: is_new_profile
type: BOOLEAN
mode: NULLABLE
- name: isp
type: STRING
mode: NULLABLE

29
sql/moz-fx-data-shared-prod/firefox_ios_derived/clients_activation_v1/checks.sql
Просмотреть файл

@ -4,14 +4,20 @@
{{ min_row_count(1, "`submission_date` = @submission_date") }}
#fail
WITH upstream_clients_count AS (
SELECT COUNT(*)
FROM `{{ project_id }}.firefox_ios.firefox_ios_clients`
WHERE first_seen_date = DATE_SUB(@submission_date, INTERVAL 6 DAY)
SELECT
COUNT(*)
FROM
`{{ project_id }}.firefox_ios.firefox_ios_clients`
WHERE
first_seen_date = DATE_SUB(@submission_date, INTERVAL 6 DAY)
),
activations_clients_count AS (
SELECT COUNT(*)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE submission_date = @submission_date
SELECT
COUNT(*)
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date
)
SELECT
IF(
@ -19,12 +25,17 @@ SELECT
ERROR("Number of client records should match for the same first_seen_date."),
NULL
);
#fail
SELECT
IF(
DATE_DIFF(submission_date, first_seen_date, DAY) <> 6,
ERROR("Day difference between values inside first_seen_date and submission_date fields should be 6."),
ERROR(
"Day difference between values inside first_seen_date and submission_date fields should be 6."
),
NULL
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE `submission_date` = @submission_date;
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
`submission_date` = @submission_date;

160
sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_events_v1/query.sql
Просмотреть файл

@ -16,15 +16,33 @@ client_product_feature_usage AS (
/*Logins*/
COUNTIF(event_category = 'logins' AND event_name = 'autofill_failed') AS logins_autofill_failed,
COUNTIF(event_category = 'logins' AND event_name = 'autofilled') AS logins_autofilled,
COUNTIF(event_category = 'logins' AND event_name = 'management_add_tapped') AS logins_management_add_tapped,
COUNTIF(event_category = 'logins' AND event_name = 'management_logins_tapped') AS logins_management_logins_tapped,
COUNTIF(
event_category = 'logins'
AND event_name = 'management_add_tapped'
) AS logins_management_add_tapped,
COUNTIF(
event_category = 'logins'
AND event_name = 'management_logins_tapped'
) AS logins_management_logins_tapped,
/*Credit Card*/
COUNTIF(event_category = 'credit_card' AND event_name = 'autofill_failed') AS cc_autofill_failed,
COUNTIF(event_category = 'credit_card' AND event_name = 'autofill_settings_tapped') AS cc_autofill_settings_tapped,
COUNTIF(event_category = 'credit_card' AND event_name = 'autofill_toggle') AS cc_autofill_toggle,
COUNTIF(
event_category = 'credit_card'
AND event_name = 'autofill_failed'
) AS cc_autofill_failed,
COUNTIF(
event_category = 'credit_card'
AND event_name = 'autofill_settings_tapped'
) AS cc_autofill_settings_tapped,
COUNTIF(
event_category = 'credit_card'
AND event_name = 'autofill_toggle'
) AS cc_autofill_toggle,
COUNTIF(event_category = 'credit_card' AND event_name = 'autofilled') AS cc_autofilled,
COUNTIF(event_category = 'credit_card' AND event_name = 'form_detected') AS cc_form_detected,
COUNTIF(event_category = 'credit_card' AND event_name = 'save_prompt_create') AS cc_save_prompt_create,
COUNTIF(
event_category = 'credit_card'
AND event_name = 'save_prompt_create'
) AS cc_save_prompt_create,
COUNTIF(event_category = 'credit_card' AND event_name = 'sync_toggle') AS cc_sync_toggle,
/*Histroy*/
COUNTIF(event_category = 'history' AND event_name = 'delete_tap') AS history_delete_tap,
@ -32,35 +50,105 @@ client_product_feature_usage AS (
COUNTIF(event_category = 'history' AND event_name = 'removed') AS history_removed,
COUNTIF(event_category = 'history' AND event_name = 'removed_all') AS history_removed_all,
COUNTIF(event_category = 'history' AND event_name = 'removed_today') AS history_removed_today,
COUNTIF(event_category = 'history' AND event_name = 'removed_today_and_yesterday') AS history_removed_today_and_yesterday,
COUNTIF(
event_category = 'history'
AND event_name = 'removed_today_and_yesterday'
) AS history_removed_today_and_yesterday,
COUNTIF(event_category = 'history' AND event_name = 'search_tap') AS history_search_tap,
/*FxA*/
COUNTIF(event_category = 'sync' AND event_name = 'disconnect') AS fxa_disconnect,
COUNTIF(event_category = 'sync' AND event_name = 'login_completed_view') AS fxa_login_completed_view,
COUNTIF(
event_category = 'sync'
AND event_name = 'login_completed_view'
) AS fxa_login_completed_view,
COUNTIF(event_category = 'sync' AND event_name = 'login_token_view') AS fxa_login_token_view,
COUNTIF(event_category = 'sync' AND event_name = 'login_view') AS fxa_login_view,
COUNTIF(event_category = 'sync' AND event_name = 'paired') AS fxa_paired,
COUNTIF(event_category = 'sync' AND event_name = 'registration_code_view') AS fxa_registration_code_view,
COUNTIF(event_category = 'sync' AND event_name = 'registration_completed_view') AS fxa_registration_completed_view,
COUNTIF(
event_category = 'sync'
AND event_name = 'registration_code_view'
) AS fxa_registration_code_view,
COUNTIF(
event_category = 'sync'
AND event_name = 'registration_completed_view'
) AS fxa_registration_completed_view,
COUNTIF(event_category = 'sync' AND event_name = 'registration_view') AS fxa_registration_view,
COUNTIF(event_category = 'sync' AND event_name = 'use_email') AS fxa_use_email,
/*Privacy*/
COUNTIF(event_category = 'preferences' AND event_name = 'private_browsing_button_tapped' AND extra.key = 'is_private' AND extra.value = 'true') AS private_browsing_button_tapped_enter_private_mode,
COUNTIF(event_category = 'preferences' AND event_name = 'private_browsing_button_tapped') AS private_browsing_button_tapped,
COUNTIF(event_category = 'tabs_tray' AND event_name = 'private_browsing_icon_tapped') AS private_browsing_icon_tapped,
COUNTIF(event_category = 'app_icon' AND event_name = 'new_private_tab_tapped') AS app_icon_new_private_tab_tapped,
COUNTIF(event_category = 'tabs_tray' AND event_name = 'new_private_tab_tapped') AS tabs_tray_new_private_tab_tapped,
COUNTIF(
event_category = 'preferences'
AND event_name = 'private_browsing_button_tapped'
AND extra.key = 'is_private'
AND extra.value = 'true'
) AS private_browsing_button_tapped_enter_private_mode,
COUNTIF(
event_category = 'preferences'
AND event_name = 'private_browsing_button_tapped'
) AS private_browsing_button_tapped,
COUNTIF(
event_category = 'tabs_tray'
AND event_name = 'private_browsing_icon_tapped'
) AS private_browsing_icon_tapped,
COUNTIF(
event_category = 'app_icon'
AND event_name = 'new_private_tab_tapped'
) AS app_icon_new_private_tab_tapped,
COUNTIF(
event_category = 'tabs_tray'
AND event_name = 'new_private_tab_tapped'
) AS tabs_tray_new_private_tab_tapped,
/*Awesomebar Location*/
COUNTIF(event_category = 'awesomebar' AND event_name = 'drag_location_bar') AS drag_location_bar,
COUNTIF(event_category = 'awesomebar' AND event_name = 'location' AND extra.value = 'top') AS location_top,
COUNTIF(event_category = 'awesomebar' AND event_name = 'location' AND extra.value = 'bottom') AS location_bottom,
COUNTIF(
event_category = 'awesomebar'
AND event_name = 'drag_location_bar'
) AS drag_location_bar,
COUNTIF(
event_category = 'awesomebar'
AND event_name = 'location'
AND extra.value = 'top'
) AS location_top,
COUNTIF(
event_category = 'awesomebar'
AND event_name = 'location'
AND extra.value = 'bottom'
) AS location_bottom,
/*Notification*/
COUNTIF(event_category = 'app' AND event_name = 'notification_permission' AND extra.key = 'status' AND extra.value = 'authorized') AS notification_status_authorized,
COUNTIF(event_category = 'app' AND event_name = 'notification_permission' AND extra.key = 'status' AND extra.value = 'notDetermined') AS notification_status_notDetermined,
COUNTIF(event_category = 'app' AND event_name = 'notification_permission' AND extra.key = 'status' AND extra.value = 'denied') AS notification_status_denied,
COUNTIF(event_category = 'app' AND event_name = 'notification_permission' AND extra.key = 'alert_setting' AND extra.value = 'notSupported') AS notification_alert_setting_not_supported,
COUNTIF(event_category = 'app' AND event_name = 'notification_permission' AND extra.key = 'alert_setting' AND extra.value = 'disabled') AS notification_alert_setting_disabled,
COUNTIF(event_category = 'app' AND event_name = 'notification_permission' AND extra.key = 'alert_setting' AND extra.value = 'enabled') AS notification_alert_setting_enabled
COUNTIF(
event_category = 'app'
AND event_name = 'notification_permission'
AND extra.key = 'status'
AND extra.value = 'authorized'
) AS notification_status_authorized,
COUNTIF(
event_category = 'app'
AND event_name = 'notification_permission'
AND extra.key = 'status'
AND extra.value = 'notDetermined'
) AS notification_status_notDetermined,
COUNTIF(
event_category = 'app'
AND event_name = 'notification_permission'
AND extra.key = 'status'
AND extra.value = 'denied'
) AS notification_status_denied,
COUNTIF(
event_category = 'app'
AND event_name = 'notification_permission'
AND extra.key = 'alert_setting'
AND extra.value = 'notSupported'
) AS notification_alert_setting_not_supported,
COUNTIF(
event_category = 'app'
AND event_name = 'notification_permission'
AND extra.key = 'alert_setting'
AND extra.value = 'disabled'
) AS notification_alert_setting_disabled,
COUNTIF(
event_category = 'app'
AND event_name = 'notification_permission'
AND extra.key = 'alert_setting'
AND extra.value = 'enabled'
) AS notification_alert_setting_enabled
FROM
firefox_ios.events_unnested
LEFT JOIN
@ -77,16 +165,34 @@ product_features_agg AS (
/*Logins*/
--autofill_failed
SUM(logins_autofill_failed) AS logins_autofill_failed,
COUNT(DISTINCT CASE WHEN logins_autofill_failed > 0 THEN client_id END) AS logins_autofill_failed_users,
COUNT(
DISTINCT
CASE
WHEN logins_autofill_failed > 0
THEN client_id
END
) AS logins_autofill_failed_users,
--logins_autofilled
SUM(logins_autofilled) AS logins_autofilled,
COUNT(DISTINCT CASE WHEN logins_autofilled > 0 THEN client_id END) AS logins_autofilled_users,
--logins_management_add_tapped
SUM(logins_management_add_tapped) AS logins_management_add_tapped,
COUNT(DISTINCT CASE WHEN logins_management_add_tapped > 0 THEN client_id END) AS logins_management_add_tapped_users,
COUNT(
DISTINCT
CASE
WHEN logins_management_add_tapped > 0
THEN client_id
END
) AS logins_management_add_tapped_users,
--logins_management_logins_tapped
SUM(logins_management_logins_tapped) AS logins_management_logins_tapped,
COUNT(DISTINCT CASE WHEN logins_management_logins_tapped > 0 THEN client_id END) AS logins_management_logins_tapped_users,
COUNT(
DISTINCT
CASE
WHEN logins_management_logins_tapped > 0
THEN client_id
END
) AS logins_management_logins_tapped_users,
/*Credit Card*/
--autofill_failed
SUM(cc_autofill_failed) AS cc_autofill_failed,

180
sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v1/query.sql
Просмотреть файл

@ -14,55 +14,66 @@ product_features AS (
client_info.client_id,
DATE(submission_timestamp) AS submission_date,
/*Logins*/
COALESCE(
SUM(metrics.counter.logins_deleted),
0
) AS logins_deleted,
COALESCE(
SUM(metrics.counter.logins_modified),
0
) AS logins_modified,
COALESCE(
SUM(metrics.counter.logins_saved),
0
) AS logins_saved,
COALESCE(
SUM(metrics.quantity.logins_saved_all),
0
) AS logins_saved_all,
COALESCE(SUM(metrics.counter.logins_deleted), 0) AS logins_deleted,
COALESCE(SUM(metrics.counter.logins_modified), 0) AS logins_modified,
COALESCE(SUM(metrics.counter.logins_saved), 0) AS logins_saved,
COALESCE(SUM(metrics.quantity.logins_saved_all), 0) AS logins_saved_all,
/*Credit Card*/
SUM(CASE WHEN metrics.boolean.credit_card_autofill_enabled THEN 1 ELSE 0 END) AS credit_card_autofill_enabled,
SUM(CASE WHEN metrics.boolean.credit_card_sync_enabled THEN 1 ELSE 0 END) AS credit_card_sync_enabled,
SUM(
CASE
WHEN metrics.boolean.credit_card_autofill_enabled
THEN 1
ELSE 0
END
) AS credit_card_autofill_enabled,
SUM(
CASE
WHEN metrics.boolean.credit_card_sync_enabled
THEN 1
ELSE 0
END
) AS credit_card_sync_enabled,
/*Bookmark*/
COALESCE(SUM(bookmarks_add_table.value), 0) AS bookmarks_add,
COALESCE(
SUM(bookmarks_delete_table.value),
0
) AS bookmarks_delete,
COALESCE(
SUM(bookmarks_edit_table.value),
0
) AS bookmarks_edit,
SUM(CASE WHEN metrics.boolean.bookmarks_has_mobile_bookmarks THEN 1 ELSE 0 END) AS has_mobile_bookmarks,
COALESCE(SUM(bookmarks_delete_table.value), 0) AS bookmarks_delete,
COALESCE(SUM(bookmarks_edit_table.value), 0) AS bookmarks_edit,
SUM(
CASE
WHEN metrics.boolean.bookmarks_has_mobile_bookmarks
THEN 1
ELSE 0
END
) AS has_mobile_bookmarks,
COALESCE(SUM(metrics.quantity.bookmarks_mobile_bookmarks_count), 0) AS mobile_bookmarks_count,
COALESCE(
SUM(bookmarks_open_table.value),
0
) AS bookmarks_open,
COALESCE(
SUM(bookmarks_view_list_table.value),
0
) AS bookmarks_view_list,
COALESCE(SUM(bookmarks_open_table.value), 0) AS bookmarks_open,
COALESCE(SUM(bookmarks_view_list_table.value), 0) AS bookmarks_view_list,
/*FxA*/
COALESCE(SUM(metrics.counter.sync_create_account_pressed), 0) AS sync_create_account_pressed,
COALESCE(SUM(metrics.counter.sync_open_tab), 0) AS sync_open_tab,
COALESCE(SUM(metrics.counter.sync_sign_in_sync_pressed), 0) AS sync_sign_in_sync_pressed,
/*Privacy*/
COALESCE(SUM(metrics.quantity.tabs_private_tabs_quantity), 0) AS tabs_private_tabs_quantity,
SUM(CASE WHEN metrics.boolean.preferences_close_private_tabs THEN 1 ELSE 0 END) AS preferences_close_private_tabs,
SUM(CASE WHEN metrics.boolean.tracking_protection_enabled THEN 1 ELSE 0 END) AS tracking_protection_enabled,
SUM(CASE WHEN metrics.string.tracking_protection_strength = 'strict' THEN 1 ELSE 0 END) AS tracking_protection_strict,
SUM(
CASE
WHEN metrics.boolean.preferences_close_private_tabs
THEN 1
ELSE 0
END
) AS preferences_close_private_tabs,
SUM(
CASE
WHEN metrics.boolean.tracking_protection_enabled
THEN 1
ELSE 0
END
) AS tracking_protection_enabled,
SUM(
CASE
WHEN metrics.string.tracking_protection_strength = 'strict'
THEN 1
ELSE 0
END
) AS tracking_protection_strict,
/*Tab Count*/
COALESCE(SUM(metrics.quantity.tabs_normal_tabs_quantity), 0) AS tabs_normal_tabs_quantity,
COALESCE(SUM(metrics.quantity.tabs_inactive_tabs_count), 0) AS tabs_inactive_tabs_count,
@ -76,12 +87,42 @@ product_features AS (
0
) AS settings_menu_set_as_default_browser_pressed,
/*Notification*/
SUM(CASE WHEN metrics.boolean.preferences_sync_notifs THEN 1 ELSE 0 END) AS preferences_sync_notifs,
SUM(CASE WHEN metrics.boolean.preferences_tips_and_features_notifs THEN 1 ELSE 0 END) AS preferences_tips_and_features_notifs,
SUM(
CASE
WHEN metrics.boolean.preferences_sync_notifs
THEN 1
ELSE 0
END
) AS preferences_sync_notifs,
SUM(
CASE
WHEN metrics.boolean.preferences_tips_and_features_notifs
THEN 1
ELSE 0
END
) AS preferences_tips_and_features_notifs,
/*Customize Home*/
SUM(CASE WHEN metrics.boolean.preferences_jump_back_in THEN 1 ELSE 0 END) AS preferences_jump_back_in,
SUM(CASE WHEN metrics.boolean.preferences_recently_visited THEN 1 ELSE 0 END) AS preferences_recently_visited,
SUM(CASE WHEN metrics.boolean.preferences_recently_saved THEN 1 ELSE 0 END) AS preferences_recently_saved,
SUM(
CASE
WHEN metrics.boolean.preferences_jump_back_in
THEN 1
ELSE 0
END
) AS preferences_jump_back_in,
SUM(
CASE
WHEN metrics.boolean.preferences_recently_visited
THEN 1
ELSE 0
END
) AS preferences_recently_visited,
SUM(
CASE
WHEN metrics.boolean.preferences_recently_saved
THEN 1
ELSE 0
END
) AS preferences_recently_saved,
SUM(CASE WHEN metrics.boolean.preferences_pocket THEN 1 ELSE 0 END) AS preferences_pocket,
COALESCE(SUM(metrics.counter.app_menu_customize_homepage), 0) AS app_menu_customize_homepage,
COALESCE(
@ -90,11 +131,16 @@ product_features AS (
) AS firefox_home_page_customize_homepage_button
FROM
firefox_ios.metrics AS metric
LEFT JOIN UNNEST(metric.metrics.labeled_counter.bookmarks_add) AS bookmarks_add_table
LEFT JOIN UNNEST(metric.metrics.labeled_counter.bookmarks_delete) AS bookmarks_delete_table
LEFT JOIN UNNEST(metric.metrics.labeled_counter.bookmarks_edit) AS bookmarks_edit_table
LEFT JOIN UNNEST(metric.metrics.labeled_counter.bookmarks_open) AS bookmarks_open_table
LEFT JOIN UNNEST(metric.metrics.labeled_counter.bookmarks_view_list) AS bookmarks_view_list_table
LEFT JOIN
UNNEST(metric.metrics.labeled_counter.bookmarks_add) AS bookmarks_add_table
LEFT JOIN
UNNEST(metric.metrics.labeled_counter.bookmarks_delete) AS bookmarks_delete_table
LEFT JOIN
UNNEST(metric.metrics.labeled_counter.bookmarks_edit) AS bookmarks_edit_table
LEFT JOIN
UNNEST(metric.metrics.labeled_counter.bookmarks_open) AS bookmarks_open_table
LEFT JOIN
UNNEST(metric.metrics.labeled_counter.bookmarks_view_list) AS bookmarks_view_list_table
WHERE
DATE(submission_timestamp) = @submission_date
GROUP BY
@ -106,40 +152,16 @@ product_features_agg AS (
submission_date,
/*Logins*/
--logins_deleted
COUNT(
DISTINCT
CASE
WHEN logins_deleted > 0
THEN client_id
END
) AS logins_deleted_users,
COUNT(DISTINCT CASE WHEN logins_deleted > 0 THEN client_id END) AS logins_deleted_users,
SUM(logins_deleted) AS logins_deleted,
--logins_modified
COUNT(
DISTINCT
CASE
WHEN logins_modified > 0
THEN client_id
END
) AS logins_modified_users,
COUNT(DISTINCT CASE WHEN logins_modified > 0 THEN client_id END) AS logins_modified_users,
SUM(logins_modified) AS logins_modified,
--logins_saved
COUNT(
DISTINCT
CASE
WHEN logins_saved > 0
THEN client_id
END
) AS logins_saved_users,
COUNT(DISTINCT CASE WHEN logins_saved > 0 THEN client_id END) AS logins_saved_users,
SUM(logins_saved) AS logins_saved,
--logins_saved_all
COUNT(
DISTINCT
CASE
WHEN logins_saved_all > 0
THEN client_id
END
) AS logins_saved_all_users,
COUNT(DISTINCT CASE WHEN logins_saved_all > 0 THEN client_id END) AS logins_saved_all_users,
SUM(logins_saved_all) AS logins_saved_all,
/*Credit Card*/
--credit card autofill enabled
@ -296,7 +318,9 @@ product_features_agg AS (
THEN client_id
END
) AS settings_menu_set_as_default_browser_pressed_users,
SUM(settings_menu_set_as_default_browser_pressed) AS settings_menu_set_as_default_browser_pressed,
SUM(
settings_menu_set_as_default_browser_pressed
) AS settings_menu_set_as_default_browser_pressed,
/*Notification*/
--preferences_sync_notifs
COUNT(

5
sql/moz-fx-data-shared-prod/firefox_ios_derived/firefox_ios_clients_v1/query.sql
Просмотреть файл

@ -199,7 +199,10 @@ SELECT
_current.metadata.adjust_info__source_ping
) AS adjust_info__source_ping
) AS metadata,
COALESCE(_previous.is_suspicious_device_client, _current.is_suspicious_device_client) AS is_suspicious_device_client,
COALESCE(
_previous.is_suspicious_device_client,
_current.is_suspicious_device_client
) AS is_suspicious_device_client,
FROM
_current
FULL OUTER JOIN

6
sql/moz-fx-data-shared-prod/firefox_ios_derived/funnel_retention_clients_week_2_v1/checks.sql
Просмотреть файл

@ -11,5 +11,7 @@ SELECT
),
NULL
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE submission_date = @submission_date;
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date;

11
sql/moz-fx-data-shared-prod/firefox_ios_derived/funnel_retention_clients_week_4_v1/checks.sql
Просмотреть файл

@ -35,12 +35,17 @@ SELECT
),
NULL
);
#fail
SELECT
IF(
DATE_DIFF(submission_date, first_seen_date, DAY) <> 27,
ERROR("Day difference between submission_date and first_seen_date is not equal to 27 as expected"),
ERROR(
"Day difference between submission_date and first_seen_date is not equal to 27 as expected"
),
NULL
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE submission_date = @submission_date;
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date;

136
sql/moz-fx-data-shared-prod/firefox_ios_derived/funnel_retention_week_4_v1/checks.sql
Просмотреть файл

@ -9,70 +9,110 @@
{{ min_row_count(1, "submission_date = @submission_date") }}
#fail
WITH new_profile_count AS (
SELECT SUM(new_profiles) FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` WHERE submission_date = @submission_date
SELECT
SUM(new_profiles)
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date
),
new_profile_upstream_count AS (
SELECT COUNT(*) FROM `{{ project_id }}.{{ dataset_id }}.funnel_retention_clients_week_4_v1` WHERE submission_date = @submission_date
SELECT
COUNT(*)
FROM
`{{ project_id }}.{{ dataset_id }}.funnel_retention_clients_week_4_v1`
WHERE
submission_date = @submission_date
)
SELECT IF(
(SELECT * FROM new_profile_count) <> (SELECT * FROM new_profile_upstream_count),
ERROR(
CONCAT(
"New profile count mismatch between this (",
(SELECT * FROM new_profile_count),
") and upstream (",
(SELECT * FROM new_profile_upstream_count),
") tables"
)
),
NULL
);
SELECT
IF(
(SELECT * FROM new_profile_count) <> (SELECT * FROM new_profile_upstream_count),
ERROR(
CONCAT(
"New profile count mismatch between this (",
(SELECT * FROM new_profile_count),
") and upstream (",
(SELECT * FROM new_profile_upstream_count),
") tables"
)
),
NULL
);
#fail
WITH repeat_user_count AS (
SELECT SUM(repeat_user) FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` WHERE submission_date = @submission_date
SELECT
SUM(repeat_user)
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date
),
repeat_user_upstream_count AS (
SELECT COUNTIF(repeat_first_month_user) FROM `{{ project_id }}.{{ dataset_id }}.funnel_retention_clients_week_4_v1` WHERE submission_date = @submission_date
SELECT
COUNTIF(repeat_first_month_user)
FROM
`{{ project_id }}.{{ dataset_id }}.funnel_retention_clients_week_4_v1`
WHERE
submission_date = @submission_date
)
SELECT IF(
(SELECT * FROM repeat_user_count) <> (SELECT * FROM repeat_user_upstream_count),
ERROR(
CONCAT(
"New profile count mismatch between this (",
(SELECT * FROM repeat_user_count),
") and upstream (",
(SELECT * FROM repeat_user_upstream_count),
") tables"
)
),
NULL
);
SELECT
IF(
(SELECT * FROM repeat_user_count) <> (SELECT * FROM repeat_user_upstream_count),
ERROR(
CONCAT(
"New profile count mismatch between this (",
(SELECT * FROM repeat_user_count),
") and upstream (",
(SELECT * FROM repeat_user_upstream_count),
") tables"
)
),
NULL
);
#fail
WITH retained_week_4_count AS (
SELECT SUM(retained_week_4) FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` WHERE submission_date = @submission_date
SELECT
SUM(retained_week_4)
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date
),
retained_week_4_upstream_count AS (
SELECT COUNTIF(retained_week_4) FROM `{{ project_id }}.{{ dataset_id }}.funnel_retention_clients_week_4_v1` WHERE submission_date = @submission_date
SELECT
COUNTIF(retained_week_4)
FROM
`{{ project_id }}.{{ dataset_id }}.funnel_retention_clients_week_4_v1`
WHERE
submission_date = @submission_date
)
SELECT IF(
(SELECT * FROM retained_week_4_count) <> (SELECT * FROM retained_week_4_upstream_count),
ERROR(
CONCAT(
"New profile count mismatch between this (",
(SELECT * FROM retained_week_4_count),
") and upstream (",
(SELECT * FROM retained_week_4_upstream_count),
") tables"
)
),
NULL
);
SELECT
IF(
(SELECT * FROM retained_week_4_count) <> (SELECT * FROM retained_week_4_upstream_count),
ERROR(
CONCAT(
"New profile count mismatch between this (",
(SELECT * FROM retained_week_4_count),
") and upstream (",
(SELECT * FROM retained_week_4_upstream_count),
") tables"
)
),
NULL
);
#fail
SELECT
IF(
DATE_DIFF(submission_date, first_seen_date, DAY) <> 27,
ERROR("Day difference between submission_date and first_seen_date is not equal to 27 as expected"),
ERROR(
"Day difference between submission_date and first_seen_date is not equal to 27 as expected"
),
NULL
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE submission_date = @submission_date;
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date;

13
sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profile_activation_v2/checks.sql
Просмотреть файл

@ -9,8 +9,11 @@ SELECT
ERROR("Number of is_new_profile TRUE values should be the same as the row count."),
NULL
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE `date` = @submission_date;
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
`date` = @submission_date;
#fail
SELECT
IF(
@ -18,5 +21,7 @@ SELECT
ERROR("Day difference between values inside `date` and submission_date fields should be 6."),
NULL
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE `date` = @submission_date;
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
`date` = @submission_date;

1
sql/moz-fx-data-shared-prod/fivetran_costs_derived/daily_connector_costs_v1/checks.sql
Просмотреть файл

@ -6,3 +6,4 @@
#fail
{{ min_row_count(1) }}

1
sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/checks.sql
Просмотреть файл

@ -3,3 +3,4 @@
#fail
{{ min_row_count(1) }}

1
sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/checks.sql
Просмотреть файл

@ -6,3 +6,4 @@
#fail
{{ min_row_count(1) }}

1
sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/checks.sql
Просмотреть файл

@ -34,3 +34,4 @@
#warn
{{ value_length(column="country", expected_length=2, where="DATE(`datetime`) = @submission_date") }}

4
sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/query.sql
Просмотреть файл

@ -1,8 +1,6 @@
SELECT
IF(_current.id IS NULL, _previous, _current).* REPLACE (
CAST(
NULL AS STRING
) AS fxa_profile_json, -- contains unhashed fxa_uid
CAST(NULL AS STRING) AS fxa_profile_json, -- contains unhashed fxa_uid
IF(_current.id IS NULL, _previous.fxa_uid, TO_HEX(SHA256(_current.fxa_uid))) AS fxa_uid
)
FROM

6
sql/moz-fx-data-shared-prod/org_mozilla_broken_site_report/user_reports/view.sql
Просмотреть файл

@ -8,9 +8,7 @@ WITH historical_reports AS (
FROM
`moz-fx-data-shared-prod.firefox_desktop_stable.broken_site_report_v1`
WHERE
DATE(
submission_timestamp
) > "2023-11-01"
DATE(submission_timestamp) > "2023-11-01"
),
live_reports AS (
SELECT
@ -38,7 +36,7 @@ all_reports AS (
historical_reports
)
SELECT
document_id as uuid,
document_id AS uuid,
CAST(submission_timestamp AS DATETIME) AS reported_at,
metrics.text2.broken_site_report_description AS comments,
metrics.url2.broken_site_report_url AS url,

7
sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/query.sql
Просмотреть файл

@ -11,12 +11,7 @@ WITH ranked_data AS (
normalized_country_code,
sample_id,
ping_info.experiments AS experiments,
ROW_NUMBER() OVER (
PARTITION BY
client_info.client_id
ORDER BY
submission_timestamp
) AS row_num
ROW_NUMBER() OVER (PARTITION BY client_info.client_id ORDER BY submission_timestamp) AS row_num
FROM
`moz-fx-data-shared-prod.fenix.metrics`
WHERE

88
sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/query.sql
Просмотреть файл

@ -3,23 +3,89 @@ SELECT
client_info.client_id AS client_id,
client_info.os AS os,
client_info.os_version AS os_version,
MAX(CASE WHEN name = 'address_bar_feature_callout_displayed' THEN 1 ELSE 0 END) is_address_bar_feature_callout_displayed,
MAX(
CASE
WHEN name = 'address_bar_feature_callout_displayed'
THEN 1
ELSE 0
END
) is_address_bar_feature_callout_displayed,
MAX(CASE WHEN name = 'address_bar_icon_clicked' THEN 1 ELSE 0 END) is_address_bar_icon_clicked,
MAX(CASE WHEN name = 'address_bar_icon_displayed' THEN 1 ELSE 0 END) is_address_bar_icon_displayed,
MAX(CASE WHEN name = 'surface_analyze_reviews_none_available_clicked' THEN 1 ELSE 0 END) is_surface_analyze_reviews_none_available_clicked,
MAX(
CASE
WHEN name = 'address_bar_icon_displayed'
THEN 1
ELSE 0
END
) is_address_bar_icon_displayed,
MAX(
CASE
WHEN name = 'surface_analyze_reviews_none_available_clicked'
THEN 1
ELSE 0
END
) is_surface_analyze_reviews_none_available_clicked,
MAX(CASE WHEN name = 'surface_closed' THEN 1 ELSE 0 END) is_surface_closed,
MAX(CASE WHEN name = 'surface_displayed' THEN 1 ELSE 0 END) is_surface_displayed,
MAX(CASE WHEN name = 'surface_expand_settings' THEN 1 ELSE 0 END) is_surface_expand_settings,
MAX(CASE WHEN name = 'surface_learn_more_clicked' THEN 1 ELSE 0 END) is_surface_learn_more_clicked,
MAX(CASE WHEN name = 'surface_no_review_reliability_available' THEN 1 ELSE 0 END) is_surface_no_review_reliability_available,
MAX(CASE WHEN name = 'surface_onboarding_displayed' THEN 1 ELSE 0 END) is_surface_onboarding_displayed,
MAX(
CASE
WHEN name = 'surface_learn_more_clicked'
THEN 1
ELSE 0
END
) is_surface_learn_more_clicked,
MAX(
CASE
WHEN name = 'surface_no_review_reliability_available'
THEN 1
ELSE 0
END
) is_surface_no_review_reliability_available,
MAX(
CASE
WHEN name = 'surface_onboarding_displayed'
THEN 1
ELSE 0
END
) is_surface_onboarding_displayed,
MAX(CASE WHEN name = 'surface_opt_in_accepted' THEN 1 ELSE 0 END) surface_opt_in_accepted,
MAX(CASE WHEN name = 'surface_reactivated_button_clicked' THEN 1 ELSE 0 END) is_surface_reactivated_button_clicked,
MAX(
CASE
WHEN name = 'surface_reactivated_button_clicked'
THEN 1
ELSE 0
END
) is_surface_reactivated_button_clicked,
MAX(CASE WHEN name = 'surface_reanalyze_clicked' THEN 1 ELSE 0 END) is_surface_reanalyze_clicked,
MAX(CASE WHEN name = 'surface_show_more_recent_reviews_clicked' THEN 1 ELSE 0 END) is_surface_show_more_recent_reviews_clicked,
MAX(CASE WHEN name = 'surface_show_privacy_policy_clicked' THEN 1 ELSE 0 END) is_surface_show_privacy_policy_clicked,
MAX(CASE WHEN name = 'surface_review_quality_explainer_url_clicked' THEN 1 ELSE 0 END) is_surface_show_quality_explainer_url_clicked,
MAX(CASE WHEN name = 'surface_show_terms_clicked' THEN 1 ELSE 0 END) is_surface_show_terms_clicked,
MAX(
CASE
WHEN name = 'surface_show_more_recent_reviews_clicked'
THEN 1
ELSE 0
END
) is_surface_show_more_recent_reviews_clicked,
MAX(
CASE
WHEN name = 'surface_show_privacy_policy_clicked'
THEN 1
ELSE 0
END
) is_surface_show_privacy_policy_clicked,
MAX(
CASE
WHEN name = 'surface_review_quality_explainer_url_clicked'
THEN 1
ELSE 0
END
) is_surface_show_quality_explainer_url_clicked,
MAX(
CASE
WHEN name = 'surface_show_terms_clicked'
THEN 1
ELSE 0
END
) is_surface_show_terms_clicked,
MAX(
CASE
WHEN name IN (

7
sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/query.sql
Просмотреть файл

@ -11,12 +11,7 @@ WITH ranked_data AS (
normalized_country_code,
sample_id,
ping_info.experiments AS experiments,
ROW_NUMBER() OVER (
PARTITION BY
client_info.client_id
ORDER BY
submission_timestamp
) AS row_num
ROW_NUMBER() OVER (PARTITION BY client_info.client_id ORDER BY submission_timestamp) AS row_num
FROM
`moz-fx-data-shared-prod.firefox_ios.metrics`
WHERE

112
sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/query.sql
Просмотреть файл

@ -3,25 +3,109 @@ SELECT
client_info.client_id AS client_id,
client_info.os AS os,
client_info.os_version AS os_version,
MAX(CASE WHEN name = 'address_bar_feature_callout_displayed' THEN 1 ELSE 0 END) is_address_bar_feature_callout_displayed,
MAX(
CASE
WHEN name = 'address_bar_feature_callout_displayed'
THEN 1
ELSE 0
END
) is_address_bar_feature_callout_displayed,
MAX(CASE WHEN name = 'address_bar_icon_clicked' THEN 1 ELSE 0 END) is_address_bar_icon_clicked,
MAX(CASE WHEN name = 'address_bar_icon_displayed' THEN 1 ELSE 0 END) is_address_bar_icon_displayed,
MAX(CASE WHEN name = 'surface_analyze_reviews_none_available_clicked' THEN 1 ELSE 0 END) is_surface_analyze_reviews_none_available_clicked,
MAX(
CASE
WHEN name = 'address_bar_icon_displayed'
THEN 1
ELSE 0
END
) is_address_bar_icon_displayed,
MAX(
CASE
WHEN name = 'surface_analyze_reviews_none_available_clicked'
THEN 1
ELSE 0
END
) is_surface_analyze_reviews_none_available_clicked,
MAX(CASE WHEN name = 'surface_closed' THEN 1 ELSE 0 END) is_surface_closed,
MAX(CASE WHEN name = 'surface_displayed' THEN 1 ELSE 0 END) is_surface_displayed,
MAX(CASE WHEN name = 'surface_settings_expand_clicked' THEN 1 ELSE 0 END) is_surface_settings_expand_clicked,
MAX(CASE WHEN name = 'surface_learn_more_clicked' THEN 1 ELSE 0 END) is_surface_learn_more_clicked,
MAX(CASE WHEN name = 'surface_no_review_reliability_available' THEN 1 ELSE 0 END) is_surface_no_review_reliability_available,
MAX(CASE WHEN name = 'surface_onboarding_displayed' THEN 1 ELSE 0 END) is_surface_onboarding_displayed,
MAX(
CASE
WHEN name = 'surface_settings_expand_clicked'
THEN 1
ELSE 0
END
) is_surface_settings_expand_clicked,
MAX(
CASE
WHEN name = 'surface_learn_more_clicked'
THEN 1
ELSE 0
END
) is_surface_learn_more_clicked,
MAX(
CASE
WHEN name = 'surface_no_review_reliability_available'
THEN 1
ELSE 0
END
) is_surface_no_review_reliability_available,
MAX(
CASE
WHEN name = 'surface_onboarding_displayed'
THEN 1
ELSE 0
END
) is_surface_onboarding_displayed,
MAX(CASE WHEN name = 'surface_opt_in_accepted' THEN 1 ELSE 0 END) is_surface_opt_in_accepted,
MAX(CASE WHEN name = 'surface_reactivated_button_clicked' THEN 1 ELSE 0 END) is_surface_reactivated_button_clicked,
MAX(
CASE
WHEN name = 'surface_reactivated_button_clicked'
THEN 1
ELSE 0
END
) is_surface_reactivated_button_clicked,
MAX(CASE WHEN name = 'surface_reanalyze_clicked' THEN 1 ELSE 0 END) is_surface_reanalyze_clicked,
MAX(CASE WHEN name = 'surface_powered_by_fakespot_link_clicked' THEN 1 ELSE 0 END) is_surface_powered_by_fakespot_link_clicked,
MAX(CASE WHEN name = 'surface_show_more_recent_reviews_clicked' THEN 1 ELSE 0 END) is_surface_show_more_reviews_button_clicked,
MAX(CASE WHEN name = 'surface_show_privacy_policy_clicked' THEN 1 ELSE 0 END) is_surface_show_privacy_policy_clicked,
MAX(CASE WHEN name = 'surface_show_quality_explainer_clicked' THEN 1 ELSE 0 END) is_surface_show_quality_explainer_url_clicked,
MAX(CASE WHEN name = 'surface_show_terms_clicked' THEN 1 ELSE 0 END) is_surface_show_terms_clicked,
MAX(CASE WHEN name = 'surface_stale_analysis_shown' THEN 1 ELSE 0 END) is_surface_stale_analysis_shown,
MAX(
CASE
WHEN name = 'surface_powered_by_fakespot_link_clicked'
THEN 1
ELSE 0
END
) is_surface_powered_by_fakespot_link_clicked,
MAX(
CASE
WHEN name = 'surface_show_more_recent_reviews_clicked'
THEN 1
ELSE 0
END
) is_surface_show_more_reviews_button_clicked,
MAX(
CASE
WHEN name = 'surface_show_privacy_policy_clicked'
THEN 1
ELSE 0
END
) is_surface_show_privacy_policy_clicked,
MAX(
CASE
WHEN name = 'surface_show_quality_explainer_clicked'
THEN 1
ELSE 0
END
) is_surface_show_quality_explainer_url_clicked,
MAX(
CASE
WHEN name = 'surface_show_terms_clicked'
THEN 1
ELSE 0
END
) is_surface_show_terms_clicked,
MAX(
CASE
WHEN name = 'surface_stale_analysis_shown'
THEN 1
ELSE 0
END
) is_surface_stale_analysis_shown,
MAX(
CASE
WHEN name IN (

5
sql/moz-fx-data-shared-prod/search/mobile_search_aggregates/view.sql
Просмотреть файл

@ -4,7 +4,10 @@ AS
SELECT
* EXCEPT (normalized_engine, normalized_app_name),
`moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine,
`mozfun.mobile_search.normalize_app_name`(app_name, os).normalized_app_name AS normalized_app_name,
`mozfun.mobile_search.normalize_app_name`(
app_name,
os
).normalized_app_name AS normalized_app_name,
search_count AS sap,
`mozfun.mobile_search.normalize_app_name`(app_name, os).normalized_app_name_os
FROM

5
sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/view.sql
Просмотреть файл

@ -4,7 +4,10 @@ AS
SELECT
* EXCEPT (normalized_engine, normalized_app_name),
`moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine,
`mozfun.mobile_search.normalize_app_name`(app_name, os).normalized_app_name AS normalized_app_name,
`mozfun.mobile_search.normalize_app_name`(
app_name,
os
).normalized_app_name AS normalized_app_name,
`mozfun.norm.browser_version_info`(app_version) AS browser_version_info,
search_count AS sap,
`mozfun.mobile_search.normalize_app_name`(app_name, os).normalized_app_name_os

5
sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql
Просмотреть файл

@ -777,10 +777,7 @@ combined_search_clients AS (
WHEN REGEXP_CONTAINS(source, '^in-content.*-follow-on')
THEN 'tagged-follow-on'
WHEN STARTS_WITH(source, 'in-content.organic')
OR STARTS_WITH(
source,
'organic.'
) -- for ios
OR STARTS_WITH(source, 'organic.') -- for ios
THEN 'organic'
WHEN search_type = 'ad-click'
OR search_type = 'search-with-ads'

2
sql/moz-fx-data-shared-prod/subscription_platform_derived/active_subscriptions_v1/query.sql
Просмотреть файл

@ -1,5 +1,5 @@
{#- format off #}
WITH product_union AS (
{#- format off #}
{%- for product in ["mozilla_vpn", "relay", "hubs"] %}
SELECT
active_date,

1
sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_v1/checks.sql
Просмотреть файл

@ -1,2 +1,3 @@
#fail
{{ row_count_within_past_partitions_avg(number_of_days=7, threshold_percentage=5) }}

48
sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_v6/query.sql
Просмотреть файл

@ -147,7 +147,9 @@ WITH base AS (
UNNEST(
[
-- We add a struct layer here b/c BQ doesn't allow nested arrays
STRUCT(payload.keyed_histograms.fx_migration_bookmarks_quantity AS keyed_histogram), -- 0
STRUCT(
payload.keyed_histograms.fx_migration_bookmarks_quantity AS keyed_histogram
), -- 0
STRUCT(payload.keyed_histograms.fx_migration_history_quantity AS keyed_histogram), -- 1
STRUCT(payload.keyed_histograms.fx_migration_logins_quantity AS keyed_histogram) -- 2
]
@ -703,12 +705,8 @@ aggregates AS (
udf.aggregate_active_addons(
ARRAY_CONCAT_AGG(active_addons ORDER BY submission_timestamp)
) AS active_addons,
CAST(
NULL AS STRING
) AS active_experiment_branch, -- deprecated
CAST(
NULL AS STRING
) AS active_experiment_id, -- deprecated
CAST(NULL AS STRING) AS active_experiment_branch, -- deprecated
CAST(NULL AS STRING) AS active_experiment_id, -- deprecated
SUM(active_ticks / (3600 / 5)) AS active_hours_sum,
mozfun.stats.mode_last(
ARRAY_AGG(addon_compatibility_check_enabled ORDER BY submission_timestamp)
@ -783,9 +781,13 @@ aggregates AS (
ARRAY_AGG(distribution_id ORDER BY submission_timestamp)
) AS distribution_id,
mozfun.stats.mode_last(ARRAY_AGG(partner_id ORDER BY submission_timestamp)) AS partner_id,
mozfun.stats.mode_last(ARRAY_AGG(distribution_version ORDER BY submission_timestamp)) AS distribution_version,
mozfun.stats.mode_last(
ARRAY_AGG(distribution_version ORDER BY submission_timestamp)
) AS distribution_version,
mozfun.stats.mode_last(ARRAY_AGG(distributor ORDER BY submission_timestamp)) AS distributor,
mozfun.stats.mode_last(ARRAY_AGG(distributor_channel ORDER BY submission_timestamp)) AS distributor_channel,
mozfun.stats.mode_last(
ARRAY_AGG(distributor_channel ORDER BY submission_timestamp)
) AS distributor_channel,
mozfun.stats.mode_last(ARRAY_AGG(e10s_enabled ORDER BY submission_timestamp)) AS e10s_enabled,
mozfun.stats.mode_last(
ARRAY_AGG(env_build_arch ORDER BY submission_timestamp)
@ -878,7 +880,9 @@ aggregates AS (
submission_timestamp
)
).*,
mozfun.stats.mode_last(ARRAY_AGG(geo_db_version ORDER BY submission_timestamp)) AS geo_db_version,
mozfun.stats.mode_last(
ARRAY_AGG(geo_db_version ORDER BY submission_timestamp)
) AS geo_db_version,
mozfun.json.mode_last(
ARRAY_AGG(
IF(
@ -999,7 +1003,9 @@ aggregates AS (
ARRAY_AGG(is_default_browser ORDER BY submission_timestamp)
) AS is_default_browser,
mozfun.stats.mode_last(ARRAY_AGG(is_wow64 ORDER BY submission_timestamp)) AS is_wow64,
mozfun.stats.mode_last(ARRAY_AGG(apple_model_id ORDER BY submission_timestamp)) AS apple_model_id,
mozfun.stats.mode_last(
ARRAY_AGG(apple_model_id ORDER BY submission_timestamp)
) AS apple_model_id,
mozfun.stats.mode_last(ARRAY_AGG(locale ORDER BY submission_timestamp)) AS locale,
mozfun.stats.mode_last(ARRAY_AGG(memory_mb ORDER BY submission_timestamp)) AS memory_mb,
mozfun.stats.mode_last(
@ -1157,8 +1163,8 @@ aggregates AS (
udf.aggregate_search_counts(ARRAY_CONCAT_AGG(search_counts ORDER BY submission_timestamp)).*,
AVG(session_restored) AS session_restored_mean,
COUNTIF(subsession_counter = 1) AS sessions_started_on_this_day,
MAX(subsession_counter)AS max_subsession_counter,
MIN(subsession_counter)AS min_subsession_counter,
MAX(subsession_counter) AS max_subsession_counter,
MIN(subsession_counter) AS min_subsession_counter,
SUM(shutdown_kill) AS shutdown_kill_sum,
SUM(subsession_length / NUMERIC '3600') AS subsession_hours_sum,
SUM(ssl_handshake_result_failure) AS ssl_handshake_result_failure_sum,
@ -1378,9 +1384,13 @@ aggregates AS (
STRUCT(ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression)), -- 96
STRUCT(ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_dynamic_wikipedia)), -- 97
STRUCT(ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_nonsponsored)), -- 98
STRUCT(ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_nonsponsored_bestmatch)), -- 99
STRUCT(
ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_nonsponsored_bestmatch)
), -- 99
STRUCT(ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_sponsored)), -- 100
STRUCT(ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_sponsored_bestmatch)), -- 101
STRUCT(
ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_sponsored_bestmatch)
), -- 101
STRUCT(ARRAY_CONCAT_AGG(contextual_services_quicksuggest_impression_weather)), -- 102
STRUCT(ARRAY_CONCAT_AGG(contextual_services_topsites_click)), -- 103
STRUCT(ARRAY_CONCAT_AGG(contextual_services_topsites_impression)), -- 104
@ -1453,8 +1463,12 @@ aggregates AS (
OFFSET(0)
] AS startup_profile_selection_reason_first,
mozfun.stats.mode_last(
ARRAY_AGG(IF(subsession_counter = 1, startup_profile_selection_reason, NULL) ORDER BY submission_timestamp ASC)
) as startup_profile_selection_first_ping_only,
ARRAY_AGG(
IF(subsession_counter = 1, startup_profile_selection_reason, NULL)
ORDER BY
submission_timestamp ASC
)
) AS startup_profile_selection_first_ping_only,
SUM(
scalar_parent_browser_ui_interaction_textrecognition_error
) AS scalar_parent_browser_ui_interaction_textrecognition_error_sum,

34
sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/query.sql
Просмотреть файл

@ -476,7 +476,14 @@ _current AS (
unioned.sample_id AS sample_id,
fsd.first_seen_date AS first_seen_date,
ssd.second_seen_date AS second_seen_date,
unioned.* EXCEPT (client_id, sample_id, first_seen_timestamp, all_dates, source_ping, source_ping_priority),
unioned.* EXCEPT (
client_id,
sample_id,
first_seen_timestamp,
all_dates,
source_ping,
source_ping_priority
),
STRUCT(
fsd.first_seen_source_ping AS first_seen_date_source_ping,
pings.reported_main_ping AS reported_main_ping,
@ -505,17 +512,18 @@ _previous AS (
FROM
`moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2`
)
SELECT
{% if is_init() %}
SELECT
*
FROM
FROM
_current
{% else %}
-- For the daily update:
-- The reported ping status in the metadata is updated when it's NULL.
-- The second_seen_date is updated when it's NULL and only if there is a
-- main ping reported on the submission_date.
-- Every other attribute remains as reported on the first_seen_date.
-- For the daily update:
-- The reported ping status in the metadata is updated when it's NULL.
-- The second_seen_date is updated when it's NULL and only if there is a
-- main ping reported on the submission_date.
-- Every other attribute remains as reported on the first_seen_date.
SELECT
IF(_previous.client_id IS NULL, _current, _previous).* REPLACE (
IF(
_previous.first_seen_date IS NOT NULL
@ -552,9 +560,9 @@ FROM
)
) AS metadata
)
FROM
_previous
FULL JOIN
_current
USING (client_id)
FROM
_previous
FULL JOIN
_current
USING (client_id)
{% endif %}

3
sql/moz-fx-data-shared-prod/telemetry_derived/crash_frames_v1/query.sql
Просмотреть файл

@ -6,7 +6,8 @@ SELECT
_module.debug_file,
_module.debug_id,
IF(
_frame.ip LIKE '0x%' AND _module.base_addr LIKE '0x%',
_frame.ip LIKE '0x%'
AND _module.base_addr LIKE '0x%',
SAFE_CAST(_frame.ip AS INT64) - SAFE_CAST(_module.base_addr AS INT64),
NULL
) AS module_offset,

106
sql/moz-fx-data-shared-prod/telemetry_derived/desktop_cohort_daily_retention_v1/query.sql
Просмотреть файл

@ -4,7 +4,7 @@ WITH submission_date_activity AS (
client_id,
days_seen_bits,
(days_visited_1_uri_bits & days_interacted_bits) AS days_seen_dau_bits,
DATE(submission_date) as submission_date
DATE(submission_date) AS submission_date
FROM
telemetry.clients_last_seen_v1
WHERE
@ -40,7 +40,7 @@ cohorts_in_range AS (
db_version,
distribution_id,
locale,
app_name as normalized_app_name,
app_name AS normalized_app_name,
normalized_channel,
normalized_os,
normalized_os_version,
@ -65,10 +65,14 @@ activity_cohort_match AS (
SELECT
cohorts_in_range.client_id AS client_id,
submission_date_activity.client_id AS active_client_id,
submission_date_activity.days_seen_bits as active_client_days_seen_bits,
submission_date_activity.days_seen_bits AS active_client_days_seen_bits,
submission_date_activity.days_seen_dau_bits,
mozfun.bits28.days_since_seen(submission_date_activity.days_seen_bits) as active_clients_days_since_seen,
mozfun.bits28.days_since_seen(submission_date_activity.days_seen_dau_bits) as dau_clients_days_since_seen,
mozfun.bits28.days_since_seen(
submission_date_activity.days_seen_bits
) AS active_clients_days_since_seen,
mozfun.bits28.days_since_seen(
submission_date_activity.days_seen_dau_bits
) AS dau_clients_days_since_seen,
cohorts_in_range.* EXCEPT (client_id)
FROM
cohorts_in_range
@ -107,36 +111,68 @@ SELECT
os_version_major,
os_version_minor,
COUNT(client_id) AS num_clients_in_cohort,
COUNTIF((active_client_id IS NOT NULL) AND (active_clients_days_since_seen = 0)) AS num_clients_active_on_day,
COUNTIF((active_client_id IS NOT NULL) AND (dau_clients_days_since_seen = 0)) AS num_clients_dau_on_day,
COUNTIF((active_client_id IS NOT NULL) AND
(COALESCE(
BIT_COUNT(mozfun.bits28.from_string('0000000000000000000001111111') & active_client_days_seen_bits) > 0,
FALSE
)) ) AS num_clients_active_atleastonce_in_last_7_days,
COUNTIF((active_client_id IS NOT NULL) AND
( COALESCE(
BIT_COUNT(active_client_days_seen_bits) > 0,
FALSE) )) AS num_clients_active_atleastonce_in_last_28_days,
COUNTIF((active_client_id IS NOT NULL) AND
DATE_DIFF(submission_date, first_seen_date, DAY) = 27 AND
( COALESCE(
BIT_COUNT(mozfun.bits28.from_string('0111111111111111111111111111') & active_client_days_seen_bits) > 0,
FALSE) )) AS num_clients_repeat_first_month_users,
COUNTIF((active_client_id IS NOT NULL) AND
(COALESCE(
BIT_COUNT(mozfun.bits28.from_string('0000000000000000000001111111') & days_seen_dau_bits) > 0,
FALSE
)) ) AS num_clients_dau_active_atleastonce_in_last_7_days,
COUNTIF((active_client_id IS NOT NULL) AND
( COALESCE(
BIT_COUNT(days_seen_dau_bits) > 0,
FALSE) )) AS num_clients_dau_active_atleastonce_in_last_28_days,
COUNTIF((active_client_id IS NOT NULL) AND
DATE_DIFF(submission_date, first_seen_date, DAY) = 27 AND
( COALESCE(
BIT_COUNT(mozfun.bits28.from_string('0111111111111111111111111111') & days_seen_dau_bits) > 0,
FALSE) )) AS num_clients_dau_repeat_first_month_users
COUNTIF(
(active_client_id IS NOT NULL)
AND (active_clients_days_since_seen = 0)
) AS num_clients_active_on_day,
COUNTIF(
(active_client_id IS NOT NULL)
AND (dau_clients_days_since_seen = 0)
) AS num_clients_dau_on_day,
COUNTIF(
(active_client_id IS NOT NULL)
AND (
COALESCE(
BIT_COUNT(
mozfun.bits28.from_string('0000000000000000000001111111') & active_client_days_seen_bits
) > 0,
FALSE
)
)
) AS num_clients_active_atleastonce_in_last_7_days,
COUNTIF(
(active_client_id IS NOT NULL)
AND (COALESCE(BIT_COUNT(active_client_days_seen_bits) > 0, FALSE))
) AS num_clients_active_atleastonce_in_last_28_days,
COUNTIF(
(active_client_id IS NOT NULL)
AND DATE_DIFF(submission_date, first_seen_date, DAY) = 27
AND (
COALESCE(
BIT_COUNT(
mozfun.bits28.from_string('0111111111111111111111111111') & active_client_days_seen_bits
) > 0,
FALSE
)
)
) AS num_clients_repeat_first_month_users,
COUNTIF(
(active_client_id IS NOT NULL)
AND (
COALESCE(
BIT_COUNT(
mozfun.bits28.from_string('0000000000000000000001111111') & days_seen_dau_bits
) > 0,
FALSE
)
)
) AS num_clients_dau_active_atleastonce_in_last_7_days,
COUNTIF(
(active_client_id IS NOT NULL)
AND (COALESCE(BIT_COUNT(days_seen_dau_bits) > 0, FALSE))
) AS num_clients_dau_active_atleastonce_in_last_28_days,
COUNTIF(
(active_client_id IS NOT NULL)
AND DATE_DIFF(submission_date, first_seen_date, DAY) = 27
AND (
COALESCE(
BIT_COUNT(
mozfun.bits28.from_string('0111111111111111111111111111') & days_seen_dau_bits
) > 0,
FALSE
)
)
) AS num_clients_dau_repeat_first_month_users
FROM
activity_cohort_match
GROUP BY

150
sql/moz-fx-data-shared-prod/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql
Просмотреть файл

@ -1,74 +1,73 @@
WITH
final_probe_extract AS (
SELECT
app_version,
COALESCE(os, "*") AS os,
COALESCE(app_build_id, "*") AS app_build_id,
process,
metric,
REPLACE(KEY, r"\x00", "") AS KEY,
client_agg_type,
metric_type,
total_users,
-- Using MAX instead of COALESCE since this is not in the GROUP BY.
MAX(
IF
(agg_type = "histogram", mozfun.glam.histogram_cast_json(aggregates), NULL)) AS histogram,
MAX(
IF
(agg_type = "histogram", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL)) AS non_norm_histogram,
MAX(
IF
(agg_type = "percentiles", mozfun.glam.histogram_cast_json(aggregates), NULL) ) AS percentiles,
MAX(
IF
(agg_type = "percentiles", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL) ) AS non_norm_percentiles
FROM
`moz-fx-data-shared-prod.telemetry_derived.client_probe_counts`
WHERE
channel = @channel
AND app_version IS NOT NULL
AND total_users > 375
AND CHAR_LENGTH(KEY) <= 200
GROUP BY
channel,
app_version,
app_build_id,
os,
metric,
metric_type,
KEY,
process,
client_agg_type,
total_users ),
glam_sample_counts AS (
SELECT
fsc1.os,
fsc1.app_version,
fsc1.app_build_id,
fsc1.metric,
fsc1.key,
fsc1.process,
fsc1.agg_type,
CASE
WHEN fsc1.agg_type IN ('max', 'min', 'sum', 'avg') AND fsc2.agg_type = 'count' THEN fsc2.total_sample
ELSE
fsc1.total_sample
END
AS total_sample
FROM
`moz-fx-data-shared-prod.telemetry_derived.glam_sample_counts_v1` fsc1
INNER JOIN
`moz-fx-data-shared-prod.telemetry_derived.glam_sample_counts_v1` fsc2
ON fsc1.os = fsc2.os
AND fsc1.app_build_id = fsc2.app_build_id
AND fsc1.app_version = fsc2.app_version
AND fsc1.metric = fsc2.metric
AND fsc1.key = fsc2.key
AND fsc1.channel = fsc2.channel
AND fsc1.process = fsc2.process
WHERE
fsc1.channel = @channel )
WITH final_probe_extract AS (
SELECT
app_version,
COALESCE(os, "*") AS os,
COALESCE(app_build_id, "*") AS app_build_id,
process,
metric,
REPLACE(KEY, r"\x00", "") AS KEY,
client_agg_type,
metric_type,
total_users,
-- Using MAX instead of COALESCE since this is not in the GROUP BY.
MAX(IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(aggregates), NULL)) AS histogram,
MAX(
IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL)
) AS non_norm_histogram,
MAX(
IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(aggregates), NULL)
) AS percentiles,
MAX(
IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL)
) AS non_norm_percentiles
FROM
`moz-fx-data-shared-prod.telemetry_derived.client_probe_counts`
WHERE
channel = @channel
AND app_version IS NOT NULL
AND total_users > 375
AND CHAR_LENGTH(KEY) <= 200
GROUP BY
channel,
app_version,
app_build_id,
os,
metric,
metric_type,
KEY,
process,
client_agg_type,
total_users
),
glam_sample_counts AS (
SELECT
fsc1.os,
fsc1.app_version,
fsc1.app_build_id,
fsc1.metric,
fsc1.key,
fsc1.process,
fsc1.agg_type,
CASE
WHEN fsc1.agg_type IN ('max', 'min', 'sum', 'avg')
AND fsc2.agg_type = 'count'
THEN fsc2.total_sample
ELSE fsc1.total_sample
END AS total_sample
FROM
`moz-fx-data-shared-prod.telemetry_derived.glam_sample_counts_v1` fsc1
INNER JOIN
`moz-fx-data-shared-prod.telemetry_derived.glam_sample_counts_v1` fsc2
ON fsc1.os = fsc2.os
AND fsc1.app_build_id = fsc2.app_build_id
AND fsc1.app_version = fsc2.app_version
AND fsc1.metric = fsc2.metric
AND fsc1.key = fsc2.key
AND fsc1.channel = fsc2.channel
AND fsc1.process = fsc2.process
WHERE
fsc1.channel = @channel
)
SELECT
cp.app_version,
cp.os,
@ -82,11 +81,10 @@ SELECT
histogram,
percentiles,
CASE
WHEN client_agg_type = '' THEN 0
ELSE
CAST(total_sample as BIGNUMERIC)
END
AS total_sample,
WHEN client_agg_type = ''
THEN 0
ELSE CAST(total_sample AS BIGNUMERIC)
END AS total_sample,
non_norm_histogram,
non_norm_percentiles
FROM

6
sql/moz-fx-data-shared-prod/telemetry_derived/glam_sample_counts_v1/query.sql
Просмотреть файл

@ -9,8 +9,7 @@ WITH histogram_data AS (
key,
h1.agg_type,
h1.aggregates,
IF(os = 'Windows'
AND channel = 'release', 10, 1) AS sample_mult
IF(os = 'Windows' AND channel = 'release', 10, 1) AS sample_mult
FROM
clients_histogram_aggregates_v2,
UNNEST(histogram_aggregates) h1
@ -24,8 +23,7 @@ scalars_data AS (
app_build_id,
channel,
scalar_aggregates,
IF(os = 'Windows'
AND channel = 'release', 10, 1) AS sample_mult
IF(os = 'Windows' AND channel = 'release', 10, 1) AS sample_mult
FROM
clients_scalar_aggregates_v1
WHERE

101
sql/moz-fx-data-shared-prod/telemetry_derived/histogram_percentiles_v1/query.sql
Просмотреть файл

@ -1,79 +1,26 @@
SELECT
* EXCEPT (aggregates,
non_norm_aggregates) REPLACE('percentiles' AS agg_type),
ARRAY<STRUCT<KEY STRING,
value FLOAT64>>[ ('0.1',
mozfun.glam.percentile(0.1,
aggregates,
metric_type)),
('1',
mozfun.glam.percentile(1,
aggregates,
metric_type)),
('5',
mozfun.glam.percentile(5,
aggregates,
metric_type)),
('25',
mozfun.glam.percentile(25,
aggregates,
metric_type)),
('50',
mozfun.glam.percentile(50,
aggregates,
metric_type)),
('75',
mozfun.glam.percentile(75,
aggregates,
metric_type)),
('95',
mozfun.glam.percentile(95,
aggregates,
metric_type)),
('99',
mozfun.glam.percentile(99,
aggregates,
metric_type)),
('99.9',
mozfun.glam.percentile(99.9,
aggregates,
metric_type)) ] AS aggregates,
ARRAY<STRUCT<KEY STRING,
value FLOAT64>>[ ('0.1',
mozfun.glam.percentile(0.1,
non_norm_aggregates,
metric_type)),
('1',
mozfun.glam.percentile(1,
non_norm_aggregates,
metric_type)),
('5',
mozfun.glam.percentile(5,
non_norm_aggregates,
metric_type)),
('25',
mozfun.glam.percentile(25,
non_norm_aggregates,
metric_type)),
('50',
mozfun.glam.percentile(50,
non_norm_aggregates,
metric_type)),
('75',
mozfun.glam.percentile(75,
non_norm_aggregates,
metric_type)),
('95',
mozfun.glam.percentile(95,
non_norm_aggregates,
metric_type)),
('99',
mozfun.glam.percentile(99,
non_norm_aggregates,
metric_type)),
('99.9',
mozfun.glam.percentile(99.9,
non_norm_aggregates,
metric_type)) ] AS non_norm_aggregates
* EXCEPT (aggregates, non_norm_aggregates) REPLACE('percentiles' AS agg_type),
ARRAY<STRUCT<KEY STRING, value FLOAT64>>[
('0.1', mozfun.glam.percentile(0.1, aggregates, metric_type)),
('1', mozfun.glam.percentile(1, aggregates, metric_type)),
('5', mozfun.glam.percentile(5, aggregates, metric_type)),
('25', mozfun.glam.percentile(25, aggregates, metric_type)),
('50', mozfun.glam.percentile(50, aggregates, metric_type)),
('75', mozfun.glam.percentile(75, aggregates, metric_type)),
('95', mozfun.glam.percentile(95, aggregates, metric_type)),
('99', mozfun.glam.percentile(99, aggregates, metric_type)),
('99.9', mozfun.glam.percentile(99.9, aggregates, metric_type))
] AS aggregates,
ARRAY<STRUCT<KEY STRING, value FLOAT64>>[
('0.1', mozfun.glam.percentile(0.1, non_norm_aggregates, metric_type)),
('1', mozfun.glam.percentile(1, non_norm_aggregates, metric_type)),
('5', mozfun.glam.percentile(5, non_norm_aggregates, metric_type)),
('25', mozfun.glam.percentile(25, non_norm_aggregates, metric_type)),
('50', mozfun.glam.percentile(50, non_norm_aggregates, metric_type)),
('75', mozfun.glam.percentile(75, non_norm_aggregates, metric_type)),
('95', mozfun.glam.percentile(95, non_norm_aggregates, metric_type)),
('99', mozfun.glam.percentile(99, non_norm_aggregates, metric_type)),
('99.9', mozfun.glam.percentile(99.9, non_norm_aggregates, metric_type))
] AS non_norm_aggregates
FROM
clients_histogram_probe_counts_v1
clients_histogram_probe_counts_v1

11
sql/moz-fx-data-shared-prod/telemetry_derived/newtab_interactions_v1/query.sql
Просмотреть файл

@ -20,7 +20,16 @@ WITH events_unnested AS (
WHERE
DATE(submission_timestamp) = @submission_date
AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket')
AND name IN ('closed', 'opened', 'impression', 'issued', 'click', 'save', 'topic_click', 'dismiss')
AND name IN (
'closed',
'opened',
'impression',
'issued',
'click',
'save',
'topic_click',
'dismiss'
)
),
categorized_events AS (
SELECT

11
sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql
Просмотреть файл

@ -20,7 +20,16 @@ WITH events_unnested AS (
WHERE
DATE(submission_timestamp) = @submission_date
AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket')
AND name IN ('closed', 'opened', 'impression', 'issued', 'click', 'save', 'topic_click', 'dismiss')
AND name IN (
'closed',
'opened',
'impression',
'issued',
'click',
'save',
'topic_click',
'dismiss'
)
),
visit_metadata AS (
SELECT

1
sql/moz-fx-data-shared-prod/telemetry_derived/ssl_ratios_v1/checks.sql
Просмотреть файл

@ -9,3 +9,4 @@
#fail
{{ in_range(["non_ssl_loads", "ssl_loads", "reporting_ratio"], 0, none, "submission_date = @submission_date") }}

100
sql/moz-fx-data-shared-prod/telemetry_derived/unified_metrics_v1/checks.sql
Просмотреть файл

@ -9,7 +9,6 @@
#fail
{{ value_length(column="client_id", expected_length=36, where="submission_date = @submission_date") }}
{#
-- Commented out due to upstream duplication issue inside Fenix data
-- which will cause this check to fail, see: bug(1803609).
@ -38,56 +37,63 @@
{{ value_length(column="country", expected_length=2, where="submission_date = @submission_date") }}
#warn
SELECT IF(
COUNTIF(normalized_app_name NOT IN (
"Firefox Desktop",
"Firefox iOS",
"Firefox iOS BrowserStack",
"Focus iOS",
"Focus iOS BrowserStack",
"Fenix",
"Fenix BrowserStack",
"Focus Android",
"Focus Android Glean",
"Focus Android Glean BrowserStack",
"Klar iOS"
)) > 0,
ERROR("Unexpected values for field normalized_app_name detected."),
null
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE submission_date = @submission_date;
SELECT
IF(
COUNTIF(
normalized_app_name NOT IN (
"Firefox Desktop",
"Firefox iOS",
"Firefox iOS BrowserStack",
"Focus iOS",
"Focus iOS BrowserStack",
"Fenix",
"Fenix BrowserStack",
"Focus Android",
"Focus Android Glean",
"Focus Android Glean BrowserStack",
"Klar iOS"
)
) > 0,
ERROR("Unexpected values for field normalized_app_name detected."),
NULL
)
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date;
#warn
SELECT IF(
COUNTIF(activity_segment NOT IN (
"casual_user",
"regular_user",
"infrequent_user",
"other",
"core_user"
)) > 0,
ERROR("Unexpected values for field activity_segment detected."),
null
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE submission_date = @submission_date;
SELECT
IF(
COUNTIF(
activity_segment NOT IN (
"casual_user",
"regular_user",
"infrequent_user",
"other",
"core_user"
)
) > 0,
ERROR("Unexpected values for field activity_segment detected."),
NULL
)
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date;
#warn
SELECT IF(
COUNTIF(normalized_channel NOT IN (
"nightly",
"aurora",
"release",
"Other",
"beta",
"esr"
)) > 0,
ERROR("Unexpected values for field normalized_channel detected."),
null
)
FROM `{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE submission_date = @submission_date;
SELECT
IF(
COUNTIF(normalized_channel NOT IN ("nightly", "aurora", "release", "Other", "beta", "esr")) > 0,
ERROR("Unexpected values for field normalized_channel detected."),
NULL
)
FROM
`{{ project_id }}.{{ dataset_id }}.{{ table_name }}`
WHERE
submission_date = @submission_date;
#warn
{{ matches_pattern(column="country", pattern="^[A-Z]{2}$", where="submission_date = @submission_date", message="Some values in this field do not adhere to the ISO 3166-1 specification (2 character country code, for example: DE).") }}

4
sql/moz-fx-data-shared-prod/udf/aggregate_map_first/udf.sql
Просмотреть файл

@ -11,9 +11,7 @@ CREATE OR REPLACE FUNCTION udf.aggregate_map_first(maps ANY TYPE) AS (
(
SELECT
* EXCEPT (value),
FIRST_VALUE(
value IGNORE NULLS
) --
FIRST_VALUE(value IGNORE NULLS) --
OVER (
PARTITION BY
key

4
sql/moz-fx-data-shared-prod/udf/country_code_to_flag/udf.sql
Просмотреть файл

@ -12,9 +12,7 @@ See:
- https://en.wikipedia.org/wiki/Regional_Indicator_Symbol
*/
CREATE OR REPLACE FUNCTION udf.country_code_to_flag(
country_code string
) AS ( --
CREATE OR REPLACE FUNCTION udf.country_code_to_flag(country_code string) AS ( --
CODE_POINTS_TO_STRING(
ARRAY(
SELECT

28
sql/moz-fx-data-shared-prod/udf/parse_desktop_telemetry_uri/udf.sql
Просмотреть файл

@ -19,27 +19,13 @@ RETURNS STRUCT<
CASE
WHEN ARRAY_LENGTH(REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")) = 8
THEN STRUCT(
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[
OFFSET(1)
], --namespace
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[
OFFSET(2)
], --document_id
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[
OFFSET(3)
], --document_type
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[
OFFSET(4)
], --app_name
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[
OFFSET(5)
], --app_version
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[
OFFSET(6)
], --app_update_channel
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[
OFFSET(7)
] --app_build_id
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[OFFSET(1)], --namespace
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[OFFSET(2)], --document_id
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[OFFSET(3)], --document_type
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[OFFSET(4)], --app_name
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[OFFSET(5)], --app_version
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[OFFSET(6)], --app_update_channel
REGEXP_EXTRACT_ALL(uri, r"/([a-zA-Z0-9_.+-]+)")[OFFSET(7)] --app_build_id
)
ELSE NULL
END

11
sql/mozfun/ltv/android_states_with_paid_v1/udf.sql
Просмотреть файл

@ -65,7 +65,16 @@ RETURNS STRING AS (
SELECT
assert.equals(
"0_dow1_organic_1_1",
ltv.android_states_with_paid_v1("abc", 0, DATE("2023-01-01"), DATE("2023-01-01"), 1, 1, 28, "US")
ltv.android_states_with_paid_v1(
"abc",
0,
DATE("2023-01-01"),
DATE("2023-01-01"),
1,
1,
28,
"US"
)
),
assert.equals(
"1_dow2_paid_1_0",

13
sql/mozfun/ltv/android_states_with_paid_v2/udf.sql
Просмотреть файл

@ -71,7 +71,18 @@ RETURNS STRING AS (
SELECT
assert.equals(
"0_dow1_organic_1_1",
ltv.android_states_with_paid_v2("abc", 0, 0, 100, DATE("2023-01-01"), DATE("2023-01-01"), 1, 1, 28, "US")
ltv.android_states_with_paid_v2(
"abc",
0,
0,
100,
DATE("2023-01-01"),
DATE("2023-01-01"),
1,
1,
28,
"US"
)
),
assert.equals(
"1_dow2_paid_1_0",

138
sql/mozfun/mobile_search/normalize_app_name/udf.sql
Просмотреть файл

@ -1,46 +1,108 @@
CREATE OR REPLACE FUNCTION mobile_search.normalize_app_name(app_name STRING, os STRING)
RETURNS STRUCT<normalized_app_name STRING, normalized_app_name_os STRING>
AS
(
RETURNS STRUCT<normalized_app_name STRING, normalized_app_name_os STRING> AS (
STRUCT(
CASE
WHEN app_name = 'Fenix' AND os = 'Android' THEN 'Firefox'
WHEN app_name = 'Fennec' AND os = 'Other' THEN 'Fennec'
WHEN app_name = 'Fennec' AND os = 'Android' THEN 'Fennec'
WHEN app_name = 'Fennec' AND os = 'iOS' THEN 'Firefox'
WHEN app_name = 'Firefox Preview' AND os = 'Android' THEN 'Firefox Preview'
WHEN app_name = 'FirefoxConnect' AND os = 'Android' THEN 'Firefox for Echo Show'
WHEN app_name = 'FirefoxForFireTV' AND os = 'Android' THEN 'Firefox for FireTV'
WHEN app_name = 'Focus Android Glean' AND os = 'Android' THEN 'Focus'
WHEN app_name = 'Focus iOS Glean' AND os = 'iOS' THEN 'Focus'
WHEN app_name = 'Klar Android Glean' AND os = 'Android' THEN 'Klar'
WHEN app_name = 'Klar iOS Glean' AND os = 'iOS' THEN 'Klar'
WHEN app_name = 'Other' AND os = 'iOS' THEN 'Other'
WHEN app_name = 'Other' AND os = 'Other' THEN 'Other'
WHEN app_name = 'Other' AND os = 'Android' THEN 'Other'
WHEN app_name = 'Zerda' AND os = 'Android' THEN 'Firefox Lite'
WHEN app_name = 'Zerda_cn' AND os = 'Android' THEN 'Firefox Lite (China)'
WHEN app_name = 'Fenix'
AND os = 'Android'
THEN 'Firefox'
WHEN app_name = 'Fennec'
AND os = 'Other'
THEN 'Fennec'
WHEN app_name = 'Fennec'
AND os = 'Android'
THEN 'Fennec'
WHEN app_name = 'Fennec'
AND os = 'iOS'
THEN 'Firefox'
WHEN app_name = 'Firefox Preview'
AND os = 'Android'
THEN 'Firefox Preview'
WHEN app_name = 'FirefoxConnect'
AND os = 'Android'
THEN 'Firefox for Echo Show'
WHEN app_name = 'FirefoxForFireTV'
AND os = 'Android'
THEN 'Firefox for FireTV'
WHEN app_name = 'Focus Android Glean'
AND os = 'Android'
THEN 'Focus'
WHEN app_name = 'Focus iOS Glean'
AND os = 'iOS'
THEN 'Focus'
WHEN app_name = 'Klar Android Glean'
AND os = 'Android'
THEN 'Klar'
WHEN app_name = 'Klar iOS Glean'
AND os = 'iOS'
THEN 'Klar'
WHEN app_name = 'Other'
AND os = 'iOS'
THEN 'Other'
WHEN app_name = 'Other'
AND os = 'Other'
THEN 'Other'
WHEN app_name = 'Other'
AND os = 'Android'
THEN 'Other'
WHEN app_name = 'Zerda'
AND os = 'Android'
THEN 'Firefox Lite'
WHEN app_name = 'Zerda_cn'
AND os = 'Android'
THEN 'Firefox Lite (China)'
ELSE NULL
END AS normalized_app_name,
END AS normalized_app_name,
CASE
WHEN app_name = 'Fenix' AND os = 'Android' THEN 'Firefox Android'
WHEN app_name = 'Fennec' AND os = 'Other' THEN 'Fennec Other'
WHEN app_name = 'Fennec' AND os = 'Android' THEN 'Legacy Firefox Android'
WHEN app_name = 'Fennec' AND os = 'iOS' THEN 'Firefox iOS'
WHEN app_name = 'Firefox Preview' AND os = 'Android' THEN 'Firefox Preview'
WHEN app_name = 'FirefoxConnect' AND os = 'Android' THEN 'Firefox for Echo Show'
WHEN app_name = 'FirefoxForFireTV' AND os = 'Android' THEN 'Firefox for FireTV'
WHEN app_name = 'Focus Android Glean' AND os = 'Android' THEN 'Focus Android'
WHEN app_name = 'Focus iOS Glean' AND os = 'iOS' THEN 'Focus iOS'
WHEN app_name = 'Klar Android Glean' AND os = 'Android' THEN 'Klar Android'
WHEN app_name = 'Klar iOS Glean' AND os = 'iOS' THEN 'Klar iOS'
WHEN app_name = 'Other' AND os = 'iOS' THEN 'Other iOS'
WHEN app_name = 'Other' AND os = 'Other' THEN 'Other'
WHEN app_name = 'Other' AND os = 'Android' THEN 'Other Android'
WHEN app_name = 'Zerda' AND os = 'Android' THEN 'Firefox Lite Android'
WHEN app_name = 'Zerda_cn' AND os = 'Android' THEN 'Firefox Lite Android (China)'
WHEN app_name = 'Fenix'
AND os = 'Android'
THEN 'Firefox Android'
WHEN app_name = 'Fennec'
AND os = 'Other'
THEN 'Fennec Other'
WHEN app_name = 'Fennec'
AND os = 'Android'
THEN 'Legacy Firefox Android'
WHEN app_name = 'Fennec'
AND os = 'iOS'
THEN 'Firefox iOS'
WHEN app_name = 'Firefox Preview'
AND os = 'Android'
THEN 'Firefox Preview'
WHEN app_name = 'FirefoxConnect'
AND os = 'Android'
THEN 'Firefox for Echo Show'
WHEN app_name = 'FirefoxForFireTV'
AND os = 'Android'
THEN 'Firefox for FireTV'
WHEN app_name = 'Focus Android Glean'
AND os = 'Android'
THEN 'Focus Android'
WHEN app_name = 'Focus iOS Glean'
AND os = 'iOS'
THEN 'Focus iOS'
WHEN app_name = 'Klar Android Glean'
AND os = 'Android'
THEN 'Klar Android'
WHEN app_name = 'Klar iOS Glean'
AND os = 'iOS'
THEN 'Klar iOS'
WHEN app_name = 'Other'
AND os = 'iOS'
THEN 'Other iOS'
WHEN app_name = 'Other'
AND os = 'Other'
THEN 'Other'
WHEN app_name = 'Other'
AND os = 'Android'
THEN 'Other Android'
WHEN app_name = 'Zerda'
AND os = 'Android'
THEN 'Firefox Lite Android'
WHEN app_name = 'Zerda_cn'
AND os = 'Android'
THEN 'Firefox Lite Android (China)'
ELSE NULL
END AS normalized_app_name_os
END AS normalized_app_name_os
)
);
@ -53,4 +115,4 @@ SELECT
assert.equals(
STRUCT('Firefox Lite' AS normalized_app_name, 'Firefox Lite Android' AS normalized_app_name_os),
mobile_search.normalize_app_name('Zerda', 'Android')
),
),

4
sql/mozfun/norm/fenix_build_to_datetime/udf.sql
Просмотреть файл

@ -7,9 +7,7 @@ CREATE OR REPLACE FUNCTION norm.fenix_build_to_datetime(app_build STRING) AS (
-- day of year (%j) or the custom single-character year used here.
THEN DATETIME_ADD(
DATETIME(
2018 + SAFE_CAST(
SUBSTR(app_build, 1, 1) AS INT64
), -- year
2018 + SAFE_CAST(SUBSTR(app_build, 1, 1) AS INT64), -- year
1, -- placeholder month
1, -- placeholder year
SAFE_CAST(SUBSTR(app_build, 5, 2) AS INT64),

77
sql/mozfun/norm/result_type_to_product_name/udf.sql
Просмотреть файл

@ -1,31 +1,58 @@
CREATE OR REPLACE FUNCTION norm.result_type_to_product_name(
res STRING
)
CREATE OR REPLACE FUNCTION norm.result_type_to_product_name(res STRING)
RETURNS STRING AS (
CASE
WHEN res IN ('autofill_origin', 'autofill_url') THEN 'autofill'
WHEN res IN ('addon') THEN 'xchannels_add_on'
WHEN res IN ('rs_amo', 'rust_amo') THEN 'suggest_add_on'
WHEN res IN ('search_suggest', 'search_history', 'search_suggest_rich') THEN 'default_partner_search_suggestion'
WHEN res IN ('search_engine') THEN 'search_engine'
WHEN res IN ('trending_search', 'trending_search_rich') THEN 'trending_suggestion'
WHEN res IN ('history') THEN 'history'
WHEN res IN ('bookmark', 'keyword') THEN 'bookmark'
WHEN res IN ('tab') THEN 'open_tab'
WHEN res IN ('merino_adm_sponsored', 'rs_adm_sponsored', 'suggest_sponsor', 'rust_adm_sponsored') THEN 'admarketplace_sponsored'
WHEN res IN ('merino_top_picks') THEN 'navigational'
WHEN res IN ('rs_adm_nonsponsored',
'merino_adm_nonsponsored',
'suggest_non_sponsor',
'rust_adm_nonsponsored') THEN 'wikipedia_enhanced'
WHEN res IN ('dynamic_wikipedia', 'merino_wikipedia') THEN 'wikipedia_dynamic'
WHEN res IN ('weather') THEN 'weather'
WHEN res IN ( 'action', 'intervention_clear', 'intervention_refresh', 'intervention_unknown', 'intervention_update' ) THEN 'quick_action'
WHEN res IN ('rs_pocket', 'rust_pocket') THEN 'pocket_collection'
CASE
WHEN res IN ('autofill_origin', 'autofill_url')
THEN 'autofill'
WHEN res IN ('addon')
THEN 'xchannels_add_on'
WHEN res IN ('rs_amo', 'rust_amo')
THEN 'suggest_add_on'
WHEN res IN ('search_suggest', 'search_history', 'search_suggest_rich')
THEN 'default_partner_search_suggestion'
WHEN res IN ('search_engine')
THEN 'search_engine'
WHEN res IN ('trending_search', 'trending_search_rich')
THEN 'trending_suggestion'
WHEN res IN ('history')
THEN 'history'
WHEN res IN ('bookmark', 'keyword')
THEN 'bookmark'
WHEN res IN ('tab')
THEN 'open_tab'
WHEN res IN (
'merino_adm_sponsored',
'rs_adm_sponsored',
'suggest_sponsor',
'rust_adm_sponsored'
)
THEN 'admarketplace_sponsored'
WHEN res IN ('merino_top_picks')
THEN 'navigational'
WHEN res IN (
'rs_adm_nonsponsored',
'merino_adm_nonsponsored',
'suggest_non_sponsor',
'rust_adm_nonsponsored'
)
THEN 'wikipedia_enhanced'
WHEN res IN ('dynamic_wikipedia', 'merino_wikipedia')
THEN 'wikipedia_dynamic'
WHEN res IN ('weather')
THEN 'weather'
WHEN res IN (
'action',
'intervention_clear',
'intervention_refresh',
'intervention_unknown',
'intervention_update'
)
THEN 'quick_action'
WHEN res IN ('rs_pocket', 'rust_pocket')
THEN 'pocket_collection'
ELSE 'other'
END
END
);
-- Tests
-- Tests
SELECT
assert.equals("other", norm.result_type_to_product_name("not a valid type"))

5
tests/format_sql/jinja_example/expect.sql
Просмотреть файл

@ -1,4 +1,5 @@
{% set options = ["a", "b", "c"] %}{# sample comment #}
{% set options = ["a", "b", "c"] %}
{# sample comment #}
SELECT
{% for option in options %}
{% if option == "a" %}
@ -7,5 +8,5 @@ SELECT
"{{ option }}" AS {{ option }},
{% endif %}
{% endfor %}
test,{# another comment #}
test, {# another comment #}
foo