Fix empty result sets for incremental core clients first seen table (#2057)

* Add tests for core_clients_first_seen_init

* Add failing test for core clients first seen

* Fix issues with core clients first seen

* Keep left join

* Update sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_first_seen_v1/query.sql

Co-authored-by: Jeff Klukas <jklukas@mozilla.com>

Co-authored-by: Jeff Klukas <jklukas@mozilla.com>
This commit is contained in:
Anthony Miyaguchi 2021-05-24 14:11:11 -07:00 коммит произвёл GitHub
Родитель c6f0c3ce81
Коммит 5b362e289f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
11 изменённых файлов: 913 добавлений и 6 удалений

Просмотреть файл

@ -1,5 +1,5 @@
CREATE TABLE
`moz-fx-data-shared-prod.telemetry_derived.core_clients_first_seen_v1`
`telemetry_derived.core_clients_first_seen_v1`
PARTITION BY
(first_seen_date)
AS

Просмотреть файл

@ -1,20 +1,29 @@
WITH clients_today AS (
WITH _current AS (
SELECT DISTINCT
client_id
FROM
telemetry.core
WHERE
DATE(submission_timestamp) = @submission_date
),
_previous AS (
SELECT
*
FROM
telemetry_derived.core_clients_first_seen_v1
WHERE
-- In the case we need to backfill older partitions of this table, we don't want newer partitions
-- to alter results of the query.
first_seen_date < @submission_date
)
SELECT
client_id,
@submission_date AS first_seen_date
FROM
clients_today
_current
LEFT JOIN
telemetry_derived.core_clients_first_seen_v1 AS cfs
_previous
USING
(client_id)
WHERE
cfs.first_seen_date < @submission_date
AND cfs.client_id IS NULL
_previous.client_id IS NULL

Просмотреть файл

@ -0,0 +1,3 @@
---
- client_id: new-instance
first_seen_date: 2021-03-02

Просмотреть файл

@ -0,0 +1,4 @@
---
- name: submission_date
type: DATE
value: 2021-03-02

Просмотреть файл

@ -0,0 +1,427 @@
[
{
"mode": "REPEATED",
"name": "accessibility_services",
"type": "STRING"
},
{
"name": "additional_properties",
"type": "STRING"
},
{
"name": "arch",
"type": "STRING"
},
{
"name": "bug_1501329_affected",
"type": "BOOLEAN"
},
{
"name": "campaign",
"type": "STRING"
},
{
"name": "client_id",
"type": "STRING"
},
{
"name": "created",
"type": "STRING"
},
{
"name": "default_browser",
"type": "BOOLEAN"
},
{
"name": "default_search",
"type": "STRING"
},
{
"name": "device",
"type": "STRING"
},
{
"name": "display_version",
"type": "STRING"
},
{
"name": "distribution_id",
"type": "STRING"
},
{
"name": "document_id",
"type": "STRING"
},
{
"name": "durations",
"type": "INTEGER"
},
{
"mode": "REPEATED",
"name": "experiments",
"type": "STRING"
},
{
"name": "flash_usage",
"type": "INTEGER"
},
{
"name": "locale",
"type": "STRING"
},
{
"fields": [
{
"fields": [
{
"name": "city",
"type": "STRING"
},
{
"name": "country",
"type": "STRING"
},
{
"name": "db_version",
"type": "STRING"
},
{
"name": "subdivision1",
"type": "STRING"
},
{
"name": "subdivision2",
"type": "STRING"
}
],
"name": "geo",
"type": "RECORD"
},
{
"fields": [
{
"name": "date",
"type": "STRING"
},
{
"name": "dnt",
"type": "STRING"
},
{
"name": "x_debug_id",
"type": "STRING"
},
{
"name": "x_pingsender_version",
"type": "STRING"
}
],
"name": "header",
"type": "RECORD"
},
{
"fields": [
{
"name": "app_build_id",
"type": "STRING"
},
{
"name": "app_name",
"type": "STRING"
},
{
"name": "app_update_channel",
"type": "STRING"
},
{
"name": "app_version",
"type": "STRING"
}
],
"name": "uri",
"type": "RECORD"
},
{
"fields": [
{
"name": "browser",
"type": "STRING"
},
{
"name": "os",
"type": "STRING"
},
{
"name": "version",
"type": "STRING"
}
],
"name": "user_agent",
"type": "RECORD"
}
],
"name": "metadata",
"type": "RECORD"
},
{
"name": "normalized_app_name",
"type": "STRING"
},
{
"name": "normalized_channel",
"type": "STRING"
},
{
"name": "normalized_country_code",
"type": "STRING"
},
{
"name": "normalized_os",
"type": "STRING"
},
{
"name": "normalized_os_version",
"type": "STRING"
},
{
"name": "os",
"type": "STRING"
},
{
"name": "osversion",
"type": "STRING"
},
{
"name": "profile_date",
"type": "INTEGER"
},
{
"name": "sample_id",
"type": "INTEGER"
},
{
"fields": [
{
"name": "key",
"type": "STRING"
},
{
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "searches",
"type": "RECORD"
},
{
"name": "seq",
"type": "INTEGER"
},
{
"name": "sessions",
"type": "INTEGER"
},
{
"name": "show_tracker_stats_share",
"type": "BOOLEAN"
},
{
"name": "submission_timestamp",
"type": "TIMESTAMP"
},
{
"name": "tz",
"type": "INTEGER"
},
{
"name": "v",
"type": "INTEGER"
},
{
"name": "campaign_id",
"type": "STRING"
},
{
"fields": [
{
"fields": [
{
"mode": "REPEATED",
"name": "active",
"type": "STRING"
},
{
"mode": "REPEATED",
"name": "disabled",
"type": "STRING"
}
],
"name": "addons",
"type": "RECORD"
},
{
"fields": [
{
"name": "pocket_stories_clicked",
"type": "INTEGER"
},
{
"name": "top_sites_clicked",
"type": "INTEGER"
}
],
"name": "new_tab",
"type": "RECORD"
},
{
"fields": [
{
"name": "bookmark_with_star",
"type": "INTEGER"
},
{
"name": "print",
"type": "INTEGER"
},
{
"name": "save_as_pdf",
"type": "INTEGER"
},
{
"name": "total_added_search_engines",
"type": "INTEGER"
},
{
"name": "total_sites_pinned_to_topsites",
"type": "INTEGER"
},
{
"name": "view_source",
"type": "INTEGER"
}
],
"name": "page_options",
"type": "RECORD"
},
{
"fields": [
{
"name": "restore_tabs",
"type": "BOOLEAN"
},
{
"name": "show_images",
"type": "STRING"
},
{
"name": "show_web_fonts",
"type": "BOOLEAN"
}
],
"name": "settings_advanced",
"type": "RECORD"
},
{
"fields": [
{
"name": "compact_tabs",
"type": "BOOLEAN"
},
{
"name": "full_screen_browsing",
"type": "BOOLEAN"
},
{
"fields": [
{
"name": "bookmarks_enabled",
"type": "BOOLEAN"
},
{
"name": "custom_homepage",
"type": "BOOLEAN"
},
{
"name": "custom_homepage_use_for_newtab",
"type": "BOOLEAN"
},
{
"name": "history_enabled",
"type": "BOOLEAN"
},
{
"name": "pocket_enabled",
"type": "BOOLEAN"
},
{
"name": "recent_bookmarks_enabled",
"type": "BOOLEAN"
},
{
"name": "topsites_enabled",
"type": "BOOLEAN"
},
{
"name": "visited_enabled",
"type": "BOOLEAN"
}
],
"name": "homepage",
"type": "RECORD"
},
{
"name": "tab_queue",
"type": "BOOLEAN"
},
{
"name": "tab_queue_usage_count",
"type": "INTEGER"
}
],
"name": "settings_general",
"type": "RECORD"
},
{
"fields": [
{
"name": "product_feature_tips",
"type": "BOOLEAN"
}
],
"name": "settings_notifications",
"type": "RECORD"
},
{
"fields": [
{
"name": "do_not_track",
"type": "BOOLEAN"
},
{
"name": "master_password",
"type": "BOOLEAN"
},
{
"name": "master_password_usage_count",
"type": "INTEGER"
}
],
"name": "settings_privacy",
"type": "RECORD"
},
{
"fields": [
{
"name": "only_over_wifi",
"type": "BOOLEAN"
}
],
"name": "sync",
"type": "RECORD"
}
],
"name": "fennec",
"type": "RECORD"
}
]

Просмотреть файл

@ -0,0 +1,7 @@
---
- sample_id: 0
client_id: new-instance
submission_timestamp: 2021-03-02T00:00:00
- sample_id: 0
client_id: existing-instance-with-new-ping
submission_timestamp: 2021-03-02T00:00:00

Просмотреть файл

@ -0,0 +1,10 @@
[
{
"type": "STRING",
"name": "client_id"
},
{
"type": "DATE",
"name": "first_seen_date"
}
]

Просмотреть файл

@ -0,0 +1,5 @@
---
- client_id: existing-instance
first_seen_date: 2021-03-01
- client_id: existing-instance-with-new-ping
first_seen_date: 2021-03-01

Просмотреть файл

@ -0,0 +1,5 @@
---
- client_id: single-instance
first_seen_date: 2021-03-01
- client_id: multiple-instances
first_seen_date: 2021-03-01

Просмотреть файл

@ -0,0 +1,427 @@
[
{
"mode": "REPEATED",
"name": "accessibility_services",
"type": "STRING"
},
{
"name": "additional_properties",
"type": "STRING"
},
{
"name": "arch",
"type": "STRING"
},
{
"name": "bug_1501329_affected",
"type": "BOOLEAN"
},
{
"name": "campaign",
"type": "STRING"
},
{
"name": "client_id",
"type": "STRING"
},
{
"name": "created",
"type": "STRING"
},
{
"name": "default_browser",
"type": "BOOLEAN"
},
{
"name": "default_search",
"type": "STRING"
},
{
"name": "device",
"type": "STRING"
},
{
"name": "display_version",
"type": "STRING"
},
{
"name": "distribution_id",
"type": "STRING"
},
{
"name": "document_id",
"type": "STRING"
},
{
"name": "durations",
"type": "INTEGER"
},
{
"mode": "REPEATED",
"name": "experiments",
"type": "STRING"
},
{
"name": "flash_usage",
"type": "INTEGER"
},
{
"name": "locale",
"type": "STRING"
},
{
"fields": [
{
"fields": [
{
"name": "city",
"type": "STRING"
},
{
"name": "country",
"type": "STRING"
},
{
"name": "db_version",
"type": "STRING"
},
{
"name": "subdivision1",
"type": "STRING"
},
{
"name": "subdivision2",
"type": "STRING"
}
],
"name": "geo",
"type": "RECORD"
},
{
"fields": [
{
"name": "date",
"type": "STRING"
},
{
"name": "dnt",
"type": "STRING"
},
{
"name": "x_debug_id",
"type": "STRING"
},
{
"name": "x_pingsender_version",
"type": "STRING"
}
],
"name": "header",
"type": "RECORD"
},
{
"fields": [
{
"name": "app_build_id",
"type": "STRING"
},
{
"name": "app_name",
"type": "STRING"
},
{
"name": "app_update_channel",
"type": "STRING"
},
{
"name": "app_version",
"type": "STRING"
}
],
"name": "uri",
"type": "RECORD"
},
{
"fields": [
{
"name": "browser",
"type": "STRING"
},
{
"name": "os",
"type": "STRING"
},
{
"name": "version",
"type": "STRING"
}
],
"name": "user_agent",
"type": "RECORD"
}
],
"name": "metadata",
"type": "RECORD"
},
{
"name": "normalized_app_name",
"type": "STRING"
},
{
"name": "normalized_channel",
"type": "STRING"
},
{
"name": "normalized_country_code",
"type": "STRING"
},
{
"name": "normalized_os",
"type": "STRING"
},
{
"name": "normalized_os_version",
"type": "STRING"
},
{
"name": "os",
"type": "STRING"
},
{
"name": "osversion",
"type": "STRING"
},
{
"name": "profile_date",
"type": "INTEGER"
},
{
"name": "sample_id",
"type": "INTEGER"
},
{
"fields": [
{
"name": "key",
"type": "STRING"
},
{
"name": "value",
"type": "INTEGER"
}
],
"mode": "REPEATED",
"name": "searches",
"type": "RECORD"
},
{
"name": "seq",
"type": "INTEGER"
},
{
"name": "sessions",
"type": "INTEGER"
},
{
"name": "show_tracker_stats_share",
"type": "BOOLEAN"
},
{
"name": "submission_timestamp",
"type": "TIMESTAMP"
},
{
"name": "tz",
"type": "INTEGER"
},
{
"name": "v",
"type": "INTEGER"
},
{
"name": "campaign_id",
"type": "STRING"
},
{
"fields": [
{
"fields": [
{
"mode": "REPEATED",
"name": "active",
"type": "STRING"
},
{
"mode": "REPEATED",
"name": "disabled",
"type": "STRING"
}
],
"name": "addons",
"type": "RECORD"
},
{
"fields": [
{
"name": "pocket_stories_clicked",
"type": "INTEGER"
},
{
"name": "top_sites_clicked",
"type": "INTEGER"
}
],
"name": "new_tab",
"type": "RECORD"
},
{
"fields": [
{
"name": "bookmark_with_star",
"type": "INTEGER"
},
{
"name": "print",
"type": "INTEGER"
},
{
"name": "save_as_pdf",
"type": "INTEGER"
},
{
"name": "total_added_search_engines",
"type": "INTEGER"
},
{
"name": "total_sites_pinned_to_topsites",
"type": "INTEGER"
},
{
"name": "view_source",
"type": "INTEGER"
}
],
"name": "page_options",
"type": "RECORD"
},
{
"fields": [
{
"name": "restore_tabs",
"type": "BOOLEAN"
},
{
"name": "show_images",
"type": "STRING"
},
{
"name": "show_web_fonts",
"type": "BOOLEAN"
}
],
"name": "settings_advanced",
"type": "RECORD"
},
{
"fields": [
{
"name": "compact_tabs",
"type": "BOOLEAN"
},
{
"name": "full_screen_browsing",
"type": "BOOLEAN"
},
{
"fields": [
{
"name": "bookmarks_enabled",
"type": "BOOLEAN"
},
{
"name": "custom_homepage",
"type": "BOOLEAN"
},
{
"name": "custom_homepage_use_for_newtab",
"type": "BOOLEAN"
},
{
"name": "history_enabled",
"type": "BOOLEAN"
},
{
"name": "pocket_enabled",
"type": "BOOLEAN"
},
{
"name": "recent_bookmarks_enabled",
"type": "BOOLEAN"
},
{
"name": "topsites_enabled",
"type": "BOOLEAN"
},
{
"name": "visited_enabled",
"type": "BOOLEAN"
}
],
"name": "homepage",
"type": "RECORD"
},
{
"name": "tab_queue",
"type": "BOOLEAN"
},
{
"name": "tab_queue_usage_count",
"type": "INTEGER"
}
],
"name": "settings_general",
"type": "RECORD"
},
{
"fields": [
{
"name": "product_feature_tips",
"type": "BOOLEAN"
}
],
"name": "settings_notifications",
"type": "RECORD"
},
{
"fields": [
{
"name": "do_not_track",
"type": "BOOLEAN"
},
{
"name": "master_password",
"type": "BOOLEAN"
},
{
"name": "master_password_usage_count",
"type": "INTEGER"
}
],
"name": "settings_privacy",
"type": "RECORD"
},
{
"fields": [
{
"name": "only_over_wifi",
"type": "BOOLEAN"
}
],
"name": "sync",
"type": "RECORD"
}
],
"name": "fennec",
"type": "RECORD"
}
]

Просмотреть файл

@ -0,0 +1,10 @@
---
- sample_id: 0
client_id: single-instance
submission_timestamp: 2021-03-01T00:00:00
- sample_id: 0
client_id: multiple-instances
submission_timestamp: 2021-03-01T00:00:00
- sample_id: 0
client_id: multiple-instances
submission_timestamp: 2021-03-02T00:00:00