From f814117456a751512e67fe93084681bc9c19b518 Mon Sep 17 00:00:00 2001 From: benmiroglio Date: Tue, 4 Jun 2019 10:32:25 -0700 Subject: [PATCH] Breakout install events by source --- TEST.json | 1 - addons_daily/utils/telemetry_data.py | 73 +++++++++++++++++++------ tests/resources/expected_results.json | 2 +- tests/test_integration.py | 1 - tests/test_telemetry.py | 79 +++++++++++++++++++++++---- 5 files changed, 126 insertions(+), 30 deletions(-) delete mode 100644 TEST.json diff --git a/TEST.json b/TEST.json deleted file mode 100644 index daf7556..0000000 --- a/TEST.json +++ /dev/null @@ -1 +0,0 @@ -[{'addon_id': 'baidu-code-update@mozillaonline.com', 'name': 'Baidu Search Update', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 1, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': None, 'uninstalls': None, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': None, 'top_10_coinstalls': {'0': 'non-system-addon2', '1': 'non-system-addon1'}, 'avg_webext_background_page_load_ms_': None, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}, {'addon_id': 'screenshots@mozilla.org', 'name': 'Firefox Screenshots', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 2, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': 2, 'uninstalls': 1, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': 4087.0, 'top_10_coinstalls': {'0': 'hotfix-update-xpi-intermediate@mozilla.com', '1': 'non-system-addon1'}, 'avg_webext_background_page_load_ms_': 2172.75, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}, {'addon_id': 'non-system-addon1', 'name': 'Non systemn Addon 1', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 1, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': None, 'uninstalls': None, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': None, 'top_10_coinstalls': {'0': 'non-system-addon1', '1': 'non-system-addon2'}, 'avg_webext_background_page_load_ms_': None, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}, {'addon_id': 'hotfix-update-xpi-intermediate@mozilla.com', 'name': 'hotfix-update-xpi-intermediate', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 1, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': None, 'uninstalls': None, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': None, 'top_10_coinstalls': {'0': 'non-system-addon1', '1': 'non-system-addon2'}, 'avg_webext_background_page_load_ms_': None, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}, {'addon_id': 'fxmonitor@mozilla.org', 'name': 'Firefox Monitor', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 1, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': None, 'uninstalls': 1, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': 4059.75, 'top_10_coinstalls': {'0': 'non-system-addon1', '1': 'hotfix-update-xpi-intermediate@mozilla.com'}, 'avg_webext_background_page_load_ms_': 1969.5, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}, {'addon_id': 'non-system-addon2', 'name': 'Non System Addon 2', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 1, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': None, 'uninstalls': None, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': None, 'top_10_coinstalls': {'0': 'non-system-addon2', '1': 'non-system-addon1'}, 'avg_webext_background_page_load_ms_': None, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}, {'addon_id': 'formautofill@mozilla.org', 'name': 'Form Autofill', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 2, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': None, 'uninstalls': None, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': 4076.5, 'top_10_coinstalls': {'0': 'non-system-addon1', '1': 'non-system-addon2'}, 'avg_webext_background_page_load_ms_': 1958.25, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}, {'addon_id': 'webcompat@mozilla.org', 'name': 'Web Compat', 'os_dist': {'Windows_NT': 1.0}, 'country_dist': {'GB': 1.0}, 'avg_time_total': 747.0, 'active_hours': 0.18194444444444444, 'disabled': None, 'avg_tabs': None, 'avg_bookmarks': None, 'avg_toolbox_opened_count': None, 'avg_uri': 33.0, 'pct_w_tracking_prot_enabled': 0.0, 'mau': 2, 'wau': 1, 'dau': 1, 'dau_prop': 1.0, 'sap_searches': {}, 'tagged_sap_searches': {'google': 10.0}, 'organic_searches': {}, 'search_with_ads': {}, 'ad_click': {}, 'installs': None, 'uninstalls': None, 'avg_webext_storage_local_get_ms_': None, 'avg_webext_storage_local_set_ms_': None, 'avg_webext_extension_startup_ms_': 4059.75, 'top_10_coinstalls': {'0': 'non-system-addon1', '1': 'non-system-addon2'}, 'avg_webext_background_page_load_ms_': 1849.25, 'avg_webext_browseraction_popup_open_ms_': None, 'avg_webext_pageaction_popup_open_ms_': None, 'avg_webext_content_script_injection_ms_': None}] \ No newline at end of file diff --git a/addons_daily/utils/telemetry_data.py b/addons_daily/utils/telemetry_data.py index 4a7c9c8..e1cb0b3 100644 --- a/addons_daily/utils/telemetry_data.py +++ b/addons_daily/utils/telemetry_data.py @@ -4,6 +4,7 @@ import pandas as pd from pyspark.sql import SQLContext, Row from pyspark.sql.window import Window from pyspark.sql.types import StringType +from itertools import chain TOP_COUNTRIES = { @@ -285,9 +286,20 @@ def get_top_addon_names(addons_expanded): def install_flow_events(events): - """ - - """ + def source_map(df, alias): + m = F.create_map( + list( + chain( + *( + (F.lit(name), F.col(name)) + for name in df.columns + if name != "addon_id" + ) + ) + ) + ).alias(alias) + return m + install_flow_events = ( events.select( [ @@ -323,23 +335,52 @@ def install_flow_events(events): ) ) - number_installs = ( - install_flow_events.where(install_flow_events.event_method == "install") - .groupby("addon_id") - .agg(F.sum("n_distinct_users").alias("installs")) + installs = ( + install_flow_events.filter("event_method = 'install'") + .groupBy("addon_id") + .pivot("source") + .agg(F.sum("n_distinct_users")) + ) + uninstalls = ( + install_flow_events.filter("event_method = 'uninstall'") + .groupBy("addon_id") + .pivot("source") + .agg(F.sum("n_distinct_users")) + ) + avg_downloads = install_flow_events.select( + "addon_id", "avg_download_time" + ).distinct() + + flows = ( + installs.na.fill(0) + .select("addon_id", source_map(installs, "installs")) + .join( + uninstalls.na.fill(0).select( + "addon_id", source_map(uninstalls, "uninstalls") + ), + on="addon_id", + how="full", + ) ) - number_uninstalls = ( - install_flow_events.where(install_flow_events.event_method == "uninstall") - .groupby("addon_id") - .agg(F.sum("n_distinct_users").alias("uninstalls")) - ) + return avg_downloads.join(flows, on="addon_id", how="full") + # number_installs = ( + # install_flow_events.where(install_flow_events.event_method == "install") + # .groupby("addon_id") + # .agg(F.sum("n_distinct_users").alias("installs")) + # ) - install_flow_events_df = number_installs.join( - number_uninstalls, "addon_id", how="full" - ) + # number_uninstalls = ( + # install_flow_events.where(install_flow_events.event_method == "uninstall") + # .groupby("addon_id") + # .agg(F.sum("n_distinct_users").alias("uninstalls")) + # ) - return install_flow_events_df + # install_flow_events_df = number_installs.join( + # number_uninstalls, "addon_id", how="full" + # ) + + # return install_flow_events_df def get_search_metrics(search_daily_df, addons_expanded): diff --git a/tests/resources/expected_results.json b/tests/resources/expected_results.json index 3159869..2f91623 100644 --- a/tests/resources/expected_results.json +++ b/tests/resources/expected_results.json @@ -1 +1 @@ -[{"addon_id": "baidu-code-update@mozillaonline.com", "name": "Baidu Search Update", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon2", "1": "non-system-addon1"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "screenshots@mozilla.org", "name": "Firefox Screenshots", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 2, "wau": 1, "dau": 1, "dau_prop":1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": 2, "uninstalls": 1, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4087.0, "top_10_coinstalls": {"0": "hotfix-update-xpi-intermediate@mozilla.com", "1": "non-system-addon1"}, "avg_webext_background_page_load_ms_": 2172.75, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "non-system-addon1", "name": "Non systemn Addon 1", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "hotfix-update-xpi-intermediate@mozilla.com", "name": "hotfix-update-xpi-intermediate", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "fxmonitor@mozilla.org", "name": "Firefox Monitor", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": null, "uninstalls": 1, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4059.75, "top_10_coinstalls": {"0": "non-system-addon1", "1": "hotfix-update-xpi-intermediate@mozilla.com"}, "avg_webext_background_page_load_ms_": 1969.5, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "non-system-addon2", "name": "Non System Addon 2", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon2", "1": "non-system-addon1"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "formautofill@mozilla.org", "name": "Form Autofill", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 2, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4076.5, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": 1958.25, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "webcompat@mozilla.org", "name": "Web Compat", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 2, "wau": 1, "dau": 1, "dau_prop":1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4059.75, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": 1849.25, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}] \ No newline at end of file +[{"addon_id": "baidu-code-update@mozillaonline.com", "name": "Baidu Search Update", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon2", "1": "non-system-addon1"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "screenshots@mozilla.org", "name": "Firefox Screenshots", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 2, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": {"amo": 2, "unknown": 0}, "uninstalls": {"system-addon": 1}, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4087.0, "top_10_coinstalls": {"0": "hotfix-update-xpi-intermediate@mozilla.com", "1": "non-system-addon1"}, "avg_webext_background_page_load_ms_": 2172.75, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "screenshots@mozilla.org", "name": "Firefox Screenshots", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 2, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": 584.5, "installs": {"amo": 2, "unknown": 0}, "uninstalls": {"system-addon": 1}, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4087.0, "top_10_coinstalls": {"0": "hotfix-update-xpi-intermediate@mozilla.com", "1": "non-system-addon1"}, "avg_webext_background_page_load_ms_": 2172.75, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "non-system-addon1", "name": "Non systemn Addon 1", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "hotfix-update-xpi-intermediate@mozilla.com", "name": "hotfix-update-xpi-intermediate", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "fxmonitor@mozilla.org", "name": "Firefox Monitor", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": null, "uninstalls": {"system-addon": 1}, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4059.75, "top_10_coinstalls": {"0": "non-system-addon1", "1": "hotfix-update-xpi-intermediate@mozilla.com"}, "avg_webext_background_page_load_ms_": 1969.5, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "non-system-addon2", "name": "Non System Addon 2", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 1, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": null, "top_10_coinstalls": {"0": "non-system-addon2", "1": "non-system-addon1"}, "avg_webext_background_page_load_ms_": null, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "formautofill@mozilla.org", "name": "Form Autofill", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 2, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4076.5, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": 1958.25, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}, {"addon_id": "webcompat@mozilla.org", "name": "Web Compat", "os_dist": {"Windows_NT": 1.0}, "country_dist": {"GB": 1.0}, "avg_time_total": 747.0, "active_hours": 0.18194444444444444, "disabled": null, "avg_tabs": null, "avg_bookmarks": null, "avg_toolbox_opened_count": null, "avg_uri": 33.0, "pct_w_tracking_prot_enabled": 0.0, "mau": 2, "wau": 1, "dau": 1, "dau_prop": 1.0, "sap_searches": {}, "tagged_sap_searches": {"google": 10.0}, "organic_searches": {}, "search_with_ads": {}, "ad_click": {}, "avg_download_time": null, "installs": null, "uninstalls": null, "avg_webext_storage_local_get_ms_": null, "avg_webext_storage_local_set_ms_": null, "avg_webext_extension_startup_ms_": 4059.75, "top_10_coinstalls": {"0": "non-system-addon1", "1": "non-system-addon2"}, "avg_webext_background_page_load_ms_": 1849.25, "avg_webext_browseraction_popup_open_ms_": null, "avg_webext_pageaction_popup_open_ms_": null, "avg_webext_content_script_injection_ms_": null}] \ No newline at end of file diff --git a/tests/test_integration.py b/tests/test_integration.py index 23bd662..4eb867d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -73,7 +73,6 @@ def test_agg(main_summary, search_clients_daily, events, raw_pings, spark): events=events, raw_pings=raw_pings, ) - agg.printSchema() # uncomment for test dev with open("TEST.json", "w") as f: f.write(str(df_to_json(agg))) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 2751e93..06e8793 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -15,6 +15,20 @@ def df_to_json(df): return [i.asDict() for i in df.collect()] +def load_test_data(prefix, spark): + root = os.path.dirname(__file__) + schema_path = os.path.join(root, "resources", "{}_schema.json".format(prefix)) + with open(schema_path) as f: + d = json.load(f) + schema = StructType.fromJson(d) + rows_path = os.path.join(root, "resources", "{}.json".format(prefix)) + # FAILFAST causes us to abort early if the data doesn't match + # the given schema. Without this there was as very annoying + # problem where dataframe.collect() would return an empty set. + frame = spark.read.json(rows_path, schema, mode="FAILFAST") + return frame + + @pytest.fixture() def spark(): spark_session = SparkSession.builder.appName("addons_daily_tests").getOrCreate() @@ -23,17 +37,12 @@ def spark(): @pytest.fixture() def main_summary(spark): - root = os.path.dirname(__file__) - schema_path = os.path.join(root, "resources", "main_summary_schema.json") - with open(schema_path) as f: - d = json.load(f) - schema = StructType.fromJson(d) - rows_path = os.path.join(root, "resources", "main_summary.json") - # FAILFAST causes us to abort early if the data doesn't match - # the given schema. Without this there was as very annoying - # problem where dataframe.collect() would return an empty set. - frame = spark.read.json(rows_path, schema, mode="FAILFAST") - return frame + return load_test_data("main_summary", spark) + + +@pytest.fixture() +def events(spark): + return load_test_data("events", spark) @pytest.fixture() @@ -431,3 +440,51 @@ def test_engagement_metrics(addons_expanded_day, main_summary_day, spark): }, ] assert output == expected_output + + +def test_install_flows(events): + output = df_to_json(install_flow_events(events)) + expected_output = [ + { + "addon_id": "screenshots@mozilla.org", + "avg_download_time": None, + "installs": {"amo": 2, "unknown": 0}, + "uninstalls": {"system-addon": 1}, + }, + { + "addon_id": "screenshots@mozilla.org", + "avg_download_time": 584.5, + "installs": {"amo": 2, "unknown": 0}, + "uninstalls": {"system-addon": 1}, + }, + { + "addon_id": "fxmonitor@mozilla.org", + "avg_download_time": None, + "installs": None, + "uninstalls": {"system-addon": 1}, + }, + { + "addon_id": "jid1-h4Ke2h5q31uuK7@jetpack", + "avg_download_time": 1704.0, + "installs": {"amo": 1, "unknown": 0}, + "uninstalls": None, + }, + { + "addon_id": "{87e997f4-ae0e-42e6-a780-ff73977188c5}", + "avg_download_time": 3015.0, + "installs": {"amo": 1, "unknown": 0}, + "uninstalls": None, + }, + { + "addon_id": "{08cc31c0-b1cb-461c-8ba2-95edd9e76a02}", + "avg_download_time": 998.0, + "installs": {"amo": 1, "unknown": 0}, + "uninstalls": None, + }, + { + "addon_id": "Directions_Found_mVBuOLkFzz@www.directionsfoundnt.com", + "avg_download_time": 572.0, + "installs": {"amo": 0, "unknown": 1}, + "uninstalls": None, + }, + ]