From d1c3c442b00d7a0a300ff203829bd7bee1e1fb20 Mon Sep 17 00:00:00 2001 From: m-d-bowerman <107562575+m-d-bowerman@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:45:12 -0700 Subject: [PATCH] Add serp events to levers daily (#6276) * serp_events added to levers daily * add fields and update schema * backfill serp data * Update query with all sap related fields * backfill date * Update sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/query.sql Co-authored-by: skahmann3 <16420065+skahmann3@users.noreply.github.com> * Update sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/query.sql Co-authored-by: skahmann3 <16420065+skahmann3@users.noreply.github.com> * remove duplicate field from schema * rename fields * rename fields * update schema * update serp_events sap definition * backfill end_date * remove backfill --------- Co-authored-by: skahmann3 <16420065+skahmann3@users.noreply.github.com> --- .../metadata.yaml | 6 +- .../search_revenue_levers_daily_v1/query.sql | 65 +++++++- .../schema.yaml | 141 +++++++++++------- 3 files changed, 158 insertions(+), 54 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml index 1027752efd..a2276a360a 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml @@ -5,18 +5,20 @@ description: |- The table is labeled as "change_controlled", which implies that changes require the approval of at least one owner. owners: - - mozilla/revenue_forecasting_data_reviewers +- mozilla/revenue_forecasting_data_reviewers labels: incremental: true schedule: daily change_controlled: true + dag: bqetl_search_dashboard scheduling: dag_name: bqetl_search_dashboard bigquery: time_partitioning: - field: submission_date type: day + field: submission_date require_partition_filter: true expiration_days: null + range_partitioning: null clustering: null references: {} diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/query.sql b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/query.sql index d0bd919273..0e396f3a5f 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/query.sql @@ -275,6 +275,50 @@ combined_search_dau AS ( LEFT JOIN eligible_markets_dau USING (submission_date, device, country) +), +desktop_serp_events AS ( + SELECT + submission_date, + `moz-fx-data-shared-prod`.udf.normalize_search_engine(search_engine) AS partner, + 'desktop' AS device, + normalized_country_code AS country, + COUNT(DISTINCT legacy_telemetry_client_id) AS serp_events_client_count, + COUNT( + DISTINCT IF(ad_blocker_inferred, legacy_telemetry_client_id, NULL) + ) AS serp_events_clients_with_ad_blocker_inferred, + COUNT( + DISTINCT IF( + REGEXP_CONTAINS(sap_source, 'urlbar') + OR sap_source IN ('searchbar', 'contextmenu', 'webextension', 'system'), + impression_id, + NULL + ) + ) AS serp_events_sap, + COUNTIF(is_tagged) AS serp_events_tagged_sap, + COUNTIF(is_tagged AND REGEXP_CONTAINS(sap_source, 'follow_on')) AS serp_events_tagged_follow_on, + SUM(num_ad_clicks) AS serp_events_ad_click, + COUNTIF(num_ads_visible > 0) AS serp_events_search_with_ads, + COUNTIF(NOT is_tagged) AS serp_events_organic, + SUM(IF(NOT is_tagged, num_ad_clicks, 0)) AS serp_events_ad_click_organic, + COUNTIF(num_ads_visible > 0 AND NOT is_tagged) AS serp_events_search_with_ads_organic, + -- serp_events does not have distribution ID or partner codes to calculate monetizable SAP + COUNTIF(ad_blocker_inferred) AS serp_events_sap_with_ad_blocker_inferred, + SUM(num_ads_visible) AS serp_events_num_ads_visible, + SUM(num_ads_blocked) AS serp_events_num_ads_blocked, + FROM + `moz-fx-data-shared-prod.firefox_desktop.serp_events` + WHERE + submission_date = @submission_date + AND `moz-fx-data-shared-prod`.udf.normalize_search_engine(search_engine) IN ( + 'Google', + 'Bing', + 'DuckDuckGo' + ) + GROUP BY + submission_date, + partner, + device, + country ) SELECT cd.submission_date, @@ -293,7 +337,20 @@ SELECT cd.organic, cd.ad_click_organic, cd.search_with_ads_organic, - cd.monetizable_sap + cd.monetizable_sap, + dse.serp_events_client_count, + dse.serp_events_clients_with_ad_blocker_inferred, + dse.serp_events_sap, + dse.serp_events_tagged_sap, + dse.serp_events_tagged_follow_on, + dse.serp_events_ad_click, + dse.serp_events_search_with_ads, + dse.serp_events_organic, + dse.serp_events_ad_click_organic, + dse.serp_events_search_with_ads_organic, + dse.serp_events_sap_with_ad_blocker_inferred, + dse.serp_events_num_ads_visible, + dse.serp_events_num_ads_blocked FROM combined_search_data cd LEFT JOIN @@ -302,3 +359,9 @@ LEFT JOIN AND cd.submission_date = du.submission_date AND cd.country = du.country AND cd.device = du.device +LEFT JOIN + desktop_serp_events dse + ON cd.submission_date = dse.submission_date + AND cd.partner = dse.partner + AND cd.country = dse.country + AND cd.device = dse.device diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/schema.yaml b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/schema.yaml index fbafed3a76..c42339be79 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/schema.yaml @@ -1,52 +1,91 @@ fields: - - mode: NULLABLE - name: submission_date - type: DATE - - mode: NULLABLE - name: partner - type: STRING - - mode: NULLABLE - name: device - type: STRING - - mode: NULLABLE - name: channel - type: STRING - - mode: NULLABLE - name: country - type: STRING - - mode: NULLABLE - name: dau - type: INTEGER - - mode: NULLABLE - name: dau_engaged_w_sap - type: INTEGER - - mode: NULLABLE - name: sap - type: INTEGER - - mode: NULLABLE - name: tagged_sap - type: INTEGER - - mode: NULLABLE - name: tagged_follow_on - type: INTEGER - - mode: NULLABLE - name: search_with_ads - type: INTEGER - - mode: NULLABLE - name: ad_click - type: INTEGER - - mode: NULLABLE - name: organic - type: INTEGER - - mode: NULLABLE - name: ad_click_organic - type: INTEGER - - mode: NULLABLE - name: search_with_ads_organic - type: INTEGER - - mode: NULLABLE - name: monetizable_sap - type: INTEGER - - mode: NULLABLE - name: dau_w_engine_as_default - type: INTEGER +- mode: NULLABLE + name: submission_date + type: DATE +- mode: NULLABLE + name: partner + type: STRING +- mode: NULLABLE + name: device + type: STRING +- mode: NULLABLE + name: channel + type: STRING +- mode: NULLABLE + name: country + type: STRING +- mode: NULLABLE + name: dau + type: INTEGER +- mode: NULLABLE + name: dau_engaged_w_sap + type: INTEGER +- mode: NULLABLE + name: sap + type: INTEGER +- mode: NULLABLE + name: tagged_sap + type: INTEGER +- mode: NULLABLE + name: tagged_follow_on + type: INTEGER +- mode: NULLABLE + name: search_with_ads + type: INTEGER +- mode: NULLABLE + name: ad_click + type: INTEGER +- mode: NULLABLE + name: organic + type: INTEGER +- mode: NULLABLE + name: ad_click_organic + type: INTEGER +- mode: NULLABLE + name: search_with_ads_organic + type: INTEGER +- mode: NULLABLE + name: monetizable_sap + type: INTEGER +- mode: NULLABLE + name: dau_w_engine_as_default + type: INTEGER +- name: serp_events_client_count + type: INTEGER + mode: NULLABLE +- name: serp_events_clients_with_ad_blocker_inferred + type: INTEGER + mode: NULLABLE +- name: serp_events_sap + type: INTEGER + mode: NULLABLE +- name: serp_events_tagged_sap + type: INTEGER + mode: NULLABLE +- name: serp_events_tagged_follow_on + type: INTEGER + mode: NULLABLE +- name: serp_events_ad_click + type: INTEGER + mode: NULLABLE +- name: serp_events_search_with_ads + type: INTEGER + mode: NULLABLE +- name: serp_events_organic + type: INTEGER + mode: NULLABLE +- name: serp_events_ad_click_organic + type: INTEGER + mode: NULLABLE +- name: serp_events_search_with_ads_organic + type: INTEGER + mode: NULLABLE +- name: serp_events_sap_with_ad_blocker_inferred + type: INTEGER + mode: NULLABLE +- name: serp_events_num_ads_visible + type: INTEGER + mode: NULLABLE +- name: serp_events_num_ads_blocked + type: INTEGER + mode: NULLABLE