From fc057b00ca5fb40fcfb3d7dfdb82a09e1b3e17c6 Mon Sep 17 00:00:00 2001 From: Chelsey Beck <64881557+chelseybeck@users.noreply.github.com> Date: Fri, 26 Apr 2024 13:32:11 -0700 Subject: [PATCH] Adding new models for braze sync (#5445) * adding models for changed and deleted users * adding filter to changed waitlists model * updating tests to include has_fxa field * updating tests to include an unchanged user * adding new line * removing unnecessary cross join --- .../changed_newsletters_sync_v1/query.sql | 5 +- .../changed_subscriptions_sync_v1/query.sql | 5 +- .../changed_users_sync_v1/metadata.yaml | 18 +++++++ .../changed_users_sync_v1/query.sql | 30 +++++++++++ .../changed_users_sync_v1/schema.yaml | 10 ++++ .../changed_users_v1/metadata.yaml | 18 +++++++ .../braze_external/changed_users_v1/query.sql | 52 +++++++++++++++++++ .../changed_users_v1/schema.yaml | 37 +++++++++++++ .../changed_waitlists_sync_v1/query.sql | 13 ++--- .../delete_users_sync_v1/metadata.yaml | 27 ++++++++++ .../delete_users_sync_v1/query.sql | 7 +++ .../delete_users_sync_v1/schema.yaml | 7 +++ .../test_braze_changed_users/expect.yaml | 31 +++++++++++ ...ta-shared-prod.braze_derived.users_v1.yaml | 31 +++++++++++ ...ternal.users_previous_day_snapshot_v1.yaml | 44 ++++++++++++++++ 15 files changed, 323 insertions(+), 12 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/schema.yaml create mode 100644 tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/expect.yaml create mode 100644 tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_derived.users_v1.yaml create mode 100644 tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1.yaml diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/query.sql index 4af842ef5d..43a2067a18 100644 --- a/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/query.sql @@ -25,9 +25,8 @@ SELECT FROM `moz-fx-data-shared-prod.braze_derived.newsletters_v1` AS newsletters CROSS JOIN - UNNEST(newsletters.newsletters) AS newsletters_array, - max_update + UNNEST(newsletters.newsletters) AS newsletters_array WHERE - newsletters_array.update_timestamp > max_update.max_update_timestamp + newsletters_array.update_timestamp > (SELECT max_update_timestamp FROM max_update) GROUP BY newsletters.external_id; diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_sync_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_sync_v1/query.sql index 9d0ec885b3..d647fbd89e 100644 --- a/sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_sync_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_sync_v1/query.sql @@ -26,9 +26,8 @@ SELECT FROM `moz-fx-data-shared-prod.braze_derived.subscriptions_v1` AS subscriptions CROSS JOIN - UNNEST(subscriptions.subscriptions) AS subscriptions_array, - max_update + UNNEST(subscriptions.subscriptions) AS subscriptions_array WHERE - subscriptions_array.update_timestamp > max_update.max_update_timestamp + subscriptions_array.update_timestamp > (SELECT max_update_timestamp FROM max_update) GROUP BY subscriptions.external_id; diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml new file mode 100644 index 0000000000..53a9d57b7d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml @@ -0,0 +1,18 @@ +friendly_name: Braze Changed Users Sync +description: |- + Changes in users from the previous day. In order to consume the least amount + of data points, we only want to sync rows changed since the last sync. This table holds + all of the changed user data that syncs to braze. + + See https://mozilla-hub.atlassian.net/browse/DENG-3182 +owners: +- cbeck@mozilla.com +labels: + incremental: false + schedule: daily + owner: cbeck +bigquery: + time_partitioning: null +scheduling: + dag_name: bqetl_braze + date_partition_parameter: null diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/query.sql new file mode 100644 index 0000000000..d2696fa946 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/query.sql @@ -0,0 +1,30 @@ +SELECT + CURRENT_TIMESTAMP() AS UPDATED_AT, + changed_users.external_id AS EXTERNAL_ID, + TO_JSON( + STRUCT( + ARRAY_AGG( + STRUCT( + changed_users.email AS email, + changed_users.mailing_country AS mailing_country, + changed_users.email_subscribe AS email_subscribe, + changed_users.basket_token AS basket_token, + changed_users.email_lang AS email_lang, + changed_users.has_fxa AS has_fxa, + changed_users.fxa_primary_email AS fxa_primary_email, + changed_users.fxa_lang AS fxa_lang, + changed_users.first_service AS first_service, + changed_users.create_timestamp AS create_timestamp, + changed_users.update_timestamp AS update_timestamp + ) + ORDER BY + changed_users.update_timestamp DESC + ) AS user_attributes + ) + ) AS PAYLOAD +FROM + `moz-fx-data-shared-prod.braze_external.changed_users_v1` AS changed_users +WHERE + changed_users.status = 'Changed' +GROUP BY + changed_users.external_id; diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/schema.yaml b/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/schema.yaml new file mode 100644 index 0000000000..2e0b6bcd39 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/schema.yaml @@ -0,0 +1,10 @@ +fields: +- mode: NULLABLE + name: UPDATED_AT + type: TIMESTAMP +- mode: NULLABLE + name: EXTERNAL_ID + type: STRING +- mode: NULLABLE + name: PAYLOAD + type: JSON diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml new file mode 100644 index 0000000000..70f67cb116 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml @@ -0,0 +1,18 @@ +friendly_name: Braze Changed Users +description: |- + This table represents changes to user profiles from the previous + DAG run. It enables us to build the changed users sync and + delete users tables. + + See https://mozilla-hub.atlassian.net/browse/DENG-3182 +owners: +- cbeck@mozilla.com +labels: + incremental: false + schedule: daily + owner: cbeck +bigquery: + time_partitioning: null +scheduling: + dag_name: bqetl_braze + date_partition_parameter: null diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql new file mode 100644 index 0000000000..32f445d57f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql @@ -0,0 +1,52 @@ +SELECT + COALESCE(current_users.external_id, previous_users.external_id) AS external_id, + CASE + WHEN current_users.external_id IS NULL + THEN 'Deleted' + WHEN NOT ( + current_users.email = previous_users.email + AND current_users.mailing_country = previous_users.mailing_country + AND current_users.email_subscribe = previous_users.email_subscribe + AND current_users.basket_token = previous_users.basket_token + AND current_users.email_lang = previous_users.email_lang + AND current_users.create_timestamp = previous_users.create_timestamp + AND current_users.update_timestamp = previous_users.update_timestamp + AND current_users.fxa_id_sha256 = previous_users.fxa_id_sha256 + AND current_users.has_fxa = previous_users.has_fxa + AND current_users.fxa_primary_email = previous_users.fxa_primary_email + AND current_users.fxa_lang = previous_users.fxa_lang + AND current_users.first_service = previous_users.first_service + ) + THEN 'Changed' + END AS status, + COALESCE(current_users.email, previous_users.email) AS email, + COALESCE(current_users.mailing_country, previous_users.mailing_country) AS mailing_country, + COALESCE(current_users.email_subscribe, previous_users.email_subscribe) AS email_subscribe, + COALESCE(current_users.basket_token, previous_users.basket_token) AS basket_token, + COALESCE(current_users.email_lang, previous_users.email_lang) AS email_lang, + COALESCE(current_users.create_timestamp, previous_users.create_timestamp) AS create_timestamp, + COALESCE(current_users.update_timestamp, previous_users.update_timestamp) AS update_timestamp, + COALESCE(current_users.has_fxa, previous_users.has_fxa) AS has_fxa, + COALESCE(current_users.fxa_primary_email, previous_users.fxa_primary_email) AS fxa_primary_email, + COALESCE(current_users.fxa_lang, previous_users.fxa_lang) AS fxa_lang, + COALESCE(current_users.first_service, previous_users.first_service) AS first_service +FROM + `moz-fx-data-shared-prod.braze_derived.users_v1` current_users +FULL OUTER JOIN + `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` previous_users + ON current_users.external_id = previous_users.external_id +WHERE + current_users.external_id IS NULL -- deleted rows + OR NOT ( + current_users.email = previous_users.email + AND current_users.mailing_country = previous_users.mailing_country + AND current_users.email_subscribe = previous_users.email_subscribe + AND current_users.basket_token = previous_users.basket_token + AND current_users.email_lang = previous_users.email_lang + AND current_users.create_timestamp = previous_users.create_timestamp + AND current_users.update_timestamp = previous_users.update_timestamp + AND current_users.has_fxa = previous_users.has_fxa + AND current_users.fxa_primary_email = previous_users.fxa_primary_email + AND current_users.fxa_lang = previous_users.fxa_lang + AND current_users.first_service = previous_users.first_service + ); -- changed rows diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/schema.yaml b/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/schema.yaml new file mode 100644 index 0000000000..c82557eeac --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/schema.yaml @@ -0,0 +1,37 @@ +fields: +- mode: NULLABLE + name: external_id + type: STRING +- mode: NULLABLE + name: email + type: STRING +- mode: NULLABLE + name: mailing_country + type: STRING +- mode: NULLABLE + name: email_subscribe + type: STRING +- mode: NULLABLE + name: basket_token + type: STRING +- mode: NULLABLE + name: email_lang + type: STRING +- mode: NULLABLE + name: create_timestamp + type: TIMESTAMP +- mode: NULLABLE + name: update_timestamp + type: TIMESTAMP +- mode: NULLABLE + name: has_fxa + type: BOOLEAN +- mode: NULLABLE + name: fxa_primary_email + type: STRING +- mode: NULLABLE + name: fxa_lang + type: STRING +- mode: NULLABLE + name: first_service + type: STRING diff --git a/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/query.sql index 64c16f20d6..29fd2422d7 100644 --- a/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/query.sql @@ -3,6 +3,8 @@ WITH max_update AS ( MAX(UPDATED_AT) AS max_update_timestamp FROM `moz-fx-data-shared-prod.braze_external.changed_waitlists_sync_v1` + LIMIT + 1 ) SELECT CURRENT_TIMESTAMP() AS UPDATED_AT, @@ -21,15 +23,14 @@ SELECT ) ORDER BY waitlists_array.update_timestamp DESC - ) AS newsletters + ) AS waitlists ) ) AS PAYLOAD FROM - `moz-fx-data-shared-prod.braze_derived.waitlists_v1` AS waitlists, - -- CROSS JOIN + `moz-fx-data-shared-prod.braze_derived.waitlists_v1` AS waitlists +CROSS JOIN UNNEST(waitlists.waitlists) AS waitlists_array - -- max_update - -- WHERE - -- waitlists_array.update_timestamp > max_update.max_update_timestamp +WHERE + waitlists_array.update_timestamp > (SELECT max_update_timestamp FROM max_update) GROUP BY waitlists.external_id; diff --git a/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml new file mode 100644 index 0000000000..9e7834038b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml @@ -0,0 +1,27 @@ +friendly_name: Braze Delete Users Sync +description: |- + Users added to this table are deleted in Braze. Deleted users + can be a result of the following actions: + + - User data deletion request + - User added to suppression list + - User requested FxA deletion - this logic is currently under + discussion and may change. The FxA/Basket/FxA teams are discussing + adding an option for users to also unsubscribe from newsletters + to grant agency. + + Braze Cloud Data Ingestion (CDI) tool documentation for deleting users: + https://www.braze.com/docs/user_guide/data_and_analytics/cloud_ingestion/delete_users/ + + See https://mozilla-hub.atlassian.net/browse/DENG-3182 +owners: +- cbeck@mozilla.com +labels: + incremental: false + schedule: daily + owner: cbeck +bigquery: + time_partitioning: null +scheduling: + dag_name: bqetl_braze + date_partition_parameter: null diff --git a/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/query.sql new file mode 100644 index 0000000000..f80c8da492 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/query.sql @@ -0,0 +1,7 @@ +SELECT + CURRENT_TIMESTAMP() AS UPDATED_AT, + external_id AS EXTERNAL_ID +FROM + `moz-fx-data-shared-prod.braze_external.changed_users_v1` +WHERE + status = 'Deleted' diff --git a/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/schema.yaml b/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/schema.yaml new file mode 100644 index 0000000000..b23a0e203a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- mode: NULLABLE + name: UPDATED_AT + type: TIMESTAMP +- mode: NULLABLE + name: EXTERNAL_ID + type: STRING diff --git a/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/expect.yaml b/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/expect.yaml new file mode 100644 index 0000000000..e03a27d8d2 --- /dev/null +++ b/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/expect.yaml @@ -0,0 +1,31 @@ +# expect braze changed users +--- +# user 1 +- external_id: user_1 + status: Changed + email: user_1@mail.com + mailing_country: country_1 + email_subscribe: subscribed + basket_token: token_1 + email_lang: DE + create_timestamp: 2020-01-01 10:00:00+00:00 + update_timestamp: 2021-01-01 14:00:00+00:00 + has_fxa: true + fxa_primary_email: user_1_primary@mail.com + fxa_lang: DE + first_service: service_1 +# user 2 +- external_id: user_2 + status: Deleted + email: user_2@mail.com + mailing_country: country_1 + email_subscribe: subscribed + basket_token: token_1 + email_lang: EN + create_timestamp: 2020-01-01 10:00:00+00:00 + update_timestamp: 2021-01-01 14:00:00+00:00 + has_fxa: false + fxa_primary_email: user_2_primary@mail.com + fxa_lang: DE + first_service: service_1 +# user 3 - unchanged diff --git a/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_derived.users_v1.yaml b/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_derived.users_v1.yaml new file mode 100644 index 0000000000..58aa5effc7 --- /dev/null +++ b/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_derived.users_v1.yaml @@ -0,0 +1,31 @@ +# braze users +--- +# user 1 - changed +- external_id: user_1 + email: user_1@mail.com + mailing_country: country_1 + email_subscribe: subscribed + basket_token: token_1 + email_lang: DE + create_timestamp: 2020-01-01 10:00:00+00:00 + update_timestamp: 2021-01-01 14:00:00+00:00 + fxa_id_sha256: fxa_user_1 + has_fxa: true + fxa_primary_email: user_1_primary@mail.com + fxa_lang: DE + first_service: service_1 +# user 2 - deleted +# user 3 - unchanged +- external_id: user_3 + email: user_3@mail.com + mailing_country: country_1 + email_subscribe: subscribed + basket_token: token_3 + email_lang: DE + create_timestamp: 2020-01-01 10:00:00+00:00 + update_timestamp: 2021-01-01 14:00:00+00:00 + fxa_id_sha256: fxa_user_3 + has_fxa: true + fxa_primary_email: user_3_primary@mail.com + fxa_lang: DE + first_service: service_1 diff --git a/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1.yaml b/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1.yaml new file mode 100644 index 0000000000..9c12671fef --- /dev/null +++ b/tests/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/test_braze_changed_users/moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1.yaml @@ -0,0 +1,44 @@ +# braze users +--- +# user 1 +- external_id: user_1 + email: user_1@mail.com + mailing_country: country_1 + email_subscribe: subscribed + basket_token: token_1 + email_lang: EN + create_timestamp: 2020-01-01 10:00:00+00:00 + update_timestamp: 2021-01-01 14:00:00+00:00 + fxa_id_sha256: fxa_user_1 + has_fxa: true + fxa_primary_email: user_1_primary@mail.com + fxa_lang: DE + first_service: service_1 +# user 2 +- external_id: user_2 + email: user_2@mail.com + mailing_country: country_1 + email_subscribe: subscribed + basket_token: token_1 + email_lang: EN + create_timestamp: 2020-01-01 10:00:00+00:00 + update_timestamp: 2021-01-01 14:00:00+00:00 + fxa_id_sha256: fxa_user_2 + has_fxa: false + fxa_primary_email: user_2_primary@mail.com + fxa_lang: DE + first_service: service_1 +# user 3 +- external_id: user_3 + email: user_3@mail.com + mailing_country: country_1 + email_subscribe: subscribed + basket_token: token_3 + email_lang: DE + create_timestamp: 2020-01-01 10:00:00+00:00 + update_timestamp: 2021-01-01 14:00:00+00:00 + fxa_id_sha256: fxa_user_3 + has_fxa: true + fxa_primary_email: user_3_primary@mail.com + fxa_lang: DE + first_service: service_1