Adding new models for braze sync (#5445)
* adding models for changed and deleted users * adding filter to changed waitlists model * updating tests to include has_fxa field * updating tests to include an unchanged user * adding new line * removing unnecessary cross join
This commit is contained in:
Chelsey Beck
GitHub
Родитель
fc45d45ae3
Коммит
fc057b00ca
|
|
@ -25,9 +25,8 @@ SELECT
|
|||
FROM
|
||||
`moz-fx-data-shared-prod.braze_derived.newsletters_v1` AS newsletters
|
||||
CROSS JOIN
|
||||
UNNEST(newsletters.newsletters) AS newsletters_array,
|
||||
max_update
|
||||
UNNEST(newsletters.newsletters) AS newsletters_array
|
||||
WHERE
|
||||
newsletters_array.update_timestamp > max_update.max_update_timestamp
|
||||
newsletters_array.update_timestamp > (SELECT max_update_timestamp FROM max_update)
|
||||
GROUP BY
|
||||
newsletters.external_id;
|
||||
|
|
|
|||
|
|
@ -26,9 +26,8 @@ SELECT
|
|||
FROM
|
||||
`moz-fx-data-shared-prod.braze_derived.subscriptions_v1` AS subscriptions
|
||||
CROSS JOIN
|
||||
UNNEST(subscriptions.subscriptions) AS subscriptions_array,
|
||||
max_update
|
||||
UNNEST(subscriptions.subscriptions) AS subscriptions_array
|
||||
WHERE
|
||||
subscriptions_array.update_timestamp > max_update.max_update_timestamp
|
||||
subscriptions_array.update_timestamp > (SELECT max_update_timestamp FROM max_update)
|
||||
GROUP BY
|
||||
subscriptions.external_id;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
friendly_name: Braze Changed Users Sync
|
||||
description: |-
|
||||
Changes in users from the previous day. In order to consume the least amount
|
||||
of data points, we only want to sync rows changed since the last sync. This table holds
|
||||
all of the changed user data that syncs to braze.
|
||||
|
||||
See https://mozilla-hub.atlassian.net/browse/DENG-3182
|
||||
owners:
|
||||
- cbeck@mozilla.com
|
||||
labels:
|
||||
incremental: false
|
||||
schedule: daily
|
||||
owner: cbeck
|
||||
bigquery:
|
||||
time_partitioning: null
|
||||
scheduling:
|
||||
dag_name: bqetl_braze
|
||||
date_partition_parameter: null
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
SELECT
|
||||
CURRENT_TIMESTAMP() AS UPDATED_AT,
|
||||
changed_users.external_id AS EXTERNAL_ID,
|
||||
TO_JSON(
|
||||
STRUCT(
|
||||
ARRAY_AGG(
|
||||
STRUCT(
|
||||
changed_users.email AS email,
|
||||
changed_users.mailing_country AS mailing_country,
|
||||
changed_users.email_subscribe AS email_subscribe,
|
||||
changed_users.basket_token AS basket_token,
|
||||
changed_users.email_lang AS email_lang,
|
||||
changed_users.has_fxa AS has_fxa,
|
||||
changed_users.fxa_primary_email AS fxa_primary_email,
|
||||
changed_users.fxa_lang AS fxa_lang,
|
||||
changed_users.first_service AS first_service,
|
||||
changed_users.create_timestamp AS create_timestamp,
|
||||
changed_users.update_timestamp AS update_timestamp
|
||||
)
|
||||
ORDER BY
|
||||
changed_users.update_timestamp DESC
|
||||
) AS user_attributes
|
||||
)
|
||||
) AS PAYLOAD
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.braze_external.changed_users_v1` AS changed_users
|
||||
WHERE
|
||||
changed_users.status = 'Changed'
|
||||
GROUP BY
|
||||
changed_users.external_id;
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: UPDATED_AT
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: EXTERNAL_ID
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: PAYLOAD
|
||||
type: JSON
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
friendly_name: Braze Changed Users
|
||||
description: |-
|
||||
This table represents changes to user profiles from the previous
|
||||
DAG run. It enables us to build the changed users sync and
|
||||
delete users tables.
|
||||
|
||||
See https://mozilla-hub.atlassian.net/browse/DENG-3182
|
||||
owners:
|
||||
- cbeck@mozilla.com
|
||||
labels:
|
||||
incremental: false
|
||||
schedule: daily
|
||||
owner: cbeck
|
||||
bigquery:
|
||||
time_partitioning: null
|
||||
scheduling:
|
||||
dag_name: bqetl_braze
|
||||
date_partition_parameter: null
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
SELECT
|
||||
COALESCE(current_users.external_id, previous_users.external_id) AS external_id,
|
||||
CASE
|
||||
WHEN current_users.external_id IS NULL
|
||||
THEN 'Deleted'
|
||||
WHEN NOT (
|
||||
current_users.email = previous_users.email
|
||||
AND current_users.mailing_country = previous_users.mailing_country
|
||||
AND current_users.email_subscribe = previous_users.email_subscribe
|
||||
AND current_users.basket_token = previous_users.basket_token
|
||||
AND current_users.email_lang = previous_users.email_lang
|
||||
AND current_users.create_timestamp = previous_users.create_timestamp
|
||||
AND current_users.update_timestamp = previous_users.update_timestamp
|
||||
AND current_users.fxa_id_sha256 = previous_users.fxa_id_sha256
|
||||
AND current_users.has_fxa = previous_users.has_fxa
|
||||
AND current_users.fxa_primary_email = previous_users.fxa_primary_email
|
||||
AND current_users.fxa_lang = previous_users.fxa_lang
|
||||
AND current_users.first_service = previous_users.first_service
|
||||
)
|
||||
THEN 'Changed'
|
||||
END AS status,
|
||||
COALESCE(current_users.email, previous_users.email) AS email,
|
||||
COALESCE(current_users.mailing_country, previous_users.mailing_country) AS mailing_country,
|
||||
COALESCE(current_users.email_subscribe, previous_users.email_subscribe) AS email_subscribe,
|
||||
COALESCE(current_users.basket_token, previous_users.basket_token) AS basket_token,
|
||||
COALESCE(current_users.email_lang, previous_users.email_lang) AS email_lang,
|
||||
COALESCE(current_users.create_timestamp, previous_users.create_timestamp) AS create_timestamp,
|
||||
COALESCE(current_users.update_timestamp, previous_users.update_timestamp) AS update_timestamp,
|
||||
COALESCE(current_users.has_fxa, previous_users.has_fxa) AS has_fxa,
|
||||
COALESCE(current_users.fxa_primary_email, previous_users.fxa_primary_email) AS fxa_primary_email,
|
||||
COALESCE(current_users.fxa_lang, previous_users.fxa_lang) AS fxa_lang,
|
||||
COALESCE(current_users.first_service, previous_users.first_service) AS first_service
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.braze_derived.users_v1` current_users
|
||||
FULL OUTER JOIN
|
||||
`moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` previous_users
|
||||
ON current_users.external_id = previous_users.external_id
|
||||
WHERE
|
||||
current_users.external_id IS NULL -- deleted rows
|
||||
OR NOT (
|
||||
current_users.email = previous_users.email
|
||||
AND current_users.mailing_country = previous_users.mailing_country
|
||||
AND current_users.email_subscribe = previous_users.email_subscribe
|
||||
AND current_users.basket_token = previous_users.basket_token
|
||||
AND current_users.email_lang = previous_users.email_lang
|
||||
AND current_users.create_timestamp = previous_users.create_timestamp
|
||||
AND current_users.update_timestamp = previous_users.update_timestamp
|
||||
AND current_users.has_fxa = previous_users.has_fxa
|
||||
AND current_users.fxa_primary_email = previous_users.fxa_primary_email
|
||||
AND current_users.fxa_lang = previous_users.fxa_lang
|
||||
AND current_users.first_service = previous_users.first_service
|
||||
); -- changed rows
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: external_id
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: email
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: mailing_country
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: email_subscribe
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: basket_token
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: email_lang
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: create_timestamp
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: update_timestamp
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: has_fxa
|
||||
type: BOOLEAN
|
||||
- mode: NULLABLE
|
||||
name: fxa_primary_email
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: fxa_lang
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: first_service
|
||||
type: STRING
|
||||
|
|
@ -3,6 +3,8 @@ WITH max_update AS (
|
|||
MAX(UPDATED_AT) AS max_update_timestamp
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.braze_external.changed_waitlists_sync_v1`
|
||||
LIMIT
|
||||
1
|
||||
)
|
||||
SELECT
|
||||
CURRENT_TIMESTAMP() AS UPDATED_AT,
|
||||
|
|
@ -21,15 +23,14 @@ SELECT
|
|||
)
|
||||
ORDER BY
|
||||
waitlists_array.update_timestamp DESC
|
||||
) AS newsletters
|
||||
) AS waitlists
|
||||
)
|
||||
) AS PAYLOAD
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.braze_derived.waitlists_v1` AS waitlists,
|
||||
-- CROSS JOIN
|
||||
`moz-fx-data-shared-prod.braze_derived.waitlists_v1` AS waitlists
|
||||
CROSS JOIN
|
||||
UNNEST(waitlists.waitlists) AS waitlists_array
|
||||
-- max_update
|
||||
-- WHERE
|
||||
-- waitlists_array.update_timestamp > max_update.max_update_timestamp
|
||||
WHERE
|
||||
waitlists_array.update_timestamp > (SELECT max_update_timestamp FROM max_update)
|
||||
GROUP BY
|
||||
waitlists.external_id;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
friendly_name: Braze Delete Users Sync
|
||||
description: |-
|
||||
Users added to this table are deleted in Braze. Deleted users
|
||||
can be a result of the following actions:
|
||||
|
||||
- User data deletion request
|
||||
- User added to suppression list
|
||||
- User requested FxA deletion - this logic is currently under
|
||||
discussion and may change. The FxA/Basket/FxA teams are discussing
|
||||
adding an option for users to also unsubscribe from newsletters
|
||||
to grant agency.
|
||||
|
||||
Braze Cloud Data Ingestion (CDI) tool documentation for deleting users:
|
||||
https://www.braze.com/docs/user_guide/data_and_analytics/cloud_ingestion/delete_users/
|
||||
|
||||
See https://mozilla-hub.atlassian.net/browse/DENG-3182
|
||||
owners:
|
||||
- cbeck@mozilla.com
|
||||
labels:
|
||||
incremental: false
|
||||
schedule: daily
|
||||
owner: cbeck
|
||||
bigquery:
|
||||
time_partitioning: null
|
||||
scheduling:
|
||||
dag_name: bqetl_braze
|
||||
date_partition_parameter: null
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
SELECT
|
||||
CURRENT_TIMESTAMP() AS UPDATED_AT,
|
||||
external_id AS EXTERNAL_ID
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.braze_external.changed_users_v1`
|
||||
WHERE
|
||||
status = 'Deleted'
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: UPDATED_AT
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: EXTERNAL_ID
|
||||
type: STRING
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# expect braze changed users
|
||||
---
|
||||
# user 1
|
||||
- external_id: user_1
|
||||
status: Changed
|
||||
email: user_1@mail.com
|
||||
mailing_country: country_1
|
||||
email_subscribe: subscribed
|
||||
basket_token: token_1
|
||||
email_lang: DE
|
||||
create_timestamp: 2020-01-01 10:00:00+00:00
|
||||
update_timestamp: 2021-01-01 14:00:00+00:00
|
||||
has_fxa: true
|
||||
fxa_primary_email: user_1_primary@mail.com
|
||||
fxa_lang: DE
|
||||
first_service: service_1
|
||||
# user 2
|
||||
- external_id: user_2
|
||||
status: Deleted
|
||||
email: user_2@mail.com
|
||||
mailing_country: country_1
|
||||
email_subscribe: subscribed
|
||||
basket_token: token_1
|
||||
email_lang: EN
|
||||
create_timestamp: 2020-01-01 10:00:00+00:00
|
||||
update_timestamp: 2021-01-01 14:00:00+00:00
|
||||
has_fxa: false
|
||||
fxa_primary_email: user_2_primary@mail.com
|
||||
fxa_lang: DE
|
||||
first_service: service_1
|
||||
# user 3 - unchanged
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# braze users
|
||||
---
|
||||
# user 1 - changed
|
||||
- external_id: user_1
|
||||
email: user_1@mail.com
|
||||
mailing_country: country_1
|
||||
email_subscribe: subscribed
|
||||
basket_token: token_1
|
||||
email_lang: DE
|
||||
create_timestamp: 2020-01-01 10:00:00+00:00
|
||||
update_timestamp: 2021-01-01 14:00:00+00:00
|
||||
fxa_id_sha256: fxa_user_1
|
||||
has_fxa: true
|
||||
fxa_primary_email: user_1_primary@mail.com
|
||||
fxa_lang: DE
|
||||
first_service: service_1
|
||||
# user 2 - deleted
|
||||
# user 3 - unchanged
|
||||
- external_id: user_3
|
||||
email: user_3@mail.com
|
||||
mailing_country: country_1
|
||||
email_subscribe: subscribed
|
||||
basket_token: token_3
|
||||
email_lang: DE
|
||||
create_timestamp: 2020-01-01 10:00:00+00:00
|
||||
update_timestamp: 2021-01-01 14:00:00+00:00
|
||||
fxa_id_sha256: fxa_user_3
|
||||
has_fxa: true
|
||||
fxa_primary_email: user_3_primary@mail.com
|
||||
fxa_lang: DE
|
||||
first_service: service_1
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# braze users
|
||||
---
|
||||
# user 1
|
||||
- external_id: user_1
|
||||
email: user_1@mail.com
|
||||
mailing_country: country_1
|
||||
email_subscribe: subscribed
|
||||
basket_token: token_1
|
||||
email_lang: EN
|
||||
create_timestamp: 2020-01-01 10:00:00+00:00
|
||||
update_timestamp: 2021-01-01 14:00:00+00:00
|
||||
fxa_id_sha256: fxa_user_1
|
||||
has_fxa: true
|
||||
fxa_primary_email: user_1_primary@mail.com
|
||||
fxa_lang: DE
|
||||
first_service: service_1
|
||||
# user 2
|
||||
- external_id: user_2
|
||||
email: user_2@mail.com
|
||||
mailing_country: country_1
|
||||
email_subscribe: subscribed
|
||||
basket_token: token_1
|
||||
email_lang: EN
|
||||
create_timestamp: 2020-01-01 10:00:00+00:00
|
||||
update_timestamp: 2021-01-01 14:00:00+00:00
|
||||
fxa_id_sha256: fxa_user_2
|
||||
has_fxa: false
|
||||
fxa_primary_email: user_2_primary@mail.com
|
||||
fxa_lang: DE
|
||||
first_service: service_1
|
||||
# user 3
|
||||
- external_id: user_3
|
||||
email: user_3@mail.com
|
||||
mailing_country: country_1
|
||||
email_subscribe: subscribed
|
||||
basket_token: token_3
|
||||
email_lang: DE
|
||||
create_timestamp: 2020-01-01 10:00:00+00:00
|
||||
update_timestamp: 2021-01-01 14:00:00+00:00
|
||||
fxa_id_sha256: fxa_user_3
|
||||
has_fxa: true
|
||||
fxa_primary_email: user_3_primary@mail.com
|
||||
fxa_lang: DE
|
||||
first_service: service_1
|
||||
Загрузка…
Ссылка в новой задаче