Feat/adding device id to fxa data (#3314)

* added fxa_users_services_devices_first_seen_v1 model and adjusted models to propagate device_id

* reverted sql/moz-fx-data-shared-prod/firefox_accounts_derived/funnel_events_source_v1/query.sql to main branch version

* implemented changes as suggested by @relud in PR#3314

* fixed missing )

* added device_id to fxa_auth_bounce_events table

* applied sql formatting

sql/moz-fx-data-shared-prod/firefox_accounts/fxa_content_auth_oauth_events/view.sql

@@ -14,6 +14,7 @@ WITH content AS (
     jsonPayload.fields.user_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
+    jsonPayload.fields.device_id,
     `timestamp`,
     receiveTimestamp,
   FROM
@@ -32,6 +33,7 @@ auth AS (
     jsonPayload.fields.user_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
+    jsonPayload.fields.device_id,
     `timestamp`,
     receiveTimestamp,
   FROM
@@ -50,6 +52,7 @@ oauth AS (
     jsonPayload.fields.user_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
+    CAST(NULL AS STRING) AS device_id,
     `timestamp`,
     receiveTimestamp,
   FROM

sql/moz-fx-data-shared-prod/firefox_accounts_derived/funnel_events_source_v1/query.sql

@@ -19,6 +19,6 @@ SELECT
   CAST([] AS ARRAY<STRUCT<key STRING, value STRING>>) AS experiments,
   *,
 FROM
-  `moz-fx-data-shared-prod`.firefox_accounts.fxa_content_auth_oauth_events
+  `moz-fx-data-shared-prod.firefox_accounts.fxa_content_auth_oauth_events`
 WHERE
   DATE(timestamp) = @submission_date

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql

@@ -6,7 +6,8 @@ SELECT
           (
             SELECT AS STRUCT
               jsonPayload.fields.* EXCEPT (device_id, user_id),
-              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id
+              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id,
+              TO_HEX(SHA256(jsonPayload.fields.device_id)) AS device_id
           ) AS fields
         )
     ) AS jsonPayload

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql

@@ -10,7 +10,7 @@ WITH base AS (
           jsonPayload.* REPLACE (
             (
               SELECT AS STRUCT
-                jsonPayload.fields.* EXCEPT (device_id, user_id) REPLACE(
+                jsonPayload.fields.* EXCEPT (user_id, device_id, deviceid) REPLACE(
                   -- See https://bugzilla.mozilla.org/show_bug.cgi?id=1707571
                   CAST(NULL AS FLOAT64) AS emailverified,
                   CAST(NULL AS FLOAT64) AS isprimary,
@@ -18,7 +18,10 @@ WITH base AS (
                   CAST(NULL AS STRING) AS id,
                   CAST(NULL AS STRING) AS metricsoptoutat
                 ),
-                TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id
+                TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id,
+                TO_HEX(
+                  SHA256(COALESCE(jsonPayload.fields.device_id, jsonPayload.fields.deviceid))
+                ) AS device_id
             ) AS fields
           )
       ) AS jsonPayload

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql

@@ -6,7 +6,8 @@ SELECT
           (
             SELECT AS STRUCT
               jsonPayload.fields.* EXCEPT (device_id, user_id),
-              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id
+              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id,
+              TO_HEX(SHA256(jsonPayload.fields.device_id)) AS device_id
           ) AS fields
         )
     ) AS jsonPayload

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/query.sql

@@ -2,6 +2,7 @@ SELECT
   timestamp,
   jsonPayload.fields.event,
   jsonPayload.fields.flow_id,
+  jsonPayload.fields.device_id,
   jsonPayload.fields.entrypoint,
   jsonPayload.fields.service,
   jsonPayload.fields.useragent,

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/query.sql

@@ -8,6 +8,7 @@ WITH base AS (
     jsonPayload.country,
     jsonPayload.entrypoint,
     jsonPayload.flow_id,
+    jsonPayload.device_id,
     jsonPayload.region,
     jsonPayload.service,
     jsonPayload.utm_campaign,

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_stdout_events_v1/query.sql

@@ -6,7 +6,8 @@ SELECT
           (
             SELECT AS STRUCT
               jsonPayload.fields.* EXCEPT (device_id, user_id),
-              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id
+              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id,
+              TO_HEX(SHA256(jsonPayload.fields.device_id)) AS device_id
           ) AS fields
         )
     ) AS jsonPayload

sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_auth_events_v1/query.sql

@@ -5,7 +5,7 @@ SELECT
         jsonPayload.* REPLACE (
           (
             SELECT AS STRUCT
-              jsonPayload.fields.* EXCEPT (device_id, user_id) REPLACE(
+              jsonPayload.fields.* EXCEPT (user_id, device_id, deviceid) REPLACE(
                 -- See https://bugzilla.mozilla.org/show_bug.cgi?id=1707571
                 CAST(NULL AS FLOAT64) AS emailverified,
                 CAST(NULL AS FLOAT64) AS isprimary,
@@ -13,7 +13,10 @@ SELECT
                 -- casting id as field type in source tables inconsistent
                 CAST(jsonPayload.fields.id AS STRING) AS id
               ),
-              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id
+              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id,
+              TO_HEX(
+                SHA256(COALESCE(jsonPayload.fields.device_id, jsonPayload.fields.deviceid))
+              ) AS device_id
           ) AS fields
         )
     ) AS jsonPayload

sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_content_events_v1/query.sql

@@ -6,7 +6,8 @@ SELECT
           (
             SELECT AS STRUCT
               jsonPayload.fields.* EXCEPT (device_id, user_id),
-              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id
+              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id,
+              TO_HEX(SHA256(jsonPayload.fields.device_id)) AS device_id
           ) AS fields
         )
     ) AS jsonPayload

sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_stdout_events_v1/query.sql

@@ -6,7 +6,8 @@ SELECT
           (
             SELECT AS STRUCT
               jsonPayload.fields.* EXCEPT (device_id, user_id),
-              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id
+              TO_HEX(SHA256(jsonPayload.fields.user_id)) AS user_id,
+              TO_HEX(SHA256(jsonPayload.fields.device_id)) AS device_id
           ) AS fields
         )
     ) AS jsonPayload