Include new fields for SubPlat in FxA events views (DENG-1006) (#3926)

* Include new fields for SubPlat in `fxa_content_auth_stdout_events`.

* Include new fields for SubPlat in `nonprod_fxa_content_auth_stdout_events`.

* Include new fields for SubPlat in `fxa_all_events`.

* Move new `time` column to be by the other timestamp columns.

* Keep `subscribed_plan_ids` as a string so it's accessible in Looker.

* Add `schema.yaml` files for FxA events ETLs.

So the tables can be successfully staged for CI for downstream ETLs/views to pass.

* Fully qualify view in `fxa_users_daily_v1` to try to get test to pass.

* Rename `time` column `event_time`.

* Include new fields for SubPlat in `nonprod_fxa_all_events`.

---------

Co-authored-by: Daniel Thorn <dthorn@mozilla.com>

sql/moz-fx-data-shared-prod/firefox_accounts/fxa_all_events/view.sql

@@ -10,8 +10,10 @@ WITH fxa_auth_events AS (
   SELECT
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     jsonPayload.fields.country,
+    JSON_VALUE(jsonPayload.fields.event_properties, "$.country_code") AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     jsonPayload.fields.os_name,
@@ -30,10 +32,12 @@ fxa_auth_bounce_events AS (
   SELECT
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     CAST(
       NULL AS STRING
     ) AS country,  -- No country field in auth_bounces
+    CAST(NULL AS STRING) AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     CAST(NULL AS STRING) AS os_name,
@@ -50,8 +54,10 @@ fxa_content_events AS (
   SELECT
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     jsonPayload.fields.country,
+    CAST(NULL AS STRING) AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     jsonPayload.fields.os_name,
@@ -69,8 +75,10 @@ fxa_oauth_events AS (
   SELECT
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     CAST(NULL AS STRING) AS country,
+    CAST(NULL AS STRING) AS country_code,
     CAST(NULL AS STRING) AS language,
     jsonPayload.fields.app_version,
     CAST(NULL AS STRING) AS os_name,
@@ -87,8 +95,10 @@ fxa_stdout_events AS (
   SELECT
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     CAST(NULL AS STRING) AS country,
+    jsonPayload.fields.country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     jsonPayload.fields.os_name,
@@ -136,12 +146,14 @@ unioned AS (
 SELECT
   `timestamp`,
   receiveTimestamp,
+  event_time,
   logger,
   fxa_log,
   event_type,
   user_id,
   device_id,
   country,
+  country_code,
   `language`,
   app_version,
   os_name,
@@ -175,11 +187,27 @@ SELECT
   JSON_VALUE(event_properties, "$.email_service") AS email_service,
   JSON_VALUE(event_properties, "$.email_template") AS email_template,
   JSON_VALUE(event_properties, "$.email_version") AS email_version,
+  JSON_VALUE(event_properties, "$.subscription_id") AS subscription_id,
   JSON_VALUE(event_properties, "$.plan_id") AS plan_id,
+  JSON_VALUE(event_properties, "$.previous_plan_id") AS previous_plan_id,
+  JSON_VALUE(event_properties, "$.subscribed_plan_ids") AS subscribed_plan_ids,
   JSON_VALUE(event_properties, "$.product_id") AS product_id,
-  JSON_VALUE(event_properties, "$.promotionCode") AS promotion_code,
+  JSON_VALUE(event_properties, "$.previous_product_id") AS previous_product_id,
+  -- `promotionCode` was renamed `promotion_code` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.promotion_code"),
+    JSON_VALUE(event_properties, "$.promotionCode")
+  ) AS promotion_code,
   JSON_VALUE(event_properties, "$.payment_provider") AS payment_provider,
+  JSON_VALUE(event_properties, "$.provider_event_id") AS provider_event_id,
   JSON_VALUE(event_properties, "$.checkout_type") AS checkout_type,
   JSON_VALUE(event_properties, "$.source_country") AS source_country,
+  -- `source_country` was renamed `country_code_source` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.country_code_source"),
+    JSON_VALUE(event_properties, "$.source_country")
+  ) AS country_code_source,
+  JSON_VALUE(event_properties, "$.error_id") AS error_id,
+  CAST(JSON_VALUE(event_properties, "$.voluntary_cancellation") AS BOOL) AS voluntary_cancellation,
 FROM
   unioned

sql/moz-fx-data-shared-prod/firefox_accounts/fxa_content_auth_stdout_events/view.sql

@@ -32,12 +32,15 @@ WITH content AS (
     jsonPayload.fields.os_name,
     jsonPayload.fields.os_version,
     jsonPayload.fields.country,
+    CAST(NULL AS STRING) AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.user_id,
+    jsonPayload.fields.device_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
   FROM
     `moz-fx-data-shared-prod.firefox_accounts_derived.fxa_content_events_v1`
 ),
@@ -50,12 +53,15 @@ auth AS (
     jsonPayload.fields.os_name,
     jsonPayload.fields.os_version,
     jsonPayload.fields.country,
+    JSON_VALUE(jsonPayload.fields.event_properties, "$.country_code") AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.user_id,
+    jsonPayload.fields.device_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
   FROM
     `moz-fx-data-shared-prod.firefox_accounts_derived.fxa_auth_events_v1`
 ),
@@ -68,12 +74,15 @@ stdout AS (
     jsonPayload.fields.os_name,
     jsonPayload.fields.os_version,
     CAST(NULL AS STRING) AS country,
+    jsonPayload.fields.country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.user_id,
+    jsonPayload.fields.device_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
   FROM
     `moz-fx-data-shared-prod.firefox_accounts_derived.fxa_stdout_events_v1`
 ),
@@ -122,12 +131,28 @@ SELECT
   JSON_VALUE(event_properties, "$.email_service") AS email_service,
   JSON_VALUE(event_properties, "$.email_template") AS email_template,
   JSON_VALUE(event_properties, "$.email_version") AS email_version,
+  JSON_VALUE(event_properties, "$.subscription_id") AS subscription_id,
   JSON_VALUE(event_properties, "$.plan_id") AS plan_id,
+  JSON_VALUE(event_properties, "$.previous_plan_id") AS previous_plan_id,
+  JSON_VALUE(event_properties, "$.subscribed_plan_ids") AS subscribed_plan_ids,
   JSON_VALUE(event_properties, "$.product_id") AS product_id,
-  JSON_VALUE(event_properties, "$.promotionCode") AS promotion_code,
+  JSON_VALUE(event_properties, "$.previous_product_id") AS previous_product_id,
+  -- `promotionCode` was renamed `promotion_code` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.promotion_code"),
+    JSON_VALUE(event_properties, "$.promotionCode")
+  ) AS promotion_code,
   JSON_VALUE(event_properties, "$.payment_provider") AS payment_provider,
+  JSON_VALUE(event_properties, "$.provider_event_id") AS provider_event_id,
   JSON_VALUE(event_properties, "$.checkout_type") AS checkout_type,
   JSON_VALUE(event_properties, "$.source_country") AS source_country,
+  -- `source_country` was renamed `country_code_source` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.country_code_source"),
+    JSON_VALUE(event_properties, "$.source_country")
+  ) AS country_code_source,
+  JSON_VALUE(event_properties, "$.error_id") AS error_id,
+  CAST(JSON_VALUE(event_properties, "$.voluntary_cancellation") AS BOOL) AS voluntary_cancellation,
 FROM
   unioned
 -- Commented out for now, to restore FxA Looker dashboards

sql/moz-fx-data-shared-prod/firefox_accounts/nonprod_fxa_all_events/view.sql

@@ -7,8 +7,10 @@ WITH auth_events AS (
     "auth" AS fxa_server,
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     jsonPayload.fields.country,
+    JSON_VALUE(jsonPayload.fields.event_properties, "$.country_code") AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     jsonPayload.fields.os_name,
@@ -28,8 +30,10 @@ content_events AS (
     "content" AS fxa_server,
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     CAST(NULL AS STRING) AS country,
+    CAST(NULL AS STRING) AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     CAST(NULL AS STRING) AS os_name,
@@ -49,8 +53,10 @@ stdout_events AS (
     "payments" AS fxa_server,
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     CAST(NULL AS STRING) AS country,
+    jsonPayload.fields.country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     jsonPayload.fields.os_name,
@@ -69,8 +75,10 @@ server_events AS (
     fxa_server,
     `timestamp`,
     receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
     jsonPayload.fields.user_id,
     jsonPayload.fields.country,
+    JSON_VALUE(jsonPayload.fields.event_properties, "$.country_code") AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.app_version,
     jsonPayload.fields.os_name,
@@ -110,11 +118,13 @@ SELECT
   fxa_server,
   `timestamp`,
   receiveTimestamp,
+  event_time,
   logger,
   event_type,
   user_id,
   device_id,
   country,
+  country_code,
   `language`,
   app_version,
   os_name,
@@ -148,11 +158,27 @@ SELECT
   JSON_VALUE(event_properties, "$.email_service") AS email_service,
   JSON_VALUE(event_properties, "$.email_template") AS email_template,
   JSON_VALUE(event_properties, "$.email_version") AS email_version,
+  JSON_VALUE(event_properties, "$.subscription_id") AS subscription_id,
   JSON_VALUE(event_properties, "$.plan_id") AS plan_id,
+  JSON_VALUE(event_properties, "$.previous_plan_id") AS previous_plan_id,
+  JSON_VALUE(event_properties, "$.subscribed_plan_ids") AS subscribed_plan_ids,
   JSON_VALUE(event_properties, "$.product_id") AS product_id,
-  JSON_VALUE(event_properties, "$.promotionCode") AS promotion_code,
+  JSON_VALUE(event_properties, "$.previous_product_id") AS previous_product_id,
+  -- `promotionCode` was renamed `promotion_code` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.promotion_code"),
+    JSON_VALUE(event_properties, "$.promotionCode")
+  ) AS promotion_code,
   JSON_VALUE(event_properties, "$.payment_provider") AS payment_provider,
+  JSON_VALUE(event_properties, "$.provider_event_id") AS provider_event_id,
   JSON_VALUE(event_properties, "$.checkout_type") AS checkout_type,
   JSON_VALUE(event_properties, "$.source_country") AS source_country,
+  -- `source_country` was renamed `country_code_source` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.country_code_source"),
+    JSON_VALUE(event_properties, "$.source_country")
+  ) AS country_code_source,
+  JSON_VALUE(event_properties, "$.error_id") AS error_id,
+  CAST(JSON_VALUE(event_properties, "$.voluntary_cancellation") AS BOOL) AS voluntary_cancellation,
 FROM
   unioned

sql/moz-fx-data-shared-prod/firefox_accounts/nonprod_fxa_content_auth_stdout_events/view.sql

@@ -14,12 +14,15 @@ WITH content AS (
     jsonPayload.fields.os_name,
     jsonPayload.fields.os_version,
     jsonPayload.fields.country,
+    CAST(NULL AS STRING) AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.user_id,
+    jsonPayload.fields.device_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
     `timestamp`,
-    receiveTimestamp
+    receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
   FROM
     `moz-fx-data-shared-prod.firefox_accounts_derived.nonprod_fxa_content_events_v1`
 ),
@@ -32,12 +35,15 @@ auth AS (
     jsonPayload.fields.os_name,
     jsonPayload.fields.os_version,
     jsonPayload.fields.country,
+    JSON_VALUE(jsonPayload.fields.event_properties, "$.country_code") AS country_code,
     jsonPayload.fields.language,
     jsonPayload.fields.user_id,
+    jsonPayload.fields.device_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
     `timestamp`,
-    receiveTimestamp
+    receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
   FROM
     `moz-fx-data-shared-prod.firefox_accounts_derived.nonprod_fxa_auth_events_v1`
 ),
@@ -50,12 +56,15 @@ stdout AS (
     jsonPayload.fields.os_name,
     jsonPayload.fields.os_version,
     CAST(NULL AS STRING) AS country,
+    jsonPayload.fields.country_code,
     CAST(NULL AS STRING) AS language,
     jsonPayload.fields.user_id,
+    jsonPayload.fields.device_id,
     jsonPayload.fields.user_properties,
     jsonPayload.fields.event_properties,
     `timestamp`,
-    receiveTimestamp
+    receiveTimestamp,
+    TIMESTAMP_MILLIS(CAST(jsonPayload.fields.time AS INT64)) AS event_time,
   FROM
     `moz-fx-data-shared-prod.firefox_accounts_derived.nonprod_fxa_stdout_events_v1`
 ),
@@ -104,11 +113,27 @@ SELECT
   JSON_VALUE(event_properties, "$.email_service") AS email_service,
   JSON_VALUE(event_properties, "$.email_template") AS email_template,
   JSON_VALUE(event_properties, "$.email_version") AS email_version,
+  JSON_VALUE(event_properties, "$.subscription_id") AS subscription_id,
   JSON_VALUE(event_properties, "$.plan_id") AS plan_id,
+  JSON_VALUE(event_properties, "$.previous_plan_id") AS previous_plan_id,
+  JSON_VALUE(event_properties, "$.subscribed_plan_ids") AS subscribed_plan_ids,
   JSON_VALUE(event_properties, "$.product_id") AS product_id,
-  JSON_VALUE(event_properties, "$.promotionCode") AS promotion_code,
+  JSON_VALUE(event_properties, "$.previous_product_id") AS previous_product_id,
+  -- `promotionCode` was renamed `promotion_code` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.promotion_code"),
+    JSON_VALUE(event_properties, "$.promotionCode")
+  ) AS promotion_code,
   JSON_VALUE(event_properties, "$.payment_provider") AS payment_provider,
+  JSON_VALUE(event_properties, "$.provider_event_id") AS provider_event_id,
   JSON_VALUE(event_properties, "$.checkout_type") AS checkout_type,
   JSON_VALUE(event_properties, "$.source_country") AS source_country,
+  -- `source_country` was renamed `country_code_source` in stdout logs.
+  COALESCE(
+    JSON_VALUE(event_properties, "$.country_code_source"),
+    JSON_VALUE(event_properties, "$.source_country")
+  ) AS country_code_source,
+  JSON_VALUE(event_properties, "$.error_id") AS error_id,
+  CAST(JSON_VALUE(event_properties, "$.voluntary_cancellation") AS BOOL) AS voluntary_cancellation,
 FROM
   unioned

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_oauth_events_v1/metadata.yaml

@@ -4,3 +4,8 @@ description: |-
   [DESCRIPTION_MISSING]
 owners:
 - data-platform-infra-wg@mozilla.com
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v1/query.sql

@@ -49,7 +49,7 @@ WITH windowed AS (
       NOT (event_type = 'fxa_rp - engage' AND service = 'fx-monitor')
     ) OVER w1 = 0 AS monitor_only
   FROM
-    `firefox_accounts.fxa_all_events`
+    `moz-fx-data-shared-prod.firefox_accounts.fxa_all_events`
   WHERE
     fxa_log IN ('auth', 'auth_bounce', 'content', 'oauth')
     AND user_id IS NOT NULL