DENG-1879 Setup import of emails table from FxA stage CloudSQL (#4493)

bqetl_project.yaml

@@ -47,6 +47,7 @@ dry_run:
   - sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau/view.sql
   - sql/moz-fx-data-shared-prod/telemetry/buildhub2/view.sql
   - sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_accounts_v1/query.sql
+  - sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend/nonprod_accounts/view.sql
   - sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend/accounts/view.sql

dags/bqetl_accounts_backend_external.py

@@ -69,3 +69,15 @@ with DAG(
         depends_on_past=False,
         task_concurrency=1,
     )
+
+    accounts_backend_external__nonprod_emails__v1 = bigquery_etl_query(
+        task_id="accounts_backend_external__nonprod_emails__v1",
+        destination_table="nonprod_emails_v1",
+        dataset_id="accounts_backend_external",
+        project_id="moz-fx-data-shared-prod",
+        owner="akomar@mozilla.com",
+        email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )

sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/metadata.yaml

@@ -0,0 +1,22 @@
+---
+friendly_name: Emails table from FxA database in stage environment
+description: |-
+  A mirror of the `emails` table from the staging FxA CloudSQL database,
+  updated daily to match the current state of the table.
+  Some fields in this table are converted to a more user-friendly, BigQuery-native format:
+    - `uid` is converted from bytes to a hex string
+    - boolean integer columns are converted to BOOL
+    - timestamp columns are converted to TIMESTAMP
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/query.sql

@@ -0,0 +1,25 @@
+SELECT
+  id,
+  normalizedEmail,
+  email,
+  TO_HEX(uid) AS uid,
+  SAFE_CAST(isVerified AS BOOL) AS isVerified,
+  SAFE_CAST(isPrimary AS BOOL) AS isPrimary,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         id,
+         normalizedEmail,
+         email,
+         uid,
+         isVerified,
+         isPrimary,
+         verifiedAt,
+         createdAt
+       FROM
+         emails
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/schema.yaml

@@ -0,0 +1,29 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: normalizedEmail
+  type: STRING
+  mode: NULLABLE
+- name: email
+  type: STRING
+  mode: NULLABLE
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: |-
+    Account ID in hexadecimal format.
+    FxA stores this as bytes, for purposes of logging or integration
+    with other systems we convert it to a hex string.
+- name: isVerified
+  type: BOOLEAN
+  mode: NULLABLE
+- name: isPrimary
+  type: BOOLEAN
+  mode: NULLABLE
+- name: verifiedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE