SVCSE-1595 Setup import of tables from staging FxA databases (#4578)

bqetl_project.yaml

@@ -52,6 +52,7 @@ dry_run:
   - sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend/accounts/view.sql
+  - sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/dataset_metadata.yaml

@@ -0,0 +1,14 @@
+---
+friendly_name: Firefox Accounts Databases nonprod (stage) External
+description: |-
+  Data extracted from the nonprod (stage) FxA backend services databases.
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa for more information.
+
+  Access to this dataset is restricted to accounts-confidential workgroup because some tables here contain sensitive data.
+dataset_base_acl: restricted
+user_facing: false
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:accounts-confidential

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accountCustomers table from nonprod (stage) fxa database
+description: >
+  A mirror of the `accountCustomers` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/query.sql

@@ -0,0 +1,17 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  stripeCustomerId,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(updatedAt AS INT)) AS updatedAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         stripeCustomerId,
+         createdAt,
+         updatedAt
+       FROM
+         fxa.accountCustomers
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/schema.yaml

@@ -0,0 +1,14 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: stripeCustomerId
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: updatedAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accountResetTokens table from nonprod (stage) fxa database
+description: >
+  A mirror of the `accountResetTokens` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt
+       FROM
+         fxa.accountResetTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/schema.yaml

@@ -0,0 +1,8 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accounts table from nonprod (stage) fxa database
+description: >
+  A mirror of the `accounts` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/query.sql

@@ -0,0 +1,37 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  normalizedEmail,
+  email,
+  SAFE_CAST(emailVerified AS BOOL) AS emailVerified,
+  verifierVersion,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifierSetAt AS INT)) AS verifierSetAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  locale,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(lockedAt AS INT)) AS lockedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(keysChangedAt AS INT)) AS keysChangedAt,
+  ecosystemAnonId,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(disabledAt AS INT)) AS disabledAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(metricsOptOutAt AS INT)) AS metricsOptOutAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         normalizedEmail,
+         email,
+         emailVerified,
+         verifierVersion,
+         verifierSetAt,
+         createdAt,
+         locale,
+         lockedAt,
+         profileChangedAt,
+         keysChangedAt,
+         ecosystemAnonId,
+         disabledAt,
+         metricsOptOutAt
+       FROM
+         fxa.accounts
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/schema.yaml

@@ -0,0 +1,44 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: normalizedEmail
+  type: STRING
+  mode: NULLABLE
+- name: email
+  type: STRING
+  mode: NULLABLE
+- name: emailVerified
+  type: BOOLEAN
+  mode: NULLABLE
+- name: verifierVersion
+  type: INTEGER
+  mode: NULLABLE
+- name: verifierSetAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: locale
+  type: STRING
+  mode: NULLABLE
+- name: lockedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: profileChangedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: keysChangedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: ecosystemAnonId
+  type: STRING
+  mode: NULLABLE
+- name: disabledAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: metricsOptOutAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: dbMetadata table from nonprod (stage) fxa database
+description: >
+  A mirror of the `dbMetadata` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  name,
+  value,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         name,
+         value
+       FROM
+         fxa.dbMetadata
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: name
+  type: STRING
+  mode: NULLABLE
+- name: value
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: deviceCommandIdentifiers table from nonprod (stage) fxa database
+description: >
+  A mirror of the `deviceCommandIdentifiers` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  commandId,
+  commandName,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         commandId,
+         commandName
+       FROM
+         fxa.deviceCommandIdentifiers
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: commandId
+  type: INTEGER
+  mode: NULLABLE
+- name: commandName
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: deviceCommands table from nonprod (stage) fxa database
+description: >
+  A mirror of the `deviceCommands` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/query.sql

@@ -0,0 +1,15 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  deviceId,
+  commandId,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         deviceId,
+         commandId
+       FROM
+         fxa.deviceCommands
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/schema.yaml

@@ -0,0 +1,11 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: deviceId
+  type: STRING
+  mode: NULLABLE
+- name: commandId
+  type: INTEGER
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: devices table from nonprod (stage) fxa database
+description: >
+  A mirror of the `devices` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/query.sql

@@ -0,0 +1,25 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  id,
+  name,
+  nameUtf8,
+  type,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  callbackPublicKey,
+  SAFE_CAST(callbackIsExpired AS BOOL) AS callbackIsExpired,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         id,
+         name,
+         nameUtf8,
+         type,
+         createdAt,
+         callbackPublicKey,
+         callbackIsExpired
+       FROM
+         fxa.devices
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/schema.yaml

@@ -0,0 +1,26 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: id
+  type: STRING
+  mode: NULLABLE
+- name: name
+  type: STRING
+  mode: NULLABLE
+- name: nameUtf8
+  type: STRING
+  mode: NULLABLE
+- name: type
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: callbackPublicKey
+  type: STRING
+  mode: NULLABLE
+- name: callbackIsExpired
+  type: BOOLEAN
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: emailBounces table from nonprod (stage) fxa database
+description: >
+  A mirror of the `emailBounces` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/query.sql

@@ -0,0 +1,21 @@
+SELECT
+  email,
+  bounceType,
+  bounceSubType,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  emailTypeId,
+  diagnosticCode,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         email,
+         bounceType,
+         bounceSubType,
+         createdAt,
+         emailTypeId,
+         diagnosticCode
+       FROM
+         fxa.emailBounces
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/schema.yaml

@@ -0,0 +1,19 @@
+fields:
+- name: email
+  type: STRING
+  mode: NULLABLE
+- name: bounceType
+  type: INTEGER
+  mode: NULLABLE
+- name: bounceSubType
+  type: INTEGER
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: emailTypeId
+  type: INTEGER
+  mode: NULLABLE
+- name: diagnosticCode
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: emailTypes table from nonprod (stage) fxa database
+description: >
+  A mirror of the `emailTypes` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  id,
+  emailType,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         id,
+         emailType
+       FROM
+         fxa.emailTypes
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: emailType
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: emails table from nonprod (stage) fxa database
+description: >
+  A mirror of the `emails` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/query.sql

@@ -0,0 +1,25 @@
+SELECT
+  id,
+  normalizedEmail,
+  email,
+  TO_HEX(uid) AS uid,
+  SAFE_CAST(isVerified AS BOOL) AS isVerified,
+  SAFE_CAST(isPrimary AS BOOL) AS isPrimary,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         id,
+         normalizedEmail,
+         email,
+         uid,
+         isVerified,
+         isPrimary,
+         verifiedAt,
+         createdAt
+       FROM
+         fxa.emails
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/schema.yaml

@@ -0,0 +1,26 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: normalizedEmail
+  type: STRING
+  mode: NULLABLE
+- name: email
+  type: STRING
+  mode: NULLABLE
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: isVerified
+  type: BOOLEAN
+  mode: NULLABLE
+- name: isPrimary
+  type: BOOLEAN
+  mode: NULLABLE
+- name: verifiedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: keyFetchTokens table from nonprod (stage) fxa database
+description: >
+  A mirror of the `keyFetchTokens` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt
+       FROM
+         fxa.keyFetchTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/schema.yaml

@@ -0,0 +1,8 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: linkedAccounts table from nonprod (stage) fxa database
+description: >
+  A mirror of the `linkedAccounts` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/query.sql

@@ -0,0 +1,19 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  id,
+  providerId,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
+  SAFE_CAST(enabled AS BOOL) AS enabled,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         id,
+         providerId,
+         authAt,
+         enabled
+       FROM
+         fxa.linkedAccounts
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/schema.yaml

@@ -0,0 +1,17 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: id
+  type: STRING
+  mode: NULLABLE
+- name: providerId
+  type: INTEGER
+  mode: NULLABLE
+- name: authAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: enabled
+  type: BOOLEAN
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: clients table from nonprod (stage) fxa_oauth database
+description: >
+  A mirror of the `clients` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/query.sql

@@ -0,0 +1,29 @@
+SELECT
+  id,
+  name,
+  imageUri,
+  redirectUri,
+  SAFE_CAST(canGrant AS BOOL) AS canGrant,
+  SAFE_CAST(publicClient AS BOOL) AS publicClient,
+  createdAt,
+  SAFE_CAST(trusted AS BOOL) AS trusted,
+  allowedScopes,
+  notes,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
+    """SELECT
+         id,
+         name,
+         imageUri,
+         redirectUri,
+         canGrant,
+         publicClient,
+         createdAt,
+         trusted,
+         allowedScopes,
+         notes
+       FROM
+         fxa_oauth.clients
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/schema.yaml

@@ -0,0 +1,31 @@
+fields:
+- name: id
+  type: STRING
+  mode: NULLABLE
+- name: name
+  type: STRING
+  mode: NULLABLE
+- name: imageUri
+  type: STRING
+  mode: NULLABLE
+- name: redirectUri
+  type: STRING
+  mode: NULLABLE
+- name: canGrant
+  type: BOOLEAN
+  mode: NULLABLE
+- name: publicClient
+  type: BOOLEAN
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: trusted
+  type: BOOLEAN
+  mode: NULLABLE
+- name: allowedScopes
+  type: STRING
+  mode: NULLABLE
+- name: notes
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: codes table from nonprod (stage) fxa_oauth database
+description: >
+  A mirror of the `codes` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/query.sql

@@ -0,0 +1,31 @@
+SELECT
+  clientId,
+  userId,
+  scope,
+  createdAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
+  amr,
+  aal,
+  SAFE_CAST(offline AS BOOL) AS offline,
+  codeChallengeMethod,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
+  sessionTokenId,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
+    """SELECT
+         clientId,
+         userId,
+         scope,
+         createdAt,
+         authAt,
+         amr,
+         aal,
+         offline,
+         codeChallengeMethod,
+         profileChangedAt,
+         sessionTokenId
+       FROM
+         fxa_oauth.codes
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/schema.yaml

@@ -0,0 +1,34 @@
+fields:
+- name: clientId
+  type: STRING
+  mode: NULLABLE
+- name: userId
+  type: STRING
+  mode: NULLABLE
+- name: scope
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: authAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: amr
+  type: STRING
+  mode: NULLABLE
+- name: aal
+  type: INTEGER
+  mode: NULLABLE
+- name: offline
+  type: BOOLEAN
+  mode: NULLABLE
+- name: codeChallengeMethod
+  type: STRING
+  mode: NULLABLE
+- name: profileChangedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: sessionTokenId
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: dbMetadata table from nonprod (stage) fxa_oauth database
+description: >
+  A mirror of the `dbMetadata` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  name,
+  value,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
+    """SELECT
+         name,
+         value
+       FROM
+         fxa_oauth.dbMetadata
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: name
+  type: STRING
+  mode: NULLABLE
+- name: value
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: refreshTokens table from nonprod (stage) fxa_oauth database
+description: >
+  A mirror of the `refreshTokens` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/query.sql

@@ -0,0 +1,21 @@
+SELECT
+  clientId,
+  userId,
+  scope,
+  createdAt,
+  lastUsedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
+    """SELECT
+         clientId,
+         userId,
+         scope,
+         createdAt,
+         lastUsedAt,
+         profileChangedAt
+       FROM
+         fxa_oauth.refreshTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/schema.yaml

@@ -0,0 +1,19 @@
+fields:
+- name: clientId
+  type: STRING
+  mode: NULLABLE
+- name: userId
+  type: STRING
+  mode: NULLABLE
+- name: scope
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: lastUsedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: profileChangedAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: scopes table from nonprod (stage) fxa_oauth database
+description: >
+  A mirror of the `scopes` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  scope,
+  SAFE_CAST(hasScopedKeys AS BOOL) AS hasScopedKeys,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
+    """SELECT
+         scope,
+         hasScopedKeys
+       FROM
+         fxa_oauth.scopes
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: scope
+  type: STRING
+  mode: NULLABLE
+- name: hasScopedKeys
+  type: BOOLEAN
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: tokens table from nonprod (stage) fxa_oauth database
+description: >
+  A mirror of the `tokens` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/query.sql

@@ -0,0 +1,23 @@
+SELECT
+  clientId,
+  userId,
+  type,
+  scope,
+  createdAt,
+  expiresAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
+    """SELECT
+         clientId,
+         userId,
+         type,
+         scope,
+         createdAt,
+         expiresAt,
+         profileChangedAt
+       FROM
+         fxa_oauth.tokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/schema.yaml

@@ -0,0 +1,22 @@
+fields:
+- name: clientId
+  type: STRING
+  mode: NULLABLE
+- name: userId
+  type: STRING
+  mode: NULLABLE
+- name: type
+  type: STRING
+  mode: NULLABLE
+- name: scope
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: expiresAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: profileChangedAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: passwordChangeTokens table from nonprod (stage) fxa database
+description: >
+  A mirror of the `passwordChangeTokens` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt
+       FROM
+         fxa.passwordChangeTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/schema.yaml

@@ -0,0 +1,8 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: passwordForgotTokens table from nonprod (stage) fxa database
+description: >
+  A mirror of the `passwordForgotTokens` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/query.sql

@@ -0,0 +1,15 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  tries,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt,
+         tries
+       FROM
+         fxa.passwordForgotTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/schema.yaml

@@ -0,0 +1,11 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: tries
+  type: INTEGER
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: paypalCustomers table from nonprod (stage) fxa database
+description: >
+  A mirror of the `paypalCustomers` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/query.sql

@@ -0,0 +1,19 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  billingAgreementId,
+  status,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(endedAt AS INT)) AS endedAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         billingAgreementId,
+         status,
+         createdAt,
+         endedAt
+       FROM
+         fxa.paypalCustomers
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/schema.yaml

@@ -0,0 +1,17 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: billingAgreementId
+  type: STRING
+  mode: NULLABLE
+- name: status
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: endedAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: avatar_providers table from nonprod (stage) fxa_profile database
+description: >
+  A mirror of the `avatar_providers` table from the nonprod (stage) `fxa_profile` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  id,
+  name,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
+    """SELECT
+         id,
+         name
+       FROM
+         fxa_profile.avatar_providers
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: name
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: avatar_selected table from nonprod (stage) fxa_profile database
+description: >
+  A mirror of the `avatar_selected` table from the nonprod (stage) `fxa_profile` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  userId,
+  avatarId,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
+    """SELECT
+         userId,
+         avatarId
+       FROM
+         fxa_profile.avatar_selected
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: userId
+  type: STRING
+  mode: NULLABLE
+- name: avatarId
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: avatars table from nonprod (stage) fxa_profile database
+description: >
+  A mirror of the `avatars` table from the nonprod (stage) `fxa_profile` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/query.sql

@@ -0,0 +1,17 @@
+SELECT
+  id,
+  url,
+  userId,
+  providerId,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
+    """SELECT
+         id,
+         url,
+         userId,
+         providerId
+       FROM
+         fxa_profile.avatars
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/schema.yaml

@@ -0,0 +1,13 @@
+fields:
+- name: id
+  type: STRING
+  mode: NULLABLE
+- name: url
+  type: STRING
+  mode: NULLABLE
+- name: userId
+  type: STRING
+  mode: NULLABLE
+- name: providerId
+  type: INTEGER
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: dbMetadata table from nonprod (stage) fxa_profile database
+description: >
+  A mirror of the `dbMetadata` table from the nonprod (stage) `fxa_profile` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  name,
+  value,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
+    """SELECT
+         name,
+         value
+       FROM
+         fxa_profile.dbMetadata
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: name
+  type: STRING
+  mode: NULLABLE
+- name: value
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: profile table from nonprod (stage) fxa_profile database
+description: >
+  A mirror of the `profile` table from the nonprod (stage) `fxa_profile` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  userId,
+  displayName,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
+    """SELECT
+         userId,
+         displayName
+       FROM
+         fxa_profile.profile
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: userId
+  type: STRING
+  mode: NULLABLE
+- name: displayName
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: recoveryCodes table from nonprod (stage) fxa database
+description: >
+  A mirror of the `recoveryCodes` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  id,
+  TO_HEX(uid) AS uid,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         id,
+         uid
+       FROM
+         fxa.recoveryCodes
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/schema.yaml

@@ -0,0 +1,8 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: recoveryKeys table from nonprod (stage) fxa database
+description: >
+  A mirror of the `recoveryKeys` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/query.sql

@@ -0,0 +1,17 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
+  SAFE_CAST(enabled AS BOOL) AS enabled,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt,
+         verifiedAt,
+         enabled
+       FROM
+         fxa.recoveryKeys
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/schema.yaml

@@ -0,0 +1,14 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: verifiedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: enabled
+  type: BOOLEAN
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: securityEventNames table from nonprod (stage) fxa database
+description: >
+  A mirror of the `securityEventNames` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  id,
+  name,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         id,
+         name
+       FROM
+         fxa.securityEventNames
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: name
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: securityEvents table from nonprod (stage) fxa database
+description: >
+  A mirror of the `securityEvents` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/query.sql

@@ -0,0 +1,23 @@
+SELECT
+  id,
+  TO_HEX(uid) AS uid,
+  nameId,
+  SAFE_CAST(verified AS BOOL) AS verified,
+  ipAddrHmac,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  tokenVerificationId,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         id,
+         uid,
+         nameId,
+         verified,
+         ipAddrHmac,
+         createdAt,
+         tokenVerificationId
+       FROM
+         fxa.securityEvents
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/schema.yaml

@@ -0,0 +1,23 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: nameId
+  type: INTEGER
+  mode: NULLABLE
+- name: verified
+  type: BOOLEAN
+  mode: NULLABLE
+- name: ipAddrHmac
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: tokenVerificationId
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: sentEmails table from nonprod (stage) fxa database
+description: >
+  A mirror of the `sentEmails` table from the nonprod (stage) `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/query.sql

@@ -0,0 +1,19 @@
+SELECT
+  id,
+  TO_HEX(uid) AS uid,
+  emailTypeId,
+  params,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(sentAt AS INT)) AS sentAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         id,
+         uid,
+         emailTypeId,
+         params,
+         sentAt
+       FROM
+         fxa.sentEmails
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/schema.yaml

@@ -0,0 +1,17 @@
+fields:
+- name: id
+  type: INTEGER
+  mode: NULLABLE
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: emailTypeId
+  type: INTEGER
+  mode: NULLABLE
+- name: params
+  type: STRING
+  mode: NULLABLE
+- name: sentAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: sessionTokens table from nonprod (stage) fxa database
+description: >
+  A mirror of the `sessionTokens` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/query.sql

@@ -0,0 +1,35 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  uaBrowser,
+  uaBrowserVersion,
+  uaOS,
+  uaOSVersion,
+  uaDeviceType,
+  lastAccessTime,
+  uaFormFactor,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
+  verificationMethod,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
+  SAFE_CAST(mustVerify AS BOOL) AS mustVerify,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt,
+         uaBrowser,
+         uaBrowserVersion,
+         uaOS,
+         uaOSVersion,
+         uaDeviceType,
+         lastAccessTime,
+         uaFormFactor,
+         authAt,
+         verificationMethod,
+         verifiedAt,
+         mustVerify
+       FROM
+         fxa.sessionTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/schema.yaml

@@ -0,0 +1,41 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: uaBrowser
+  type: STRING
+  mode: NULLABLE
+- name: uaBrowserVersion
+  type: STRING
+  mode: NULLABLE
+- name: uaOS
+  type: STRING
+  mode: NULLABLE
+- name: uaOSVersion
+  type: STRING
+  mode: NULLABLE
+- name: uaDeviceType
+  type: STRING
+  mode: NULLABLE
+- name: lastAccessTime
+  type: INTEGER
+  mode: NULLABLE
+- name: uaFormFactor
+  type: STRING
+  mode: NULLABLE
+- name: authAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: verificationMethod
+  type: INTEGER
+  mode: NULLABLE
+- name: verifiedAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: mustVerify
+  type: BOOLEAN
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: signinCodes table from nonprod (stage) fxa database
+description: >
+  A mirror of the `signinCodes` table from the nonprod (stage) `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - akomar@mozilla.com
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/query.sql

@@ -0,0 +1,15 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  flowId,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
+    """SELECT
+         uid,
+         createdAt,
+         flowId
+       FROM
+         fxa.signinCodes
+    """
+  )