mozilla
/
bigquery-etl
зеркало из https://github.com/mozilla/bigquery-etl.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

SVCSE-1595 Setup import of tables from staging FxA databases (#4578)

This commit is contained in:
akkomar 2023-11-23 17:45:15 +01:00 коммит произвёл GitHub
Родитель fe0db11287
Коммит a47bcedc28
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
113 изменённых файлов: 1909 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
bqetl_project.yaml
Просмотреть файл

@ -52,6 +52,7 @@ dry_run:
- sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
- sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
- sql/moz-fx-data-shared-prod/accounts_backend/accounts/view.sql
- sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql

14
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/dataset_metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,14 @@
---
friendly_name: Firefox Accounts Databases nonprod (stage) External
description: |-
Data extracted from the nonprod (stage) FxA backend services databases.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa for more information.
Access to this dataset is restricted to accounts-confidential workgroup because some tables here contain sensitive data.
dataset_base_acl: restricted
user_facing: false
labels: {}
workgroup_access:
- role: roles/bigquery.dataViewer
members:
- workgroup:accounts-confidential

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: accountCustomers table from nonprod (stage) fxa database
description: >
A mirror of the `accountCustomers` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

17
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,17 @@
SELECT
TO_HEX(uid) AS uid,
stripeCustomerId,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(updatedAt AS INT)) AS updatedAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
stripeCustomerId,
createdAt,
updatedAt
FROM
fxa.accountCustomers
"""
)

14
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,14 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: stripeCustomerId
type: STRING
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: updatedAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: accountResetTokens table from nonprod (stage) fxa database
description: >
A mirror of the `accountResetTokens` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
TO_HEX(uid) AS uid,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
createdAt
FROM
fxa.accountResetTokens
"""
)

8
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,8 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: createdAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: accounts table from nonprod (stage) fxa database
description: >
A mirror of the `accounts` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

37
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,37 @@
SELECT
TO_HEX(uid) AS uid,
normalizedEmail,
email,
SAFE_CAST(emailVerified AS BOOL) AS emailVerified,
verifierVersion,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifierSetAt AS INT)) AS verifierSetAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
locale,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(lockedAt AS INT)) AS lockedAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(keysChangedAt AS INT)) AS keysChangedAt,
ecosystemAnonId,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(disabledAt AS INT)) AS disabledAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(metricsOptOutAt AS INT)) AS metricsOptOutAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
normalizedEmail,
email,
emailVerified,
verifierVersion,
verifierSetAt,
createdAt,
locale,
lockedAt,
profileChangedAt,
keysChangedAt,
ecosystemAnonId,
disabledAt,
metricsOptOutAt
FROM
fxa.accounts
"""
)

44
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,44 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: normalizedEmail
type: STRING
mode: NULLABLE
- name: email
type: STRING
mode: NULLABLE
- name: emailVerified
type: BOOLEAN
mode: NULLABLE
- name: verifierVersion
type: INTEGER
mode: NULLABLE
- name: verifierSetAt
type: TIMESTAMP
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: locale
type: STRING
mode: NULLABLE
- name: lockedAt
type: TIMESTAMP
mode: NULLABLE
- name: profileChangedAt
type: TIMESTAMP
mode: NULLABLE
- name: keysChangedAt
type: TIMESTAMP
mode: NULLABLE
- name: ecosystemAnonId
type: STRING
mode: NULLABLE
- name: disabledAt
type: TIMESTAMP
mode: NULLABLE
- name: metricsOptOutAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: dbMetadata table from nonprod (stage) fxa database
description: >
A mirror of the `dbMetadata` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
name,
value,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
name,
value
FROM
fxa.dbMetadata
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: name
type: STRING
mode: NULLABLE
- name: value
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: deviceCommandIdentifiers table from nonprod (stage) fxa database
description: >
A mirror of the `deviceCommandIdentifiers` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
commandId,
commandName,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
commandId,
commandName
FROM
fxa.deviceCommandIdentifiers
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: commandId
type: INTEGER
mode: NULLABLE
- name: commandName
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: deviceCommands table from nonprod (stage) fxa database
description: >
A mirror of the `deviceCommands` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

15
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,15 @@
SELECT
TO_HEX(uid) AS uid,
deviceId,
commandId,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
deviceId,
commandId
FROM
fxa.deviceCommands
"""
)

11
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,11 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: deviceId
type: STRING
mode: NULLABLE
- name: commandId
type: INTEGER
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: devices table from nonprod (stage) fxa database
description: >
A mirror of the `devices` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

25
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,25 @@
SELECT
TO_HEX(uid) AS uid,
id,
name,
nameUtf8,
type,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
callbackPublicKey,
SAFE_CAST(callbackIsExpired AS BOOL) AS callbackIsExpired,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
id,
name,
nameUtf8,
type,
createdAt,
callbackPublicKey,
callbackIsExpired
FROM
fxa.devices
"""
)

26
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,26 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: id
type: STRING
mode: NULLABLE
- name: name
type: STRING
mode: NULLABLE
- name: nameUtf8
type: STRING
mode: NULLABLE
- name: type
type: STRING
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: callbackPublicKey
type: STRING
mode: NULLABLE
- name: callbackIsExpired
type: BOOLEAN
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: emailBounces table from nonprod (stage) fxa database
description: >
A mirror of the `emailBounces` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

21
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,21 @@
SELECT
email,
bounceType,
bounceSubType,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
emailTypeId,
diagnosticCode,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
email,
bounceType,
bounceSubType,
createdAt,
emailTypeId,
diagnosticCode
FROM
fxa.emailBounces
"""
)

19
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,19 @@
fields:
- name: email
type: STRING
mode: NULLABLE
- name: bounceType
type: INTEGER
mode: NULLABLE
- name: bounceSubType
type: INTEGER
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: emailTypeId
type: INTEGER
mode: NULLABLE
- name: diagnosticCode
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: emailTypes table from nonprod (stage) fxa database
description: >
A mirror of the `emailTypes` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
id,
emailType,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
id,
emailType
FROM
fxa.emailTypes
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: id
type: INTEGER
mode: NULLABLE
- name: emailType
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: emails table from nonprod (stage) fxa database
description: >
A mirror of the `emails` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

25
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,25 @@
SELECT
id,
normalizedEmail,
email,
TO_HEX(uid) AS uid,
SAFE_CAST(isVerified AS BOOL) AS isVerified,
SAFE_CAST(isPrimary AS BOOL) AS isPrimary,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
id,
normalizedEmail,
email,
uid,
isVerified,
isPrimary,
verifiedAt,
createdAt
FROM
fxa.emails
"""
)

26
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,26 @@
fields:
- name: id
type: INTEGER
mode: NULLABLE
- name: normalizedEmail
type: STRING
mode: NULLABLE
- name: email
type: STRING
mode: NULLABLE
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: isVerified
type: BOOLEAN
mode: NULLABLE
- name: isPrimary
type: BOOLEAN
mode: NULLABLE
- name: verifiedAt
type: TIMESTAMP
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: keyFetchTokens table from nonprod (stage) fxa database
description: >
A mirror of the `keyFetchTokens` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
TO_HEX(uid) AS uid,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
createdAt
FROM
fxa.keyFetchTokens
"""
)

8
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,8 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: createdAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: linkedAccounts table from nonprod (stage) fxa database
description: >
A mirror of the `linkedAccounts` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

19
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,19 @@
SELECT
TO_HEX(uid) AS uid,
id,
providerId,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
SAFE_CAST(enabled AS BOOL) AS enabled,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
id,
providerId,
authAt,
enabled
FROM
fxa.linkedAccounts
"""
)

17
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,17 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: id
type: STRING
mode: NULLABLE
- name: providerId
type: INTEGER
mode: NULLABLE
- name: authAt
type: TIMESTAMP
mode: NULLABLE
- name: enabled
type: BOOLEAN
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: clients table from nonprod (stage) fxa_oauth database
description: >
A mirror of the `clients` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

29
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,29 @@
SELECT
id,
name,
imageUri,
redirectUri,
SAFE_CAST(canGrant AS BOOL) AS canGrant,
SAFE_CAST(publicClient AS BOOL) AS publicClient,
createdAt,
SAFE_CAST(trusted AS BOOL) AS trusted,
allowedScopes,
notes,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
"""SELECT
id,
name,
imageUri,
redirectUri,
canGrant,
publicClient,
createdAt,
trusted,
allowedScopes,
notes
FROM
fxa_oauth.clients
"""
)

31
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,31 @@
fields:
- name: id
type: STRING
mode: NULLABLE
- name: name
type: STRING
mode: NULLABLE
- name: imageUri
type: STRING
mode: NULLABLE
- name: redirectUri
type: STRING
mode: NULLABLE
- name: canGrant
type: BOOLEAN
mode: NULLABLE
- name: publicClient
type: BOOLEAN
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: trusted
type: BOOLEAN
mode: NULLABLE
- name: allowedScopes
type: STRING
mode: NULLABLE
- name: notes
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: codes table from nonprod (stage) fxa_oauth database
description: >
A mirror of the `codes` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

31
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,31 @@
SELECT
clientId,
userId,
scope,
createdAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
amr,
aal,
SAFE_CAST(offline AS BOOL) AS offline,
codeChallengeMethod,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
sessionTokenId,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
"""SELECT
clientId,
userId,
scope,
createdAt,
authAt,
amr,
aal,
offline,
codeChallengeMethod,
profileChangedAt,
sessionTokenId
FROM
fxa_oauth.codes
"""
)

34
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,34 @@
fields:
- name: clientId
type: STRING
mode: NULLABLE
- name: userId
type: STRING
mode: NULLABLE
- name: scope
type: STRING
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: authAt
type: TIMESTAMP
mode: NULLABLE
- name: amr
type: STRING
mode: NULLABLE
- name: aal
type: INTEGER
mode: NULLABLE
- name: offline
type: BOOLEAN
mode: NULLABLE
- name: codeChallengeMethod
type: STRING
mode: NULLABLE
- name: profileChangedAt
type: TIMESTAMP
mode: NULLABLE
- name: sessionTokenId
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: dbMetadata table from nonprod (stage) fxa_oauth database
description: >
A mirror of the `dbMetadata` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
name,
value,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
"""SELECT
name,
value
FROM
fxa_oauth.dbMetadata
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: name
type: STRING
mode: NULLABLE
- name: value
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: refreshTokens table from nonprod (stage) fxa_oauth database
description: >
A mirror of the `refreshTokens` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

21
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,21 @@
SELECT
clientId,
userId,
scope,
createdAt,
lastUsedAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
"""SELECT
clientId,
userId,
scope,
createdAt,
lastUsedAt,
profileChangedAt
FROM
fxa_oauth.refreshTokens
"""
)

19
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,19 @@
fields:
- name: clientId
type: STRING
mode: NULLABLE
- name: userId
type: STRING
mode: NULLABLE
- name: scope
type: STRING
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: lastUsedAt
type: TIMESTAMP
mode: NULLABLE
- name: profileChangedAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: scopes table from nonprod (stage) fxa_oauth database
description: >
A mirror of the `scopes` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
scope,
SAFE_CAST(hasScopedKeys AS BOOL) AS hasScopedKeys,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
"""SELECT
scope,
hasScopedKeys
FROM
fxa_oauth.scopes
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: scope
type: STRING
mode: NULLABLE
- name: hasScopedKeys
type: BOOLEAN
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: tokens table from nonprod (stage) fxa_oauth database
description: >
A mirror of the `tokens` table from the nonprod (stage) `fxa_oauth` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

23
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,23 @@
SELECT
clientId,
userId,
type,
scope,
createdAt,
expiresAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth",
"""SELECT
clientId,
userId,
type,
scope,
createdAt,
expiresAt,
profileChangedAt
FROM
fxa_oauth.tokens
"""
)

22
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,22 @@
fields:
- name: clientId
type: STRING
mode: NULLABLE
- name: userId
type: STRING
mode: NULLABLE
- name: type
type: STRING
mode: NULLABLE
- name: scope
type: STRING
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: expiresAt
type: TIMESTAMP
mode: NULLABLE
- name: profileChangedAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: passwordChangeTokens table from nonprod (stage) fxa database
description: >
A mirror of the `passwordChangeTokens` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
TO_HEX(uid) AS uid,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
createdAt
FROM
fxa.passwordChangeTokens
"""
)

8
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,8 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: createdAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: passwordForgotTokens table from nonprod (stage) fxa database
description: >
A mirror of the `passwordForgotTokens` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

15
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,15 @@
SELECT
TO_HEX(uid) AS uid,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
tries,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
createdAt,
tries
FROM
fxa.passwordForgotTokens
"""
)

11
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,11 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: tries
type: INTEGER
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: paypalCustomers table from nonprod (stage) fxa database
description: >
A mirror of the `paypalCustomers` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

19
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,19 @@
SELECT
TO_HEX(uid) AS uid,
billingAgreementId,
status,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(endedAt AS INT)) AS endedAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
billingAgreementId,
status,
createdAt,
endedAt
FROM
fxa.paypalCustomers
"""
)

17
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,17 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: billingAgreementId
type: STRING
mode: NULLABLE
- name: status
type: STRING
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: endedAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: avatar_providers table from nonprod (stage) fxa_profile database
description: >
A mirror of the `avatar_providers` table from the nonprod (stage) `fxa_profile` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
id,
name,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
"""SELECT
id,
name
FROM
fxa_profile.avatar_providers
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: id
type: INTEGER
mode: NULLABLE
- name: name
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: avatar_selected table from nonprod (stage) fxa_profile database
description: >
A mirror of the `avatar_selected` table from the nonprod (stage) `fxa_profile` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
userId,
avatarId,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
"""SELECT
userId,
avatarId
FROM
fxa_profile.avatar_selected
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: userId
type: STRING
mode: NULLABLE
- name: avatarId
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: avatars table from nonprod (stage) fxa_profile database
description: >
A mirror of the `avatars` table from the nonprod (stage) `fxa_profile` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

17
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,17 @@
SELECT
id,
url,
userId,
providerId,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
"""SELECT
id,
url,
userId,
providerId
FROM
fxa_profile.avatars
"""
)

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,13 @@
fields:
- name: id
type: STRING
mode: NULLABLE
- name: url
type: STRING
mode: NULLABLE
- name: userId
type: STRING
mode: NULLABLE
- name: providerId
type: INTEGER
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: dbMetadata table from nonprod (stage) fxa_profile database
description: >
A mirror of the `dbMetadata` table from the nonprod (stage) `fxa_profile` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
name,
value,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
"""SELECT
name,
value
FROM
fxa_profile.dbMetadata
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: name
type: STRING
mode: NULLABLE
- name: value
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: profile table from nonprod (stage) fxa_profile database
description: >
A mirror of the `profile` table from the nonprod (stage) `fxa_profile` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
userId,
displayName,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile",
"""SELECT
userId,
displayName
FROM
fxa_profile.profile
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: userId
type: STRING
mode: NULLABLE
- name: displayName
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: recoveryCodes table from nonprod (stage) fxa database
description: >
A mirror of the `recoveryCodes` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
id,
TO_HEX(uid) AS uid,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
id,
uid
FROM
fxa.recoveryCodes
"""
)

8
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,8 @@
fields:
- name: id
type: INTEGER
mode: NULLABLE
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: recoveryKeys table from nonprod (stage) fxa database
description: >
A mirror of the `recoveryKeys` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

17
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,17 @@
SELECT
TO_HEX(uid) AS uid,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
SAFE_CAST(enabled AS BOOL) AS enabled,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
createdAt,
verifiedAt,
enabled
FROM
fxa.recoveryKeys
"""
)

14
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,14 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: verifiedAt
type: TIMESTAMP
mode: NULLABLE
- name: enabled
type: BOOLEAN
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: securityEventNames table from nonprod (stage) fxa database
description: >
A mirror of the `securityEventNames` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

13
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,13 @@
SELECT
id,
name,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
id,
name
FROM
fxa.securityEventNames
"""
)

7
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,7 @@
fields:
- name: id
type: INTEGER
mode: NULLABLE
- name: name
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: securityEvents table from nonprod (stage) fxa database
description: >
A mirror of the `securityEvents` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

23
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,23 @@
SELECT
id,
TO_HEX(uid) AS uid,
nameId,
SAFE_CAST(verified AS BOOL) AS verified,
ipAddrHmac,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
tokenVerificationId,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
id,
uid,
nameId,
verified,
ipAddrHmac,
createdAt,
tokenVerificationId
FROM
fxa.securityEvents
"""
)

23
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,23 @@
fields:
- name: id
type: INTEGER
mode: NULLABLE
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: nameId
type: INTEGER
mode: NULLABLE
- name: verified
type: BOOLEAN
mode: NULLABLE
- name: ipAddrHmac
type: STRING
mode: NULLABLE
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: tokenVerificationId
type: STRING
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: sentEmails table from nonprod (stage) fxa database
description: >
A mirror of the `sentEmails` table from the nonprod (stage) `fxa` CloudSQL database,
updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

19
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,19 @@
SELECT
id,
TO_HEX(uid) AS uid,
emailTypeId,
params,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(sentAt AS INT)) AS sentAt,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
id,
uid,
emailTypeId,
params,
sentAt
FROM
fxa.sentEmails
"""
)

17
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,17 @@
fields:
- name: id
type: INTEGER
mode: NULLABLE
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: emailTypeId
type: INTEGER
mode: NULLABLE
- name: params
type: STRING
mode: NULLABLE
- name: sentAt
type: TIMESTAMP
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: sessionTokens table from nonprod (stage) fxa database
description: >
A mirror of the `sessionTokens` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

35
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,35 @@
SELECT
TO_HEX(uid) AS uid,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
uaBrowser,
uaBrowserVersion,
uaOS,
uaOSVersion,
uaDeviceType,
lastAccessTime,
uaFormFactor,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
verificationMethod,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
SAFE_CAST(mustVerify AS BOOL) AS mustVerify,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
createdAt,
uaBrowser,
uaBrowserVersion,
uaOS,
uaOSVersion,
uaDeviceType,
lastAccessTime,
uaFormFactor,
authAt,
verificationMethod,
verifiedAt,
mustVerify
FROM
fxa.sessionTokens
"""
)

41
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,41 @@
fields:
- name: uid
type: STRING
mode: NULLABLE
description: Account ID in hexadecimal format.
- name: createdAt
type: TIMESTAMP
mode: NULLABLE
- name: uaBrowser
type: STRING
mode: NULLABLE
- name: uaBrowserVersion
type: STRING
mode: NULLABLE
- name: uaOS
type: STRING
mode: NULLABLE
- name: uaOSVersion
type: STRING
mode: NULLABLE
- name: uaDeviceType
type: STRING
mode: NULLABLE
- name: lastAccessTime
type: INTEGER
mode: NULLABLE
- name: uaFormFactor
type: STRING
mode: NULLABLE
- name: authAt
type: TIMESTAMP
mode: NULLABLE
- name: verificationMethod
type: INTEGER
mode: NULLABLE
- name: verifiedAt
type: TIMESTAMP
mode: NULLABLE
- name: mustVerify
type: BOOLEAN
mode: NULLABLE

18
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
---
friendly_name: signinCodes table from nonprod (stage) fxa database
description: >
A mirror of the `signinCodes` table from the nonprod (stage) `fxa` CloudSQL database,
excluding columns containing confidential data, updated daily to match the current state of the table.
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
owners:
- akomar@mozilla.com
labels:
application: accounts_backend
schedule: daily
scheduling:
dag_name: bqetl_accounts_backend_external
# destination is the whole table, not a single partition,
# so don't use date_partition_parameter
date_partition_parameter: null
referenced_tables: []

15
sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,15 @@
SELECT
TO_HEX(uid) AS uid,
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
flowId,
FROM
EXTERNAL_QUERY(
"moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa",
"""SELECT
uid,
createdAt,
flowId
FROM
fxa.signinCodes
"""
)

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше