SVCSE-1595 Setup import of tables from production FxA databases (#4597)
This commit is contained in:
akkomar
GitHub
Родитель
ec297972c6
Коммит
e1a94c9e4a
|
|
@ -52,6 +52,7 @@ dry_run:
|
|||
- sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/accounts_backend/accounts/view.sql
|
||||
- sql/moz-fx-data-shared-prod/accounts_db_external/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
---
|
||||
friendly_name: Firefox Accounts Databases production External
|
||||
description: |-
|
||||
Data extracted from the production FxA backend services databases.
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa for more information.
|
||||
|
||||
Access to this dataset is restricted to accounts-confidential workgroup because some tables here contain sensitive data.
|
||||
dataset_base_acl: restricted
|
||||
user_facing: false
|
||||
labels: {}
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:accounts-confidential
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: accountCustomers table from production fxa database
|
||||
description: >
|
||||
A mirror of the `accountCustomers` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
stripeCustomerId,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(updatedAt AS INT)) AS updatedAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
stripeCustomerId,
|
||||
createdAt,
|
||||
updatedAt
|
||||
FROM
|
||||
fxa.accountCustomers
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: stripeCustomerId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: updatedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: accountResetTokens table from production fxa database
|
||||
description: >
|
||||
A mirror of the `accountResetTokens` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
createdAt
|
||||
FROM
|
||||
fxa.accountResetTokens
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: accounts table from production fxa database
|
||||
description: >
|
||||
A mirror of the `accounts` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
normalizedEmail,
|
||||
email,
|
||||
SAFE_CAST(emailVerified AS BOOL) AS emailVerified,
|
||||
verifierVersion,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifierSetAt AS INT)) AS verifierSetAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
locale,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(lockedAt AS INT)) AS lockedAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(keysChangedAt AS INT)) AS keysChangedAt,
|
||||
ecosystemAnonId,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(disabledAt AS INT)) AS disabledAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(metricsOptOutAt AS INT)) AS metricsOptOutAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
normalizedEmail,
|
||||
email,
|
||||
emailVerified,
|
||||
verifierVersion,
|
||||
verifierSetAt,
|
||||
createdAt,
|
||||
locale,
|
||||
lockedAt,
|
||||
profileChangedAt,
|
||||
keysChangedAt,
|
||||
ecosystemAnonId,
|
||||
disabledAt,
|
||||
metricsOptOutAt
|
||||
FROM
|
||||
fxa.accounts
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: normalizedEmail
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: emailVerified
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: verifierVersion
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: verifierSetAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: locale
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: lockedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: profileChangedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: keysChangedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: ecosystemAnonId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: disabledAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: metricsOptOutAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: dbMetadata table from production fxa database
|
||||
description: >
|
||||
A mirror of the `dbMetadata` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
name,
|
||||
value,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
name,
|
||||
value
|
||||
FROM
|
||||
fxa.dbMetadata
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: value
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: deviceCommandIdentifiers table from production fxa database
|
||||
description: >
|
||||
A mirror of the `deviceCommandIdentifiers` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
commandId,
|
||||
commandName,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
commandId,
|
||||
commandName
|
||||
FROM
|
||||
fxa.deviceCommandIdentifiers
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: commandId
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: commandName
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: deviceCommands table from production fxa database
|
||||
description: >
|
||||
A mirror of the `deviceCommands` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
deviceId,
|
||||
commandId,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
deviceId,
|
||||
commandId
|
||||
FROM
|
||||
fxa.deviceCommands
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: deviceId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: commandId
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: devices table from production fxa database
|
||||
description: >
|
||||
A mirror of the `devices` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
id,
|
||||
name,
|
||||
nameUtf8,
|
||||
type,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
callbackPublicKey,
|
||||
SAFE_CAST(callbackIsExpired AS BOOL) AS callbackIsExpired,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
id,
|
||||
name,
|
||||
nameUtf8,
|
||||
type,
|
||||
createdAt,
|
||||
callbackPublicKey,
|
||||
callbackIsExpired
|
||||
FROM
|
||||
fxa.devices
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: nameUtf8
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: type
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: callbackPublicKey
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: callbackIsExpired
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: emailBounces table from production fxa database
|
||||
description: >
|
||||
A mirror of the `emailBounces` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
SELECT
|
||||
email,
|
||||
bounceType,
|
||||
bounceSubType,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
emailTypeId,
|
||||
diagnosticCode,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
email,
|
||||
bounceType,
|
||||
bounceSubType,
|
||||
createdAt,
|
||||
emailTypeId,
|
||||
diagnosticCode
|
||||
FROM
|
||||
fxa.emailBounces
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
fields:
|
||||
- name: email
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: bounceType
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: bounceSubType
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: emailTypeId
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: diagnosticCode
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: emailTypes table from production fxa database
|
||||
description: >
|
||||
A mirror of the `emailTypes` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
id,
|
||||
emailType,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
id,
|
||||
emailType
|
||||
FROM
|
||||
fxa.emailTypes
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: emailType
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: emails table from production fxa database
|
||||
description: >
|
||||
A mirror of the `emails` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
SELECT
|
||||
id,
|
||||
normalizedEmail,
|
||||
email,
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE_CAST(isVerified AS BOOL) AS isVerified,
|
||||
SAFE_CAST(isPrimary AS BOOL) AS isPrimary,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
id,
|
||||
normalizedEmail,
|
||||
email,
|
||||
uid,
|
||||
isVerified,
|
||||
isPrimary,
|
||||
verifiedAt,
|
||||
createdAt
|
||||
FROM
|
||||
fxa.emails
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: normalizedEmail
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: isVerified
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: isPrimary
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: verifiedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: keyFetchTokens table from production fxa database
|
||||
description: >
|
||||
A mirror of the `keyFetchTokens` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
createdAt
|
||||
FROM
|
||||
fxa.keyFetchTokens
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: linkedAccounts table from production fxa database
|
||||
description: >
|
||||
A mirror of the `linkedAccounts` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
id,
|
||||
providerId,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
|
||||
SAFE_CAST(enabled AS BOOL) AS enabled,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
id,
|
||||
providerId,
|
||||
authAt,
|
||||
enabled
|
||||
FROM
|
||||
fxa.linkedAccounts
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: providerId
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: authAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: enabled
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: clients table from production fxa_oauth database
|
||||
description: >
|
||||
A mirror of the `clients` table from the production `fxa_oauth` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
SELECT
|
||||
id,
|
||||
name,
|
||||
imageUri,
|
||||
redirectUri,
|
||||
SAFE_CAST(canGrant AS BOOL) AS canGrant,
|
||||
SAFE_CAST(publicClient AS BOOL) AS publicClient,
|
||||
createdAt,
|
||||
SAFE_CAST(trusted AS BOOL) AS trusted,
|
||||
allowedScopes,
|
||||
notes,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-oauth-prod-prod-fxa-oauth",
|
||||
"""SELECT
|
||||
id,
|
||||
name,
|
||||
imageUri,
|
||||
redirectUri,
|
||||
canGrant,
|
||||
publicClient,
|
||||
createdAt,
|
||||
trusted,
|
||||
allowedScopes,
|
||||
notes
|
||||
FROM
|
||||
fxa_oauth.clients
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: imageUri
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: redirectUri
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: canGrant
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: publicClient
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: trusted
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: allowedScopes
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: notes
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: codes table from production fxa_oauth database
|
||||
description: >
|
||||
A mirror of the `codes` table from the production `fxa_oauth` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
SELECT
|
||||
clientId,
|
||||
userId,
|
||||
scope,
|
||||
createdAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
|
||||
amr,
|
||||
aal,
|
||||
SAFE_CAST(offline AS BOOL) AS offline,
|
||||
codeChallengeMethod,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
|
||||
sessionTokenId,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-oauth-prod-prod-fxa-oauth",
|
||||
"""SELECT
|
||||
clientId,
|
||||
userId,
|
||||
scope,
|
||||
createdAt,
|
||||
authAt,
|
||||
amr,
|
||||
aal,
|
||||
offline,
|
||||
codeChallengeMethod,
|
||||
profileChangedAt,
|
||||
sessionTokenId
|
||||
FROM
|
||||
fxa_oauth.codes
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
fields:
|
||||
- name: clientId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: userId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: scope
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: authAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: amr
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: aal
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: offline
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: codeChallengeMethod
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: profileChangedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: sessionTokenId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: dbMetadata table from production fxa_oauth database
|
||||
description: >
|
||||
A mirror of the `dbMetadata` table from the production `fxa_oauth` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
name,
|
||||
value,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-oauth-prod-prod-fxa-oauth",
|
||||
"""SELECT
|
||||
name,
|
||||
value
|
||||
FROM
|
||||
fxa_oauth.dbMetadata
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: value
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: refreshTokens table from production fxa_oauth database
|
||||
description: >
|
||||
A mirror of the `refreshTokens` table from the production `fxa_oauth` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
SELECT
|
||||
clientId,
|
||||
userId,
|
||||
scope,
|
||||
createdAt,
|
||||
lastUsedAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-oauth-prod-prod-fxa-oauth",
|
||||
"""SELECT
|
||||
clientId,
|
||||
userId,
|
||||
scope,
|
||||
createdAt,
|
||||
lastUsedAt,
|
||||
profileChangedAt
|
||||
FROM
|
||||
fxa_oauth.refreshTokens
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
fields:
|
||||
- name: clientId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: userId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: scope
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: lastUsedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: profileChangedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: scopes table from production fxa_oauth database
|
||||
description: >
|
||||
A mirror of the `scopes` table from the production `fxa_oauth` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
scope,
|
||||
SAFE_CAST(hasScopedKeys AS BOOL) AS hasScopedKeys,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-oauth-prod-prod-fxa-oauth",
|
||||
"""SELECT
|
||||
scope,
|
||||
hasScopedKeys
|
||||
FROM
|
||||
fxa_oauth.scopes
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: scope
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: hasScopedKeys
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: tokens table from production fxa_oauth database
|
||||
description: >
|
||||
A mirror of the `tokens` table from the production `fxa_oauth` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
SELECT
|
||||
clientId,
|
||||
userId,
|
||||
type,
|
||||
scope,
|
||||
createdAt,
|
||||
expiresAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-oauth-prod-prod-fxa-oauth",
|
||||
"""SELECT
|
||||
clientId,
|
||||
userId,
|
||||
type,
|
||||
scope,
|
||||
createdAt,
|
||||
expiresAt,
|
||||
profileChangedAt
|
||||
FROM
|
||||
fxa_oauth.tokens
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
fields:
|
||||
- name: clientId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: userId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: type
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: scope
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: expiresAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: profileChangedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: passwordChangeTokens table from production fxa database
|
||||
description: >
|
||||
A mirror of the `passwordChangeTokens` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
createdAt
|
||||
FROM
|
||||
fxa.passwordChangeTokens
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: passwordForgotTokens table from production fxa database
|
||||
description: >
|
||||
A mirror of the `passwordForgotTokens` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
tries,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
createdAt,
|
||||
tries
|
||||
FROM
|
||||
fxa.passwordForgotTokens
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: tries
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: paypalCustomers table from production fxa database
|
||||
description: >
|
||||
A mirror of the `paypalCustomers` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
billingAgreementId,
|
||||
status,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(endedAt AS INT)) AS endedAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
billingAgreementId,
|
||||
status,
|
||||
createdAt,
|
||||
endedAt
|
||||
FROM
|
||||
fxa.paypalCustomers
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: billingAgreementId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: status
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: endedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: avatar_providers table from production fxa_profile database
|
||||
description: >
|
||||
A mirror of the `avatar_providers` table from the production `fxa_profile` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
id,
|
||||
name,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa-profile",
|
||||
"""SELECT
|
||||
id,
|
||||
name
|
||||
FROM
|
||||
fxa_profile.avatar_providers
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: avatar_selected table from production fxa_profile database
|
||||
description: >
|
||||
A mirror of the `avatar_selected` table from the production `fxa_profile` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
userId,
|
||||
avatarId,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa-profile",
|
||||
"""SELECT
|
||||
userId,
|
||||
avatarId
|
||||
FROM
|
||||
fxa_profile.avatar_selected
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: userId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: avatarId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: avatars table from production fxa_profile database
|
||||
description: >
|
||||
A mirror of the `avatars` table from the production `fxa_profile` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
SELECT
|
||||
id,
|
||||
url,
|
||||
userId,
|
||||
providerId,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa-profile",
|
||||
"""SELECT
|
||||
id,
|
||||
url,
|
||||
userId,
|
||||
providerId
|
||||
FROM
|
||||
fxa_profile.avatars
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: url
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: userId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: providerId
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: dbMetadata table from production fxa_profile database
|
||||
description: >
|
||||
A mirror of the `dbMetadata` table from the production `fxa_profile` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
name,
|
||||
value,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa-profile",
|
||||
"""SELECT
|
||||
name,
|
||||
value
|
||||
FROM
|
||||
fxa_profile.dbMetadata
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: value
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: profile table from production fxa_profile database
|
||||
description: >
|
||||
A mirror of the `profile` table from the production `fxa_profile` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
userId,
|
||||
displayName,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa-profile",
|
||||
"""SELECT
|
||||
userId,
|
||||
displayName
|
||||
FROM
|
||||
fxa_profile.profile
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: userId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: displayName
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: recoveryCodes table from production fxa database
|
||||
description: >
|
||||
A mirror of the `recoveryCodes` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid
|
||||
FROM
|
||||
fxa.recoveryCodes
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: recoveryKeys table from production fxa database
|
||||
description: >
|
||||
A mirror of the `recoveryKeys` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
|
||||
SAFE_CAST(enabled AS BOOL) AS enabled,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
createdAt,
|
||||
verifiedAt,
|
||||
enabled
|
||||
FROM
|
||||
fxa.recoveryKeys
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: verifiedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: enabled
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: securityEventNames table from production fxa database
|
||||
description: >
|
||||
A mirror of the `securityEventNames` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
SELECT
|
||||
id,
|
||||
name,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
id,
|
||||
name
|
||||
FROM
|
||||
fxa.securityEventNames
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: securityEvents table from production fxa database
|
||||
description: >
|
||||
A mirror of the `securityEvents` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
SELECT
|
||||
id,
|
||||
TO_HEX(uid) AS uid,
|
||||
nameId,
|
||||
SAFE_CAST(verified AS BOOL) AS verified,
|
||||
ipAddrHmac,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
tokenVerificationId,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
id,
|
||||
uid,
|
||||
nameId,
|
||||
verified,
|
||||
ipAddrHmac,
|
||||
createdAt,
|
||||
tokenVerificationId
|
||||
FROM
|
||||
fxa.securityEvents
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: nameId
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: verified
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
- name: ipAddrHmac
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: tokenVerificationId
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: sentEmails table from production fxa database
|
||||
description: >
|
||||
A mirror of the `sentEmails` table from the production `fxa` CloudSQL database,
|
||||
updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
SELECT
|
||||
id,
|
||||
TO_HEX(uid) AS uid,
|
||||
emailTypeId,
|
||||
params,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(sentAt AS INT)) AS sentAt,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
id,
|
||||
uid,
|
||||
emailTypeId,
|
||||
params,
|
||||
sentAt
|
||||
FROM
|
||||
fxa.sentEmails
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
fields:
|
||||
- name: id
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: emailTypeId
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: params
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: sentAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: sessionTokens table from production fxa database
|
||||
description: >
|
||||
A mirror of the `sessionTokens` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
uaBrowser,
|
||||
uaBrowserVersion,
|
||||
uaOS,
|
||||
uaOSVersion,
|
||||
uaDeviceType,
|
||||
lastAccessTime,
|
||||
uaFormFactor,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt,
|
||||
verificationMethod,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt,
|
||||
SAFE_CAST(mustVerify AS BOOL) AS mustVerify,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
createdAt,
|
||||
uaBrowser,
|
||||
uaBrowserVersion,
|
||||
uaOS,
|
||||
uaOSVersion,
|
||||
uaDeviceType,
|
||||
lastAccessTime,
|
||||
uaFormFactor,
|
||||
authAt,
|
||||
verificationMethod,
|
||||
verifiedAt,
|
||||
mustVerify
|
||||
FROM
|
||||
fxa.sessionTokens
|
||||
"""
|
||||
)
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
fields:
|
||||
- name: uid
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Account ID in hexadecimal format.
|
||||
- name: createdAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: uaBrowser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: uaBrowserVersion
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: uaOS
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: uaOSVersion
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: uaDeviceType
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: lastAccessTime
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: uaFormFactor
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: authAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: verificationMethod
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: verifiedAt
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: mustVerify
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
friendly_name: signinCodes table from production fxa database
|
||||
description: >
|
||||
A mirror of the `signinCodes` table from the production `fxa` CloudSQL database,
|
||||
excluding columns containing confidential data, updated daily to match the current state of the table.
|
||||
|
||||
See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
|
||||
owners:
|
||||
- akomar@mozilla.com
|
||||
labels:
|
||||
application: accounts_backend
|
||||
schedule: daily
|
||||
scheduling:
|
||||
dag_name: bqetl_accounts_backend_external
|
||||
# destination is the whole table, not a single partition,
|
||||
# so don't use date_partition_parameter
|
||||
date_partition_parameter: null
|
||||
referenced_tables: []
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
SELECT
|
||||
TO_HEX(uid) AS uid,
|
||||
SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
|
||||
flowId,
|
||||
FROM
|
||||
EXTERNAL_QUERY(
|
||||
"moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
|
||||
"""SELECT
|
||||
uid,
|
||||
createdAt,
|
||||
flowId
|
||||
FROM
|
||||
fxa.signinCodes
|
||||
"""
|
||||
)
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче