diff --git a/bqetl_project.yaml b/bqetl_project.yaml index 529636d530..ef2eaddebc 100644 --- a/bqetl_project.yaml +++ b/bqetl_project.yaml @@ -52,6 +52,7 @@ dry_run: - sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql - sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql - sql/moz-fx-data-shared-prod/accounts_backend/accounts/view.sql + - sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/dataset_metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/dataset_metadata.yaml new file mode 100644 index 0000000000..2868dd8a86 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/dataset_metadata.yaml @@ -0,0 +1,14 @@ +--- +friendly_name: Firefox Accounts Databases nonprod (stage) External +description: |- + Data extracted from the nonprod (stage) FxA backend services databases. + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa for more information. + + Access to this dataset is restricted to accounts-confidential workgroup because some tables here contain sensitive data. +dataset_base_acl: restricted +user_facing: false +labels: {} +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:accounts-confidential diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/metadata.yaml new file mode 100644 index 0000000000..0bff7f91d9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: accountCustomers table from nonprod (stage) fxa database +description: > + A mirror of the `accountCustomers` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/query.sql new file mode 100644 index 0000000000..dc6da1dbde --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/query.sql @@ -0,0 +1,17 @@ +SELECT + TO_HEX(uid) AS uid, + stripeCustomerId, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(updatedAt AS INT)) AS updatedAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + stripeCustomerId, + createdAt, + updatedAt + FROM + fxa.accountCustomers + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/schema.yaml new file mode 100644 index 0000000000..fb9aba6f33 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_customers_v1/schema.yaml @@ -0,0 +1,14 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: stripeCustomerId + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: updatedAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..4e6628abff --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: accountResetTokens table from nonprod (stage) fxa database +description: > + A mirror of the `accountResetTokens` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/query.sql new file mode 100644 index 0000000000..ee261938e8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt + FROM + fxa.accountResetTokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/schema.yaml new file mode 100644 index 0000000000..0aaaf1bcab --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_account_reset_tokens_v1/schema.yaml @@ -0,0 +1,8 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/metadata.yaml new file mode 100644 index 0000000000..94a3d505fb --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: accounts table from nonprod (stage) fxa database +description: > + A mirror of the `accounts` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/query.sql new file mode 100644 index 0000000000..d9e2d44b79 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/query.sql @@ -0,0 +1,37 @@ +SELECT + TO_HEX(uid) AS uid, + normalizedEmail, + email, + SAFE_CAST(emailVerified AS BOOL) AS emailVerified, + verifierVersion, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifierSetAt AS INT)) AS verifierSetAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + locale, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(lockedAt AS INT)) AS lockedAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(keysChangedAt AS INT)) AS keysChangedAt, + ecosystemAnonId, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(disabledAt AS INT)) AS disabledAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(metricsOptOutAt AS INT)) AS metricsOptOutAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + normalizedEmail, + email, + emailVerified, + verifierVersion, + verifierSetAt, + createdAt, + locale, + lockedAt, + profileChangedAt, + keysChangedAt, + ecosystemAnonId, + disabledAt, + metricsOptOutAt + FROM + fxa.accounts + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/schema.yaml new file mode 100644 index 0000000000..4ca26c538c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_accounts_v1/schema.yaml @@ -0,0 +1,44 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: normalizedEmail + type: STRING + mode: NULLABLE +- name: email + type: STRING + mode: NULLABLE +- name: emailVerified + type: BOOLEAN + mode: NULLABLE +- name: verifierVersion + type: INTEGER + mode: NULLABLE +- name: verifierSetAt + type: TIMESTAMP + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: locale + type: STRING + mode: NULLABLE +- name: lockedAt + type: TIMESTAMP + mode: NULLABLE +- name: profileChangedAt + type: TIMESTAMP + mode: NULLABLE +- name: keysChangedAt + type: TIMESTAMP + mode: NULLABLE +- name: ecosystemAnonId + type: STRING + mode: NULLABLE +- name: disabledAt + type: TIMESTAMP + mode: NULLABLE +- name: metricsOptOutAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/metadata.yaml new file mode 100644 index 0000000000..3afcee82f8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: dbMetadata table from nonprod (stage) fxa database +description: > + A mirror of the `dbMetadata` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/query.sql new file mode 100644 index 0000000000..cabac16ef4 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + name, + value, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + name, + value + FROM + fxa.dbMetadata + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/schema.yaml new file mode 100644 index 0000000000..f22672dc48 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_db_metadata_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: name + type: STRING + mode: NULLABLE +- name: value + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/metadata.yaml new file mode 100644 index 0000000000..1ae097d3ef --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: deviceCommandIdentifiers table from nonprod (stage) fxa database +description: > + A mirror of the `deviceCommandIdentifiers` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/query.sql new file mode 100644 index 0000000000..0fb4eb1a36 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + commandId, + commandName, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + commandId, + commandName + FROM + fxa.deviceCommandIdentifiers + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/schema.yaml new file mode 100644 index 0000000000..5e3ccd0279 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_command_identifiers_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: commandId + type: INTEGER + mode: NULLABLE +- name: commandName + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/metadata.yaml new file mode 100644 index 0000000000..aa2e7f14dd --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: deviceCommands table from nonprod (stage) fxa database +description: > + A mirror of the `deviceCommands` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/query.sql new file mode 100644 index 0000000000..d1d54dac67 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/query.sql @@ -0,0 +1,15 @@ +SELECT + TO_HEX(uid) AS uid, + deviceId, + commandId, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + deviceId, + commandId + FROM + fxa.deviceCommands + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/schema.yaml new file mode 100644 index 0000000000..786e69bf47 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_device_commands_v1/schema.yaml @@ -0,0 +1,11 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: deviceId + type: STRING + mode: NULLABLE +- name: commandId + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/metadata.yaml new file mode 100644 index 0000000000..5a035d7a65 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: devices table from nonprod (stage) fxa database +description: > + A mirror of the `devices` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/query.sql new file mode 100644 index 0000000000..7d26804d9b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/query.sql @@ -0,0 +1,25 @@ +SELECT + TO_HEX(uid) AS uid, + id, + name, + nameUtf8, + type, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + callbackPublicKey, + SAFE_CAST(callbackIsExpired AS BOOL) AS callbackIsExpired, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + id, + name, + nameUtf8, + type, + createdAt, + callbackPublicKey, + callbackIsExpired + FROM + fxa.devices + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/schema.yaml new file mode 100644 index 0000000000..ae9821c316 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_devices_v1/schema.yaml @@ -0,0 +1,26 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: id + type: STRING + mode: NULLABLE +- name: name + type: STRING + mode: NULLABLE +- name: nameUtf8 + type: STRING + mode: NULLABLE +- name: type + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: callbackPublicKey + type: STRING + mode: NULLABLE +- name: callbackIsExpired + type: BOOLEAN + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/metadata.yaml new file mode 100644 index 0000000000..e902e5a27d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: emailBounces table from nonprod (stage) fxa database +description: > + A mirror of the `emailBounces` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/query.sql new file mode 100644 index 0000000000..c36872fc25 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/query.sql @@ -0,0 +1,21 @@ +SELECT + email, + bounceType, + bounceSubType, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + emailTypeId, + diagnosticCode, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + email, + bounceType, + bounceSubType, + createdAt, + emailTypeId, + diagnosticCode + FROM + fxa.emailBounces + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/schema.yaml new file mode 100644 index 0000000000..65181c6c74 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_bounces_v1/schema.yaml @@ -0,0 +1,19 @@ +fields: +- name: email + type: STRING + mode: NULLABLE +- name: bounceType + type: INTEGER + mode: NULLABLE +- name: bounceSubType + type: INTEGER + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: emailTypeId + type: INTEGER + mode: NULLABLE +- name: diagnosticCode + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/metadata.yaml new file mode 100644 index 0000000000..45979c8967 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: emailTypes table from nonprod (stage) fxa database +description: > + A mirror of the `emailTypes` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/query.sql new file mode 100644 index 0000000000..a4aada6bed --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + id, + emailType, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + id, + emailType + FROM + fxa.emailTypes + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/schema.yaml new file mode 100644 index 0000000000..a4cbc8255c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_email_types_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: emailType + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/metadata.yaml new file mode 100644 index 0000000000..f7e6bdd0a9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: emails table from nonprod (stage) fxa database +description: > + A mirror of the `emails` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/query.sql new file mode 100644 index 0000000000..20efe1b2b9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/query.sql @@ -0,0 +1,25 @@ +SELECT + id, + normalizedEmail, + email, + TO_HEX(uid) AS uid, + SAFE_CAST(isVerified AS BOOL) AS isVerified, + SAFE_CAST(isPrimary AS BOOL) AS isPrimary, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + id, + normalizedEmail, + email, + uid, + isVerified, + isPrimary, + verifiedAt, + createdAt + FROM + fxa.emails + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/schema.yaml new file mode 100644 index 0000000000..18bb730afa --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_emails_v1/schema.yaml @@ -0,0 +1,26 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: normalizedEmail + type: STRING + mode: NULLABLE +- name: email + type: STRING + mode: NULLABLE +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: isVerified + type: BOOLEAN + mode: NULLABLE +- name: isPrimary + type: BOOLEAN + mode: NULLABLE +- name: verifiedAt + type: TIMESTAMP + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..083d504713 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: keyFetchTokens table from nonprod (stage) fxa database +description: > + A mirror of the `keyFetchTokens` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/query.sql new file mode 100644 index 0000000000..a21d3df181 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt + FROM + fxa.keyFetchTokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/schema.yaml new file mode 100644 index 0000000000..0aaaf1bcab --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_key_fetch_tokens_v1/schema.yaml @@ -0,0 +1,8 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/metadata.yaml new file mode 100644 index 0000000000..932201e237 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: linkedAccounts table from nonprod (stage) fxa database +description: > + A mirror of the `linkedAccounts` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/query.sql new file mode 100644 index 0000000000..9c0514d870 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/query.sql @@ -0,0 +1,19 @@ +SELECT + TO_HEX(uid) AS uid, + id, + providerId, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt, + SAFE_CAST(enabled AS BOOL) AS enabled, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + id, + providerId, + authAt, + enabled + FROM + fxa.linkedAccounts + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/schema.yaml new file mode 100644 index 0000000000..a31940ca6a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_linked_accounts_v1/schema.yaml @@ -0,0 +1,17 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: id + type: STRING + mode: NULLABLE +- name: providerId + type: INTEGER + mode: NULLABLE +- name: authAt + type: TIMESTAMP + mode: NULLABLE +- name: enabled + type: BOOLEAN + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/metadata.yaml new file mode 100644 index 0000000000..5c4ab26b81 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: clients table from nonprod (stage) fxa_oauth database +description: > + A mirror of the `clients` table from the nonprod (stage) `fxa_oauth` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/query.sql new file mode 100644 index 0000000000..6c2a8c4d88 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/query.sql @@ -0,0 +1,29 @@ +SELECT + id, + name, + imageUri, + redirectUri, + SAFE_CAST(canGrant AS BOOL) AS canGrant, + SAFE_CAST(publicClient AS BOOL) AS publicClient, + createdAt, + SAFE_CAST(trusted AS BOOL) AS trusted, + allowedScopes, + notes, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth", + """SELECT + id, + name, + imageUri, + redirectUri, + canGrant, + publicClient, + createdAt, + trusted, + allowedScopes, + notes + FROM + fxa_oauth.clients + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/schema.yaml new file mode 100644 index 0000000000..84597a0f0e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_clients_v1/schema.yaml @@ -0,0 +1,31 @@ +fields: +- name: id + type: STRING + mode: NULLABLE +- name: name + type: STRING + mode: NULLABLE +- name: imageUri + type: STRING + mode: NULLABLE +- name: redirectUri + type: STRING + mode: NULLABLE +- name: canGrant + type: BOOLEAN + mode: NULLABLE +- name: publicClient + type: BOOLEAN + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: trusted + type: BOOLEAN + mode: NULLABLE +- name: allowedScopes + type: STRING + mode: NULLABLE +- name: notes + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/metadata.yaml new file mode 100644 index 0000000000..e5e199a4cf --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: codes table from nonprod (stage) fxa_oauth database +description: > + A mirror of the `codes` table from the nonprod (stage) `fxa_oauth` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/query.sql new file mode 100644 index 0000000000..bcf08a324c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/query.sql @@ -0,0 +1,31 @@ +SELECT + clientId, + userId, + scope, + createdAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt, + amr, + aal, + SAFE_CAST(offline AS BOOL) AS offline, + codeChallengeMethod, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt, + sessionTokenId, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth", + """SELECT + clientId, + userId, + scope, + createdAt, + authAt, + amr, + aal, + offline, + codeChallengeMethod, + profileChangedAt, + sessionTokenId + FROM + fxa_oauth.codes + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/schema.yaml new file mode 100644 index 0000000000..caf387f383 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_codes_v1/schema.yaml @@ -0,0 +1,34 @@ +fields: +- name: clientId + type: STRING + mode: NULLABLE +- name: userId + type: STRING + mode: NULLABLE +- name: scope + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: authAt + type: TIMESTAMP + mode: NULLABLE +- name: amr + type: STRING + mode: NULLABLE +- name: aal + type: INTEGER + mode: NULLABLE +- name: offline + type: BOOLEAN + mode: NULLABLE +- name: codeChallengeMethod + type: STRING + mode: NULLABLE +- name: profileChangedAt + type: TIMESTAMP + mode: NULLABLE +- name: sessionTokenId + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/metadata.yaml new file mode 100644 index 0000000000..893a3bd006 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: dbMetadata table from nonprod (stage) fxa_oauth database +description: > + A mirror of the `dbMetadata` table from the nonprod (stage) `fxa_oauth` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/query.sql new file mode 100644 index 0000000000..89153da074 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + name, + value, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth", + """SELECT + name, + value + FROM + fxa_oauth.dbMetadata + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/schema.yaml new file mode 100644 index 0000000000..f22672dc48 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_db_metadata_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: name + type: STRING + mode: NULLABLE +- name: value + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..db5950a749 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: refreshTokens table from nonprod (stage) fxa_oauth database +description: > + A mirror of the `refreshTokens` table from the nonprod (stage) `fxa_oauth` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/query.sql new file mode 100644 index 0000000000..e6351be123 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/query.sql @@ -0,0 +1,21 @@ +SELECT + clientId, + userId, + scope, + createdAt, + lastUsedAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth", + """SELECT + clientId, + userId, + scope, + createdAt, + lastUsedAt, + profileChangedAt + FROM + fxa_oauth.refreshTokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/schema.yaml new file mode 100644 index 0000000000..3529886573 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_refresh_tokens_v1/schema.yaml @@ -0,0 +1,19 @@ +fields: +- name: clientId + type: STRING + mode: NULLABLE +- name: userId + type: STRING + mode: NULLABLE +- name: scope + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: lastUsedAt + type: TIMESTAMP + mode: NULLABLE +- name: profileChangedAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/metadata.yaml new file mode 100644 index 0000000000..a3f42ba042 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: scopes table from nonprod (stage) fxa_oauth database +description: > + A mirror of the `scopes` table from the nonprod (stage) `fxa_oauth` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/query.sql new file mode 100644 index 0000000000..38de8a7058 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + scope, + SAFE_CAST(hasScopedKeys AS BOOL) AS hasScopedKeys, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth", + """SELECT + scope, + hasScopedKeys + FROM + fxa_oauth.scopes + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/schema.yaml new file mode 100644 index 0000000000..1416d78638 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_scopes_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: scope + type: STRING + mode: NULLABLE +- name: hasScopedKeys + type: BOOLEAN + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..abcbd6b310 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: tokens table from nonprod (stage) fxa_oauth database +description: > + A mirror of the `tokens` table from the nonprod (stage) `fxa_oauth` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/query.sql new file mode 100644 index 0000000000..ed3e78fbd3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/query.sql @@ -0,0 +1,23 @@ +SELECT + clientId, + userId, + type, + scope, + createdAt, + expiresAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-oauth-nonprod-stage-fxa-oauth", + """SELECT + clientId, + userId, + type, + scope, + createdAt, + expiresAt, + profileChangedAt + FROM + fxa_oauth.tokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/schema.yaml new file mode 100644 index 0000000000..d8f87d7018 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_oauth_tokens_v1/schema.yaml @@ -0,0 +1,22 @@ +fields: +- name: clientId + type: STRING + mode: NULLABLE +- name: userId + type: STRING + mode: NULLABLE +- name: type + type: STRING + mode: NULLABLE +- name: scope + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: expiresAt + type: TIMESTAMP + mode: NULLABLE +- name: profileChangedAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..c8723a4512 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: passwordChangeTokens table from nonprod (stage) fxa database +description: > + A mirror of the `passwordChangeTokens` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/query.sql new file mode 100644 index 0000000000..dcb673ee7e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt + FROM + fxa.passwordChangeTokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/schema.yaml new file mode 100644 index 0000000000..0aaaf1bcab --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_change_tokens_v1/schema.yaml @@ -0,0 +1,8 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..55774bbd17 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: passwordForgotTokens table from nonprod (stage) fxa database +description: > + A mirror of the `passwordForgotTokens` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/query.sql new file mode 100644 index 0000000000..d5f69c0e7c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/query.sql @@ -0,0 +1,15 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + tries, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt, + tries + FROM + fxa.passwordForgotTokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/schema.yaml new file mode 100644 index 0000000000..a29b29dde4 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_password_forgot_tokens_v1/schema.yaml @@ -0,0 +1,11 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: tries + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/metadata.yaml new file mode 100644 index 0000000000..13e58d7b58 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: paypalCustomers table from nonprod (stage) fxa database +description: > + A mirror of the `paypalCustomers` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/query.sql new file mode 100644 index 0000000000..737a0e0eb5 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/query.sql @@ -0,0 +1,19 @@ +SELECT + TO_HEX(uid) AS uid, + billingAgreementId, + status, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(endedAt AS INT)) AS endedAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + billingAgreementId, + status, + createdAt, + endedAt + FROM + fxa.paypalCustomers + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/schema.yaml new file mode 100644 index 0000000000..dc177eeaa2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_paypal_customers_v1/schema.yaml @@ -0,0 +1,17 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: billingAgreementId + type: STRING + mode: NULLABLE +- name: status + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: endedAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/metadata.yaml new file mode 100644 index 0000000000..ff3cd4fe31 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: avatar_providers table from nonprod (stage) fxa_profile database +description: > + A mirror of the `avatar_providers` table from the nonprod (stage) `fxa_profile` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/query.sql new file mode 100644 index 0000000000..fc3d1166e7 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + id, + name, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile", + """SELECT + id, + name + FROM + fxa_profile.avatar_providers + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/schema.yaml new file mode 100644 index 0000000000..29e71b54c9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_providers_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: name + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/metadata.yaml new file mode 100644 index 0000000000..625e66a8c8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: avatar_selected table from nonprod (stage) fxa_profile database +description: > + A mirror of the `avatar_selected` table from the nonprod (stage) `fxa_profile` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/query.sql new file mode 100644 index 0000000000..8de55a2a9a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + userId, + avatarId, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile", + """SELECT + userId, + avatarId + FROM + fxa_profile.avatar_selected + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/schema.yaml new file mode 100644 index 0000000000..039d7f2076 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatar_selected_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: userId + type: STRING + mode: NULLABLE +- name: avatarId + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/metadata.yaml new file mode 100644 index 0000000000..de29f32538 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: avatars table from nonprod (stage) fxa_profile database +description: > + A mirror of the `avatars` table from the nonprod (stage) `fxa_profile` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/query.sql new file mode 100644 index 0000000000..ad09f1a0fa --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/query.sql @@ -0,0 +1,17 @@ +SELECT + id, + url, + userId, + providerId, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile", + """SELECT + id, + url, + userId, + providerId + FROM + fxa_profile.avatars + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/schema.yaml new file mode 100644 index 0000000000..e7be0d0b34 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_avatars_v1/schema.yaml @@ -0,0 +1,13 @@ +fields: +- name: id + type: STRING + mode: NULLABLE +- name: url + type: STRING + mode: NULLABLE +- name: userId + type: STRING + mode: NULLABLE +- name: providerId + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/metadata.yaml new file mode 100644 index 0000000000..551994c854 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: dbMetadata table from nonprod (stage) fxa_profile database +description: > + A mirror of the `dbMetadata` table from the nonprod (stage) `fxa_profile` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/query.sql new file mode 100644 index 0000000000..71e3c28767 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + name, + value, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile", + """SELECT + name, + value + FROM + fxa_profile.dbMetadata + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/schema.yaml new file mode 100644 index 0000000000..f22672dc48 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_db_metadata_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: name + type: STRING + mode: NULLABLE +- name: value + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/metadata.yaml new file mode 100644 index 0000000000..52958d16c5 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: profile table from nonprod (stage) fxa_profile database +description: > + A mirror of the `profile` table from the nonprod (stage) `fxa_profile` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/query.sql new file mode 100644 index 0000000000..e0e0a819d8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + userId, + displayName, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa-profile", + """SELECT + userId, + displayName + FROM + fxa_profile.profile + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/schema.yaml new file mode 100644 index 0000000000..e8722a7954 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_profile_profile_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: userId + type: STRING + mode: NULLABLE +- name: displayName + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/metadata.yaml new file mode 100644 index 0000000000..2df83fc54e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: recoveryCodes table from nonprod (stage) fxa database +description: > + A mirror of the `recoveryCodes` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/query.sql new file mode 100644 index 0000000000..fb68349981 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + id, + TO_HEX(uid) AS uid, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + id, + uid + FROM + fxa.recoveryCodes + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/schema.yaml new file mode 100644 index 0000000000..a87605c180 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_codes_v1/schema.yaml @@ -0,0 +1,8 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/metadata.yaml new file mode 100644 index 0000000000..39f4f6f8c8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: recoveryKeys table from nonprod (stage) fxa database +description: > + A mirror of the `recoveryKeys` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/query.sql new file mode 100644 index 0000000000..820ea23a45 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/query.sql @@ -0,0 +1,17 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt, + SAFE_CAST(enabled AS BOOL) AS enabled, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt, + verifiedAt, + enabled + FROM + fxa.recoveryKeys + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/schema.yaml new file mode 100644 index 0000000000..a60631ef23 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_recovery_keys_v1/schema.yaml @@ -0,0 +1,14 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: verifiedAt + type: TIMESTAMP + mode: NULLABLE +- name: enabled + type: BOOLEAN + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/metadata.yaml new file mode 100644 index 0000000000..cd682ad9f9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: securityEventNames table from nonprod (stage) fxa database +description: > + A mirror of the `securityEventNames` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/query.sql new file mode 100644 index 0000000000..ef5a472519 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + id, + name, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + id, + name + FROM + fxa.securityEventNames + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/schema.yaml new file mode 100644 index 0000000000..29e71b54c9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_event_names_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: name + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/metadata.yaml new file mode 100644 index 0000000000..d763cd66cf --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: securityEvents table from nonprod (stage) fxa database +description: > + A mirror of the `securityEvents` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/query.sql new file mode 100644 index 0000000000..f6a24ed85e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/query.sql @@ -0,0 +1,23 @@ +SELECT + id, + TO_HEX(uid) AS uid, + nameId, + SAFE_CAST(verified AS BOOL) AS verified, + ipAddrHmac, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + tokenVerificationId, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + id, + uid, + nameId, + verified, + ipAddrHmac, + createdAt, + tokenVerificationId + FROM + fxa.securityEvents + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/schema.yaml new file mode 100644 index 0000000000..704e7c3264 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_security_events_v1/schema.yaml @@ -0,0 +1,23 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: nameId + type: INTEGER + mode: NULLABLE +- name: verified + type: BOOLEAN + mode: NULLABLE +- name: ipAddrHmac + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: tokenVerificationId + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/metadata.yaml new file mode 100644 index 0000000000..50ebcf4a08 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: sentEmails table from nonprod (stage) fxa database +description: > + A mirror of the `sentEmails` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/query.sql new file mode 100644 index 0000000000..8f841252b0 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/query.sql @@ -0,0 +1,19 @@ +SELECT + id, + TO_HEX(uid) AS uid, + emailTypeId, + params, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(sentAt AS INT)) AS sentAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + id, + uid, + emailTypeId, + params, + sentAt + FROM + fxa.sentEmails + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/schema.yaml new file mode 100644 index 0000000000..cc0276c434 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_sent_emails_v1/schema.yaml @@ -0,0 +1,17 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: emailTypeId + type: INTEGER + mode: NULLABLE +- name: params + type: STRING + mode: NULLABLE +- name: sentAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..dbb5863669 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: sessionTokens table from nonprod (stage) fxa database +description: > + A mirror of the `sessionTokens` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/query.sql new file mode 100644 index 0000000000..d7de3dc847 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/query.sql @@ -0,0 +1,35 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + uaBrowser, + uaBrowserVersion, + uaOS, + uaOSVersion, + uaDeviceType, + lastAccessTime, + uaFormFactor, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(authAt AS INT)) AS authAt, + verificationMethod, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifiedAt AS INT)) AS verifiedAt, + SAFE_CAST(mustVerify AS BOOL) AS mustVerify, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt, + uaBrowser, + uaBrowserVersion, + uaOS, + uaOSVersion, + uaDeviceType, + lastAccessTime, + uaFormFactor, + authAt, + verificationMethod, + verifiedAt, + mustVerify + FROM + fxa.sessionTokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/schema.yaml new file mode 100644 index 0000000000..2703c2b837 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_session_tokens_v1/schema.yaml @@ -0,0 +1,41 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: uaBrowser + type: STRING + mode: NULLABLE +- name: uaBrowserVersion + type: STRING + mode: NULLABLE +- name: uaOS + type: STRING + mode: NULLABLE +- name: uaOSVersion + type: STRING + mode: NULLABLE +- name: uaDeviceType + type: STRING + mode: NULLABLE +- name: lastAccessTime + type: INTEGER + mode: NULLABLE +- name: uaFormFactor + type: STRING + mode: NULLABLE +- name: authAt + type: TIMESTAMP + mode: NULLABLE +- name: verificationMethod + type: INTEGER + mode: NULLABLE +- name: verifiedAt + type: TIMESTAMP + mode: NULLABLE +- name: mustVerify + type: BOOLEAN + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/metadata.yaml new file mode 100644 index 0000000000..856273120a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: signinCodes table from nonprod (stage) fxa database +description: > + A mirror of the `signinCodes` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/query.sql new file mode 100644 index 0000000000..88216f3e6f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/query.sql @@ -0,0 +1,15 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + flowId, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt, + flowId + FROM + fxa.signinCodes + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/schema.yaml new file mode 100644 index 0000000000..6b418d11f1 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_signin_codes_v1/schema.yaml @@ -0,0 +1,11 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: flowId + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/metadata.yaml new file mode 100644 index 0000000000..dabc68adbe --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: totp table from nonprod (stage) fxa database +description: > + A mirror of the `totp` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/query.sql new file mode 100644 index 0000000000..b683de64c6 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/query.sql @@ -0,0 +1,19 @@ +SELECT + id, + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, + SAFE_CAST(verified AS BOOL) AS verified, + SAFE_CAST(enabled AS BOOL) AS enabled, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + id, + uid, + createdAt, + verified, + enabled + FROM + fxa.totp + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/schema.yaml new file mode 100644 index 0000000000..a18585964e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_totp_v1/schema.yaml @@ -0,0 +1,17 @@ +fields: +- name: id + type: INTEGER + mode: NULLABLE +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE +- name: verified + type: BOOLEAN + mode: NULLABLE +- name: enabled + type: BOOLEAN + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/metadata.yaml new file mode 100644 index 0000000000..1414ab512a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: unblockCodes table from nonprod (stage) fxa database +description: > + A mirror of the `unblockCodes` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/query.sql new file mode 100644 index 0000000000..767833818f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/query.sql @@ -0,0 +1,13 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + createdAt + FROM + fxa.unblockCodes + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/schema.yaml new file mode 100644 index 0000000000..0aaaf1bcab --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unblock_codes_v1/schema.yaml @@ -0,0 +1,8 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: createdAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/metadata.yaml new file mode 100644 index 0000000000..1da4c49dfe --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: unverifiedTokens table from nonprod (stage) fxa database +description: > + A mirror of the `unverifiedTokens` table from the nonprod (stage) `fxa` CloudSQL database, + excluding columns containing confidential data, updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/query.sql new file mode 100644 index 0000000000..ba253ceda5 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/query.sql @@ -0,0 +1,17 @@ +SELECT + TO_HEX(uid) AS uid, + SAFE_CAST(mustVerify AS BOOL) AS mustVerify, + SAFE.TIMESTAMP_MILLIS( + SAFE_CAST(tokenVerificationCodeExpiresAt AS INT) + ) AS tokenVerificationCodeExpiresAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + mustVerify, + tokenVerificationCodeExpiresAt + FROM + fxa.unverifiedTokens + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/schema.yaml new file mode 100644 index 0000000000..b53626a70a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_unverified_tokens_v1/schema.yaml @@ -0,0 +1,11 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: mustVerify + type: BOOLEAN + mode: NULLABLE +- name: tokenVerificationCodeExpiresAt + type: TIMESTAMP + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/metadata.yaml new file mode 100644 index 0000000000..a80253e7d5 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/metadata.yaml @@ -0,0 +1,18 @@ +--- +friendly_name: verificationReminders table from nonprod (stage) fxa database +description: > + A mirror of the `verificationReminders` table from the nonprod (stage) `fxa` CloudSQL database, + updated daily to match the current state of the table. + + See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa +owners: + - akomar@mozilla.com +labels: + application: accounts_backend + schedule: daily +scheduling: + dag_name: bqetl_accounts_backend_external + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + referenced_tables: [] diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/query.sql b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/query.sql new file mode 100644 index 0000000000..3061c0e38e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/query.sql @@ -0,0 +1,15 @@ +SELECT + TO_HEX(uid) AS uid, + type, + SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, +FROM + EXTERNAL_QUERY( + "moz-fx-fxa-nonprod.us.fxa-rds-nonprod-stage-fxa", + """SELECT + uid, + type, + createdAt + FROM + fxa.verificationReminders + """ + ) diff --git a/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/schema.yaml b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/schema.yaml new file mode 100644 index 0000000000..1ee40660e0 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/fxa_verification_reminders_v1/schema.yaml @@ -0,0 +1,11 @@ +fields: +- name: uid + type: STRING + mode: NULLABLE + description: Account ID in hexadecimal format. +- name: type + type: STRING + mode: NULLABLE +- name: createdAt + type: TIMESTAMP + mode: NULLABLE