diff --git a/bqetl_project.yaml b/bqetl_project.yaml index 65dd07deed..d85d6a7fbf 100644 --- a/bqetl_project.yaml +++ b/bqetl_project.yaml @@ -53,6 +53,7 @@ dry_run: - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/query.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/query.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/query.sql + - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/query.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1/query.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/query.sql - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/query.sql diff --git a/dags/bqetl_fxa_events.py b/dags/bqetl_fxa_events.py index f990342828..0fcef9164f 100644 --- a/dags/bqetl_fxa_events.py +++ b/dags/bqetl_fxa_events.py @@ -264,13 +264,14 @@ with DAG( depends_on_past=False, ) - firefox_accounts_derived__fxa_log_device_command_events__v1 = bigquery_etl_query( - task_id="firefox_accounts_derived__fxa_log_device_command_events__v1", - destination_table="fxa_log_device_command_events_v1", + firefox_accounts_derived__fxa_log_device_command_events__v2 = bigquery_etl_query( + task_id="firefox_accounts_derived__fxa_log_device_command_events__v2", + destination_table="fxa_log_device_command_events_v2", dataset_id="firefox_accounts_derived", project_id="moz-fx-data-shared-prod", - owner="dthorn@mozilla.com", - email=["dthorn@mozilla.com", "telemetry-alerts@mozilla.com"], + owner="kik@mozilla.com", + email=["dthorn@mozilla.com", "kik@mozilla.com", "telemetry-alerts@mozilla.com"], + start_date=datetime.datetime(2023, 9, 7, 0, 0), date_partition_parameter="submission_date", depends_on_past=False, ) diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts/fxa_log_device_command_events/view.sql b/sql/moz-fx-data-shared-prod/firefox_accounts/fxa_log_device_command_events/view.sql index 02f0142625..06e4bc0820 100755 --- a/sql/moz-fx-data-shared-prod/firefox_accounts/fxa_log_device_command_events/view.sql +++ b/sql/moz-fx-data-shared-prod/firefox_accounts/fxa_log_device_command_events/view.sql @@ -5,3 +5,8 @@ SELECT * FROM `moz-fx-data-shared-prod.firefox_accounts_derived.fxa_log_device_command_events_v1` +UNION ALL +SELECT + * +FROM + `moz-fx-data-shared-prod.firefox_accounts_derived.fxa_log_device_command_events_v2` diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml index 35bc9cb6b0..49eb5eab73 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml @@ -2,8 +2,11 @@ friendly_name: FxA Log Device Command Events description: |- A subset of FxA auth server logs related to "send tab" activity. - See https://bugzilla.mozilla.org/show_bug.cgi?id=1649074 + + This table contains event data from the FxA service when it was still running in AWS. + All new events come from GCP deployment of FxA and are available in the v2 of this table. + The migration started on 2023-09-07 and was concluded by 2023-09-15. owners: - dthorn@mozilla.com labels: @@ -11,7 +14,8 @@ labels: incremental: true schedule: daily scheduling: - dag_name: bqetl_fxa_events + # descheduled due to source table change. v2 of the query retrieves the data from the new source. + # dag_name: bqetl_fxa_events # This query references secret keys that are not available for dry runs, # so we must explicitly write out dependencies. In this case, the query # depends only on fxa logs produced via Stackdriver integration, so no other diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/schema.yaml new file mode 100644 index 0000000000..75b6e8e185 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/schema.yaml @@ -0,0 +1,34 @@ +fields: +- name: timestamp + type: TIMESTAMP + mode: NULLABLE +- name: type + type: STRING + mode: NULLABLE +- name: user_id + type: STRING + mode: NULLABLE +- name: index + type: STRING + mode: NULLABLE +- name: command + type: STRING + mode: NULLABLE +- name: target + type: STRING + mode: NULLABLE +- name: target_os + type: STRING + mode: NULLABLE +- name: target_type + type: STRING + mode: NULLABLE +- name: sender + type: STRING + mode: NULLABLE +- name: sender_os + type: STRING + mode: NULLABLE +- name: sender_type + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/metadata.yaml new file mode 100644 index 0000000000..7a02a7ae57 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/metadata.yaml @@ -0,0 +1,33 @@ +--- +friendly_name: FxA Log Device Command Events +description: |- + A subset of FxA auth server logs related to "send tab" activity. + See https://bugzilla.mozilla.org/show_bug.cgi?id=1649074 + + This table contains FxA log device command events from the GCP based FxA deployment. + The migration from AWS to GCP started on 2023-09-07 and concluded by 2023-09-15. + And the v1 version of the table contains event data prior to the migration coming to a conclusion. +owners: + - kik@mozilla.com +labels: + application: fxa + incremental: true + schedule: daily +scheduling: + dag_name: bqetl_fxa_events + # v2 created as the result of the source table changing post AWS to GCP migration. + # The start date represents when the migration started. + start_date: "2023-09-07" + # This query references secret keys that are not available for dry runs, + # so we must explicitly write out dependencies. In this case, the query + # depends only on fxa logs produced via Stackdriver integration, so no other + # scheduled tasks are involved and the referenced_tables list is empty. + referenced_tables: [] +bigquery: + time_partitioning: + type: day + field: timestamp + require_partition_filter: false + clustering: + fields: + - command diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/query.sql b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/query.sql new file mode 100644 index 0000000000..c4b5e2845a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/query.sql @@ -0,0 +1,42 @@ +WITH hmac_key AS ( + SELECT + AEAD.DECRYPT_BYTES( + (SELECT keyset FROM `moz-fx-dataops-secrets.airflow_query_keys.fxa_prod`), + ciphertext, + CAST(key_id AS BYTES) + ) AS value + FROM + `moz-fx-data-shared-prod.firefox_accounts_derived.encrypted_keys_v1` + WHERE + key_id = 'fxa_hmac_prod' +) +SELECT + `timestamp`, + jsonPayload.type, + TO_HEX( + udf.hmac_sha256((SELECT * FROM hmac_key), CAST(jsonPayload.fields.uid AS BYTES)) + ) AS user_id, + TO_HEX( + udf.hmac_sha256( + (SELECT * FROM hmac_key), + CAST(FORMAT('%d', CAST(jsonPayload.fields.index AS INT64)) AS BYTES) + ) + ) AS index, + jsonPayload.fields.command, + TO_HEX( + udf.hmac_sha256((SELECT * FROM hmac_key), CAST(jsonPayload.fields.target AS BYTES)) + ) AS target, + jsonPayload.fields.targetOS AS target_os, + jsonPayload.fields.targetType AS target_type, + TO_HEX( + udf.hmac_sha256((SELECT * FROM hmac_key), CAST(jsonPayload.fields.sender AS BYTES)) + ) AS sender, + jsonPayload.fields.senderOS AS sender_os, + jsonPayload.fields.senderType AS sender_type, +FROM + `moz-fx-fxa-prod.gke_fxa_prod_log.stderr` +WHERE + jsonPayload.type LIKE 'device.command.%' + AND DATE(`timestamp`) = @submission_date + -- Only include events after AWS tp GCP migration started. + AND DATE(`timestamp`) >= "2023-09-07" diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/schema.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/schema.yaml new file mode 100644 index 0000000000..75b6e8e185 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/schema.yaml @@ -0,0 +1,34 @@ +fields: +- name: timestamp + type: TIMESTAMP + mode: NULLABLE +- name: type + type: STRING + mode: NULLABLE +- name: user_id + type: STRING + mode: NULLABLE +- name: index + type: STRING + mode: NULLABLE +- name: command + type: STRING + mode: NULLABLE +- name: target + type: STRING + mode: NULLABLE +- name: target_os + type: STRING + mode: NULLABLE +- name: target_type + type: STRING + mode: NULLABLE +- name: sender + type: STRING + mode: NULLABLE +- name: sender_os + type: STRING + mode: NULLABLE +- name: sender_type + type: STRING + mode: NULLABLE