mozilla
/
bigquery-etl
зеркало из https://github.com/mozilla/bigquery-etl.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

DSRE-1681 Create GA3 and GA4 SUMO data tables in shared-prod (#6179) 

* initial commit work in progress

* DSRE-1681 - create new tables for GA3 and GA4 SUMO data, copies of raw

* DSRE-1681 remove clustering from both metadata.yaml files
This commit is contained in:
Katie Windau 2024-09-12 11:45:00 -05:00 коммит произвёл GitHub
Родитель a770744e63
Коммит 1e5e3ff926
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
8 изменённых файлов: 1608 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
sql/moz-fx-data-shared-prod/sumo_ga/ga3_events/view.sql Normal file
Просмотреть файл

@ -0,0 +1,7 @@
CREATE OR REPLACE VIEW
`moz-fx-data-shared-prod.sumo_ga.ga3_events`
AS
SELECT
*
FROM
`moz-fx-data-shared-prod.sumo_ga_derived.ga3_events_v1`

7
sql/moz-fx-data-shared-prod/sumo_ga/ga4_events/view.sql Normal file
Просмотреть файл

@ -0,0 +1,7 @@
CREATE OR REPLACE VIEW
`moz-fx-data-shared-prod.sumo_ga.ga4_events`
AS
SELECT
*
FROM
`moz-fx-data-shared-prod.sumo_ga_derived.ga4_events_v1`

16
sql/moz-fx-data-shared-prod/sumo_ga_derived/ga3_events_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,16 @@
friendly_name: GA3 Events Data for SUMO
description: |-
GA3 Events Data for SUMO (support.mozilla.org)
owners:
- kwindau@mozilla.com
labels:
incremental: true
owner1: kwindau@mozilla.com
bigquery:
time_partitioning:
type: day
field: submission_date
require_partition_filter: false
expiration_days: null
range_partitioning: null
references: {}

7
sql/moz-fx-data-shared-prod/sumo_ga_derived/ga3_events_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,7 @@
SELECT
ga.*,
@submission_date AS submission_date
FROM
`moz-fx-data-marketing-prod.65912487.ga_sessions_*` ga
WHERE
_TABLE_SUFFIX = FORMAT_DATE('%Y%m%d', @submission_date)

1063
sql/moz-fx-data-shared-prod/sumo_ga_derived/ga3_events_v1/schema.yaml Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

18
sql/moz-fx-data-shared-prod/sumo_ga_derived/ga4_events_v1/metadata.yaml Normal file
Просмотреть файл

@ -0,0 +1,18 @@
friendly_name: GA4 Events Data for SUMO
description: |-
GA4 Events Data for SUMO (support.mozilla.org)
owners:
- kwindau@mozilla.com
labels:
incremental: true
owner1: kwindau@mozilla.com
scheduling:
dag_name: bqetl_google_analytics_derived_ga4
bigquery:
time_partitioning:
type: day
field: submission_date
require_partition_filter: false
expiration_days: null
range_partitioning: null
references: {}

7
sql/moz-fx-data-shared-prod/sumo_ga_derived/ga4_events_v1/query.sql Normal file
Просмотреть файл

@ -0,0 +1,7 @@
SELECT
ga.*,
@submission_date AS submission_date
FROM
`moz-fx-data-marketing-prod.analytics_314403930.events_*` ga
WHERE
_TABLE_SUFFIX = FORMAT_DATE('%Y%m%d', @submission_date)

483
sql/moz-fx-data-shared-prod/sumo_ga_derived/ga4_events_v1/schema.yaml Normal file
Просмотреть файл

@ -0,0 +1,483 @@
fields:
- name: event_date
type: STRING
mode: NULLABLE
- name: event_timestamp
type: INTEGER
mode: NULLABLE
- name: event_name
type: STRING
mode: NULLABLE
- name: event_params
type: RECORD
mode: REPEATED
fields:
- name: key
type: STRING
mode: NULLABLE
- name: value
type: RECORD
mode: NULLABLE
fields:
- name: string_value
type: STRING
mode: NULLABLE
- name: int_value
type: INTEGER
mode: NULLABLE
- name: float_value
type: FLOAT
mode: NULLABLE
- name: double_value
type: FLOAT
mode: NULLABLE
- name: event_previous_timestamp
type: INTEGER
mode: NULLABLE
- name: event_value_in_usd
type: FLOAT
mode: NULLABLE
- name: event_bundle_sequence_id
type: INTEGER
mode: NULLABLE
- name: event_server_timestamp_offset
type: INTEGER
mode: NULLABLE
- name: user_id
type: STRING
mode: NULLABLE
- name: user_pseudo_id
type: STRING
mode: NULLABLE
- name: privacy_info
type: RECORD
mode: NULLABLE
fields:
- name: analytics_storage
type: STRING
mode: NULLABLE
- name: ads_storage
type: STRING
mode: NULLABLE
- name: uses_transient_token
type: STRING
mode: NULLABLE
- name: user_properties
type: RECORD
mode: REPEATED
fields:
- name: key
type: STRING
mode: NULLABLE
- name: value
type: RECORD
mode: NULLABLE
fields:
- name: string_value
type: STRING
mode: NULLABLE
- name: int_value
type: INTEGER
mode: NULLABLE
- name: float_value
type: FLOAT
mode: NULLABLE
- name: double_value
type: FLOAT
mode: NULLABLE
- name: set_timestamp_micros
type: INTEGER
mode: NULLABLE
- name: user_first_touch_timestamp
type: INTEGER
mode: NULLABLE
- name: user_ltv
type: RECORD
mode: NULLABLE
fields:
- name: revenue
type: FLOAT
mode: NULLABLE
- name: currency
type: STRING
mode: NULLABLE
- name: device
type: RECORD
mode: NULLABLE
fields:
- name: category
type: STRING
mode: NULLABLE
- name: mobile_brand_name
type: STRING
mode: NULLABLE
- name: mobile_model_name
type: STRING
mode: NULLABLE
- name: mobile_marketing_name
type: STRING
mode: NULLABLE
- name: mobile_os_hardware_model
type: STRING
mode: NULLABLE
- name: operating_system
type: STRING
mode: NULLABLE
- name: operating_system_version
type: STRING
mode: NULLABLE
- name: vendor_id
type: STRING
mode: NULLABLE
- name: advertising_id
type: STRING
mode: NULLABLE
- name: language
type: STRING
mode: NULLABLE
- name: is_limited_ad_tracking
type: STRING
mode: NULLABLE
- name: time_zone_offset_seconds
type: INTEGER
mode: NULLABLE
- name: browser
type: STRING
mode: NULLABLE
- name: browser_version
type: STRING
mode: NULLABLE
- name: web_info
type: RECORD
mode: NULLABLE
fields:
- name: browser
type: STRING
mode: NULLABLE
- name: browser_version
type: STRING
mode: NULLABLE
- name: hostname
type: STRING
mode: NULLABLE
- name: geo
type: RECORD
mode: NULLABLE
fields:
- name: city
type: STRING
mode: NULLABLE
- name: country
type: STRING
mode: NULLABLE
- name: continent
type: STRING
mode: NULLABLE
- name: region
type: STRING
mode: NULLABLE
- name: sub_continent
type: STRING
mode: NULLABLE
- name: metro
type: STRING
mode: NULLABLE
- name: app_info
type: RECORD
mode: NULLABLE
fields:
- name: id
type: STRING
mode: NULLABLE
- name: version
type: STRING
mode: NULLABLE
- name: install_store
type: STRING
mode: NULLABLE
- name: firebase_app_id
type: STRING
mode: NULLABLE
- name: install_source
type: STRING
mode: NULLABLE
- name: traffic_source
type: RECORD
mode: NULLABLE
fields:
- name: name
type: STRING
mode: NULLABLE
- name: medium
type: STRING
mode: NULLABLE
- name: source
type: STRING
mode: NULLABLE
- name: stream_id
type: STRING
mode: NULLABLE
- name: platform
type: STRING
mode: NULLABLE
- name: event_dimensions
type: RECORD
mode: NULLABLE
fields:
- name: hostname
type: STRING
mode: NULLABLE
- name: ecommerce
type: RECORD
mode: NULLABLE
fields:
- name: total_item_quantity
type: INTEGER
mode: NULLABLE
- name: purchase_revenue_in_usd
type: FLOAT
mode: NULLABLE
- name: purchase_revenue
type: FLOAT
mode: NULLABLE
- name: refund_value_in_usd
type: FLOAT
mode: NULLABLE
- name: refund_value
type: FLOAT
mode: NULLABLE
- name: shipping_value_in_usd
type: FLOAT
mode: NULLABLE
- name: shipping_value
type: FLOAT
mode: NULLABLE
- name: tax_value_in_usd
type: FLOAT
mode: NULLABLE
- name: tax_value
type: FLOAT
mode: NULLABLE
- name: unique_items
type: INTEGER
mode: NULLABLE
- name: transaction_id
type: STRING
mode: NULLABLE
- name: items
type: RECORD
mode: REPEATED
fields:
- name: item_id
type: STRING
mode: NULLABLE
- name: item_name
type: STRING
mode: NULLABLE
- name: item_brand
type: STRING
mode: NULLABLE
- name: item_variant
type: STRING
mode: NULLABLE
- name: item_category
type: STRING
mode: NULLABLE
- name: item_category2
type: STRING
mode: NULLABLE
- name: item_category3
type: STRING
mode: NULLABLE
- name: item_category4
type: STRING
mode: NULLABLE
- name: item_category5
type: STRING
mode: NULLABLE
- name: price_in_usd
type: FLOAT
mode: NULLABLE
- name: price
type: FLOAT
mode: NULLABLE
- name: quantity
type: INTEGER
mode: NULLABLE
- name: item_revenue_in_usd
type: FLOAT
mode: NULLABLE
- name: item_revenue
type: FLOAT
mode: NULLABLE
- name: item_refund_in_usd
type: FLOAT
mode: NULLABLE
- name: item_refund
type: FLOAT
mode: NULLABLE
- name: coupon
type: STRING
mode: NULLABLE
- name: affiliation
type: STRING
mode: NULLABLE
- name: location_id
type: STRING
mode: NULLABLE
- name: item_list_id
type: STRING
mode: NULLABLE
- name: item_list_name
type: STRING
mode: NULLABLE
- name: item_list_index
type: STRING
mode: NULLABLE
- name: promotion_id
type: STRING
mode: NULLABLE
- name: promotion_name
type: STRING
mode: NULLABLE
- name: creative_name
type: STRING
mode: NULLABLE
- name: creative_slot
type: STRING
mode: NULLABLE
- name: item_params
type: RECORD
mode: REPEATED
fields:
- name: key
type: STRING
mode: NULLABLE
- name: value
type: RECORD
mode: NULLABLE
fields:
- name: string_value
type: STRING
mode: NULLABLE
- name: int_value
type: INTEGER
mode: NULLABLE
- name: float_value
type: FLOAT
mode: NULLABLE
- name: double_value
type: FLOAT
mode: NULLABLE
- name: collected_traffic_source
type: RECORD
mode: NULLABLE
fields:
- name: manual_campaign_id
type: STRING
mode: NULLABLE
- name: manual_campaign_name
type: STRING
mode: NULLABLE
- name: manual_source
type: STRING
mode: NULLABLE
- name: manual_medium
type: STRING
mode: NULLABLE
- name: manual_term
type: STRING
mode: NULLABLE
- name: manual_content
type: STRING
mode: NULLABLE
- name: manual_source_platform
type: STRING
mode: NULLABLE
- name: manual_creative_format
type: STRING
mode: NULLABLE
- name: manual_marketing_tactic
type: STRING
mode: NULLABLE
- name: gclid
type: STRING
mode: NULLABLE
- name: dclid
type: STRING
mode: NULLABLE
- name: srsltid
type: STRING
mode: NULLABLE
- name: is_active_user
type: BOOLEAN
mode: NULLABLE
- name: batch_event_index
type: INTEGER
mode: NULLABLE
- name: batch_page_id
type: INTEGER
mode: NULLABLE
- name: batch_ordering_id
type: INTEGER
mode: NULLABLE
- name: session_traffic_source_last_click
type: RECORD
mode: NULLABLE
fields:
- name: manual_campaign
type: RECORD
mode: NULLABLE
fields:
- name: campaign_id
type: STRING
mode: NULLABLE
- name: campaign_name
type: STRING
mode: NULLABLE
- name: source
type: STRING
mode: NULLABLE
- name: medium
type: STRING
mode: NULLABLE
- name: term
type: STRING
mode: NULLABLE
- name: content
type: STRING
mode: NULLABLE
- name: source_platform
type: STRING
mode: NULLABLE
- name: creative_format
type: STRING
mode: NULLABLE
- name: marketing_tactic
type: STRING
mode: NULLABLE
- name: google_ads_campaign
type: RECORD
mode: NULLABLE
fields:
- name: customer_id
type: STRING
mode: NULLABLE
- name: account_name
type: STRING
mode: NULLABLE
- name: campaign_id
type: STRING
mode: NULLABLE
- name: campaign_name
type: STRING
mode: NULLABLE
- name: ad_group_id
type: STRING
mode: NULLABLE
- name: ad_group_name
type: STRING
mode: NULLABLE
- name: submission_date
type: DATE
mode: NULLABLE