Correct BigQuery partitioning/clustering metadata in ETL `metadata.yaml` files (#3500)

* Add missing BigQuery partitioning/clustering metadata.
* Correct existing BigQuery partitioning/clustering metadata.
* Allow partition `field` metadata field to be omitted.
* List partition `type` metadata field first.

bigquery_etl/metadata/parse_metadata.py

@@ -57,8 +57,8 @@ class PartitionType(enum.Enum):
 class PartitionMetadata:
     """Metadata for defining BigQuery table partitions."""
 
-    field: str
     type: PartitionType
+    field: Optional[str] = attr.ib(None)
     require_partition_filter: bool = attr.ib(True)
     expiration_days: Optional[float] = attr.ib(None)
 

docs/cookbooks/creating_a_derived_dataset.md

@@ -58,8 +58,8 @@ owners:
   - wlachance@mozilla.com
 bigquery:
   time_partitioning:
-    field: date
     type: day
+    field: date
     require_partition_filter: true
     expiration_days: null
   clustering:
@@ -91,8 +91,8 @@ owners:
   - wlachance@mozilla.com
 bigquery:
   time_partitioning:
-    field: date
     type: day
+    field: date
     require_partition_filter: true
     expiration_days: null
   clustering:

sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_v1/metadata.yaml

@@ -25,3 +25,8 @@ scheduling:
     # explicit query file path is necessary because the destination table
     # includes a partition identifier that is not in the path
     sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_v1/query.sql
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/flows_v1/metadata.yaml

@@ -9,3 +9,8 @@ labels:
 scheduling:
   dag_name: bqetl_subplat
   query_project: moz-fx-data-shared-prod
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-experiments/monitoring/query_cost_v1/metadata.yaml

@@ -7,3 +7,8 @@ owners:
 scheduling:
   dag_name: bqetl_experiments_daily
   referenced_tables: [['region-us', 'INFORMATION_SCHEMA', 'JOBS_BY_PROJECT']]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_timestamp
+    require_partition_filter: false

sql/moz-fx-data-marketing-prod/acoustic/contact_current_snapshot_v1/metadata.yaml

@@ -21,13 +21,12 @@ scheduling:
 
 bigquery:
   time_partitioning:
-    field: last_modified_date
     type: day
+    field: last_modified_date
     require_partition_filter: false
   clustering:
     fields:
-      - double_opt_in
       - has_opted_out_of_email
+      - double_opt_in
       - email_lang
       - mailing_country
-      - cohort

sql/moz-fx-data-marketing-prod/acoustic/contact_raw_v1/metadata.yaml

@@ -34,6 +34,12 @@ scheduling:
 
 bigquery:
   time_partitioning:
-    field: last_modified_date
     type: day
+    field: last_modified_date
     require_partition_filter: false
+  clustering:
+    fields:
+      - has_opted_out_of_email
+      - double_opt_in
+      - email_lang
+      - mailing_country

sql/moz-fx-data-marketing-prod/acoustic/contact_v1/metadata.yaml

@@ -24,13 +24,12 @@ scheduling:
 
 bigquery:
   time_partitioning:
-    field: last_modified_date
     type: day
+    field: last_modified_date
     require_partition_filter: false
   clustering:
     fields:
-      - double_opt_in
       - has_opted_out_of_email
+      - double_opt_in
       - email_lang
       - mailing_country
-      - cohort

sql/moz-fx-data-marketing-prod/acoustic/raw_recipient_raw_v1/metadata.yaml

@@ -37,6 +37,11 @@ scheduling:
 
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false
+  clustering:
+    fields:
+      - event_type
+      - recipient_type
+      - body_type

sql/moz-fx-data-marketing-prod/acoustic/raw_recipient_v1/metadata.yaml

@@ -24,8 +24,8 @@ scheduling:
 
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false
   clustering:
     fields:

sql/moz-fx-data-marketing-prod/ga_derived/blogs_daily_summary_v1/metadata.yaml

@@ -10,3 +10,14 @@ scheduling:
   referenced_tables:
     - ['moz-fx-data-marketing-prod', 'ga_derived', 'blogs_goals_v1']
     - ['moz-fx-data-marketing-prod', 'ga_derived', 'blogs_sessions_v1']
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - country
+      - browser
+      - blog
+      - subblog

sql/moz-fx-data-marketing-prod/ga_derived/blogs_goals_v1/metadata.yaml

@@ -9,3 +9,8 @@ scheduling:
   dag_name: bqetl_google_analytics_derived
   referenced_tables:
     - ['moz-fx-data-marketing-prod', 'ga_derived', 'blogs_empty_check_v1']
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false

sql/moz-fx-data-marketing-prod/ga_derived/blogs_landing_page_summary_v1/metadata.yaml

@@ -11,3 +11,14 @@ scheduling:
   referenced_tables:
     - ["moz-fx-data-marketing-prod", "ga_derived", "blogs_goals_v1"]
     - ["moz-fx-data-marketing-prod", "ga_derived", "blogs_sessions_v1"]
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - cleaned_landing_page
+      - browser
+      - blog
+      - subblog

sql/moz-fx-data-marketing-prod/ga_derived/blogs_sessions_v1/metadata.yaml

@@ -10,3 +10,8 @@ scheduling:
   dag_name: bqetl_google_analytics_derived
   referenced_tables:
     - ["moz-fx-data-marketing-prod", "ga_derived", "blogs_empty_check_v1"]
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false

sql/moz-fx-data-marketing-prod/ga_derived/www_site_downloads_v1/metadata.yaml

@@ -7,3 +7,8 @@ owners:
   - ascholtz@mozilla.com
 scheduling:
   dag_name: bqetl_google_analytics_derived
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false

sql/moz-fx-data-marketing-prod/ga_derived/www_site_events_metrics_v1/metadata.yaml

@@ -7,3 +7,14 @@ owners:
   - ascholtz@mozilla.com
 scheduling:
   dag_name: bqetl_google_analytics_derived
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - page_name
+      - event_category
+      - event_action
+      - event_label

sql/moz-fx-data-marketing-prod/ga_derived/www_site_hits_v1/metadata.yaml

@@ -9,3 +9,8 @@ scheduling:
   dag_name: bqetl_google_analytics_derived
   referenced_tables:
     - ['moz-fx-data-marketing-prod', 'ga_derived', 'www_site_empty_check_v1']
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false

sql/moz-fx-data-marketing-prod/ga_derived/www_site_landing_page_metrics_v1/metadata.yaml

@@ -7,3 +7,14 @@ owners:
   - ascholtz@mozilla.com
 scheduling:
   dag_name: bqetl_google_analytics_derived
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - page_name
+      - country
+      - locale
+      - medium

sql/moz-fx-data-marketing-prod/ga_derived/www_site_metrics_summary_v1/metadata.yaml

@@ -7,3 +7,14 @@ owners:
   - ascholtz@mozilla.com
 scheduling:
   dag_name: bqetl_google_analytics_derived
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - country
+      - browser
+      - source
+      - medium

sql/moz-fx-data-marketing-prod/ga_derived/www_site_page_metrics_v1/metadata.yaml

@@ -7,3 +7,14 @@ owners:
   - ascholtz@mozilla.com
 scheduling:
   dag_name: bqetl_google_analytics_derived
+bigquery:
+  time_partitioning:
+    type: day
+    field: date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - page_name
+      - country
+      - locale
+      - medium

sql/moz-fx-data-marketing-prod/iprospect/adspend_meta_v1/metadata.yaml

@@ -13,6 +13,6 @@ scheduling:
   depends_on_past: true
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false

sql/moz-fx-data-marketing-prod/iprospect/adspend_raw_v1/metadata.yaml

@@ -16,6 +16,6 @@ scheduling:
   arguments: ["--date", "{{ ds }}"]
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false

sql/moz-fx-data-marketing-prod/iprospect/adspend_v1/metadata.yaml

@@ -18,6 +18,6 @@ scheduling:
 
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: date
     require_partition_filter: false

sql/moz-fx-data-shared-prod/activity_stream_bi/impression_stats_by_experiment_v1/metadata.yaml

@@ -10,6 +10,15 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_activity_stream
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_timestamp
+    require_partition_filter: true
+    expiration_days: 180
+  clustering:
+    fields:
+      - experiment_id
 workgroup_access:
   - role: roles/bigquery.dataViewer
     members:

sql/moz-fx-data-shared-prod/activity_stream_bi/impression_stats_flat_v1/metadata.yaml

@@ -8,6 +8,16 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_activity_stream
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_timestamp
+    require_partition_filter: true
+    expiration_days: 180
+  clustering:
+    fields:
+      - release_channel
+      - sample_id
 workgroup_access:
   - role: roles/bigquery.dataViewer
     members:

sql/moz-fx-data-shared-prod/amo_dev/amo_stats_dau_v2/metadata.yaml

@@ -10,3 +10,11 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - addon_id

sql/moz-fx-data-shared-prod/amo_dev/amo_stats_installs_v3/metadata.yaml

@@ -10,3 +10,11 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - hashed_addon_id

sql/moz-fx-data-shared-prod/amo_prod/amo_stats_dau_v2/metadata.yaml

@@ -15,3 +15,11 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - addon_id

sql/moz-fx-data-shared-prod/amo_prod/amo_stats_installs_v3/metadata.yaml

@@ -15,3 +15,11 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - hashed_addon_id

sql/moz-fx-data-shared-prod/amo_prod/desktop_addons_by_client_v1/metadata.yaml

@@ -15,3 +15,11 @@ scheduling:
   # query to get it, and that would be slow because main_v4 is referenced
   referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_stable',
                        'main_v4']]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - sample_id

sql/moz-fx-data-shared-prod/amo_prod/fenix_addons_by_client_v1/metadata.yaml

@@ -12,3 +12,11 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - sample_id

sql/moz-fx-data-shared-prod/contextual_services_derived/adm_forecasting_v1/metadata.yaml

@@ -10,7 +10,7 @@ scheduling:
   dag_name: bqetl_ctxsvc_derived
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: true
     expiration_days: null

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_spons_tiles_v1/metadata.yaml

@@ -8,5 +8,9 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_ctxsvc_derived
-bigquery: null
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
 references: {}

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/metadata.yaml

@@ -8,5 +8,9 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_ctxsvc_derived
-bigquery: null
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
 references: {}

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/metadata.yaml

@@ -9,3 +9,12 @@ labels:
 scheduling:
   dag_name: bqetl_ctxsvc_derived
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - source
+      - event_type

sql/moz-fx-data-shared-prod/domain_metadata_derived/top_domains_v1/metadata.yaml

@@ -13,8 +13,8 @@ labels:
 
 bigquery:
   time_partitioning:
-    field: submission_date
     type: month
+    field: submission_date
     require_partition_filter: true
 
 scheduling:

sql/moz-fx-data-shared-prod/fenix_derived/attributable_clients_v1/metadata.yaml

@@ -26,9 +26,9 @@ scheduling:
     execution_delta: 0h
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
-    require_partition_filter: true
+    field: submission_date
+    require_partition_filter: false
   clustering:
     fields:
     - adjust_network

sql/moz-fx-data-shared-prod/fenix_derived/clients_yearly_v1/metadata.yaml

@@ -22,8 +22,8 @@ scheduling:
     execution_delta: 1h
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: true
   clustering:
     fields:

sql/moz-fx-data-shared-prod/fenix_derived/event_types_history_v1/metadata.yaml

@@ -17,3 +17,12 @@ labels:
 scheduling:
   dag_name: bqetl_fenix_event_rollup
   depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - category
+      - event

sql/moz-fx-data-shared-prod/fenix_derived/events_daily_v1/metadata.yaml

@@ -20,3 +20,11 @@ scheduling:
       'event_types_v1'
     ],
   ]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - sample_id

sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/metadata.yaml

@@ -43,8 +43,8 @@ scheduling:
   - submission_date:DATE:{{ds}}
 bigquery:
   time_partitioning:
-    field: first_seen_date
     type: day
+    field: first_seen_date
     require_partition_filter: false
   clustering:
     fields:

sql/moz-fx-data-shared-prod/fenix_derived/new_profile_activation_v1/metadata.yaml

@@ -7,8 +7,8 @@ labels:
   incremental: true
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: true
     expiration_days: null
   clustering:

sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/metadata.yaml

@@ -14,8 +14,8 @@ scheduling:
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
 bigquery:
   time_partitioning:
-    field: date
     type: day
+    field: date
     require_partition_filter: false
     expiration_days: null
   clustering: null

sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/metadata.yaml

@@ -13,8 +13,8 @@ scheduling:
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
 bigquery:
   time_partitioning:
-    field: date
     type: day
+    field: date
     require_partition_filter: false
     expiration_days: null
   clustering: null

sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/metadata.yaml

@@ -17,3 +17,12 @@ labels:
 scheduling:
   dag_name: bqetl_event_rollup
   depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - category
+      - event

sql/moz-fx-data-shared-prod/firefox_accounts_derived/events_daily_v1/metadata.yaml

@@ -20,3 +20,11 @@ scheduling:
       'event_types_v1'
     ],
   ]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - sample_id

sql/moz-fx-data-shared-prod/firefox_accounts_derived/exact_mau28_v1/metadata.yaml

@@ -9,3 +9,8 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_fxa_events
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/funnel_events_source_v1/metadata.yaml

@@ -13,4 +13,8 @@ scheduling:
   dag_name: bqetl_event_rollup
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
   task_name: funnel_events_source__v1
-bigquery: null
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/metadata.yaml

@@ -9,3 +9,11 @@ labels:
   application: fxa
   incremental: true
   schedule: daily
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date_pacific
+    require_partition_filter: false
+  clustering:
+    fields:
+      - user_id

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/metadata.yaml

@@ -15,3 +15,8 @@ labels:
 #   dag_name: bqetl_fxa_events
 #   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
 #   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/metadata.yaml

@@ -11,3 +11,8 @@ scheduling:
   dag_name: bqetl_fxa_events
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: true

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/metadata.yaml

@@ -11,3 +11,8 @@ scheduling:
   dag_name: bqetl_fxa_events
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/metadata.yaml

@@ -16,3 +16,8 @@ scheduling:
   # depends only on fxa logs produced via Stackdriver integration, so no other
   # scheduled tasks are involved and the referenced_tables list is empty.
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_timestamp
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/metadata.yaml

@@ -14,3 +14,11 @@ labels:
 scheduling:
   dag_name: bqetl_fxa_events
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false
+  clustering:
+    fields:
+      - event

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/metadata.yaml

@@ -14,3 +14,11 @@ labels:
 scheduling:
   dag_name: bqetl_fxa_events
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false
+  clustering:
+    fields:
+      - event

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml

@@ -17,3 +17,11 @@ scheduling:
   # depends only on fxa logs produced via Stackdriver integration, so no other
   # scheduled tasks are involved and the referenced_tables list is empty.
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false
+  clustering:
+    fields:
+      - command

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_stdout_events_v1/metadata.yaml

@@ -10,3 +10,8 @@ labels:
 scheduling:
   dag_name: bqetl_fxa_events
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v1/metadata.yaml

@@ -15,8 +15,8 @@ scheduling:
   dag_name: bqetl_fxa_events
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false
   clustering:
     fields:

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_first_seen_v1/metadata.yaml

@@ -21,8 +21,8 @@ scheduling:
   - submission_date:DATE:{{ds}}
 bigquery:
   time_partitioning:
-    field: first_seen_date
     type: day
+    field: first_seen_date
     require_partition_filter: false
   clustering:
     fields:

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/metadata.yaml

@@ -21,8 +21,8 @@ scheduling:
   start_date: '2019-04-23'
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: true
     expiration_days: null
   clustering:

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v1/metadata.yaml

@@ -10,3 +10,12 @@ labels:
 scheduling:
   dag_name: bqetl_fxa_events
   date_partition_parameter: submission_date
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - service
+      - user_id

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_daily_v1/metadata.yaml

@@ -24,8 +24,8 @@ scheduling:
   date_partition_parameter: submission_date
 bigquery:
   time_partitioning:
-    field: timestamp
     type: day
+    field: timestamp
     require_partition_filter: true
   clustering:
     fields:

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_first_seen_v1/metadata.yaml

@@ -25,8 +25,8 @@ scheduling:
     ]
 bigquery:
   time_partitioning:
-    field: first_seen_date
     type: day
+    field: first_seen_date
     require_partition_filter: true
   clustering:
     fields:

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_last_seen_v1/metadata.yaml

@@ -22,8 +22,8 @@ scheduling:
     ]
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: true
   clustering:
     fields:

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1/metadata.yaml

@@ -15,3 +15,12 @@ labels:
 #   # making it incremental is possible but nuanced since it windows over
 #   # events that may cross the midnight boundary.
 #   date_partition_parameter: null
+bigquery:
+  time_partitioning:
+    type: day
+    field: first_service_timestamp
+    require_partition_filter: false
+  clustering:
+    fields:
+      - service
+      - user_id

sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/metadata.yaml

@@ -14,3 +14,12 @@ labels:
 #   dag_name: bqetl_fxa_events
 #   depends_on_past: true
 #   start_date: '2019-10-08'
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - service
+      - user_id

sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_auth_events_v1/metadata.yaml

@@ -12,3 +12,8 @@ scheduling:
   dag_name: bqetl_fxa_events
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: true

sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_content_events_v1/metadata.yaml

@@ -12,3 +12,8 @@ scheduling:
   dag_name: bqetl_fxa_events
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
   referenced_tables: []
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: true

sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_stdout_events_v1/metadata.yaml

@@ -11,3 +11,8 @@ labels:
 scheduling:
   dag_name: bqetl_fxa_events
   arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: true

sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profile_activation_v1/metadata.yaml

@@ -7,8 +7,8 @@ labels:
   incremental: true
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: true
     expiration_days: null
   clustering:

sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/metadata.yaml

@@ -76,8 +76,8 @@ scheduling:
     - health_v4
 bigquery:
   time_partitioning:
-    field: datetime
     type: day
+    field: datetime
     require_partition_filter: null
   clustering:
     fields:

sql/moz-fx-data-shared-prod/messaging_system_derived/cfr_exact_mau28_by_dimensions_v1/metadata.yaml

@@ -8,3 +8,8 @@ labels:
   incremental: true
 scheduling:
   dag_name: bqetl_messaging_system
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/messaging_system_derived/cfr_users_daily_v1/metadata.yaml

@@ -7,3 +7,11 @@ labels:
   incremental: true
 scheduling:
   dag_name: bqetl_messaging_system
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - release_channel

sql/moz-fx-data-shared-prod/messaging_system_derived/cfr_users_last_seen_v1/metadata.yaml

@@ -10,3 +10,11 @@ labels:
 scheduling:
   dag_name: bqetl_messaging_system
   depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - release_channel

sql/moz-fx-data-shared-prod/messaging_system_derived/event_types_history_v1/metadata.yaml

@@ -17,3 +17,12 @@ labels:
 scheduling:
   dag_name: bqetl_event_rollup
   depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - category
+      - event

sql/moz-fx-data-shared-prod/messaging_system_derived/events_daily_v1/metadata.yaml

@@ -20,3 +20,11 @@ scheduling:
       'event_types_v1'
     ],
   ]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - sample_id

sql/moz-fx-data-shared-prod/messaging_system_derived/onboarding_exact_mau28_by_dimensions_v1/metadata.yaml

@@ -10,3 +10,8 @@ labels:
 scheduling:
   dag_name: bqetl_messaging_system
   task_name: messaging_system_onboarding_exact_mau28_by_dimensions
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/messaging_system_derived/onboarding_users_daily_v1/metadata.yaml

@@ -7,3 +7,11 @@ labels:
   incremental: true
 scheduling:
   dag_name: bqetl_messaging_system
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - release_channel

sql/moz-fx-data-shared-prod/messaging_system_derived/onboarding_users_last_seen_v1/metadata.yaml

@@ -10,3 +10,11 @@ labels:
 scheduling:
   dag_name: bqetl_messaging_system
   depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - release_channel

sql/moz-fx-data-shared-prod/messaging_system_derived/snippets_exact_mau28_by_dimensions_v1/metadata.yaml

@@ -10,3 +10,8 @@ labels:
 scheduling:
   dag_name: bqetl_messaging_system
   task_name: messaging_system_snippets_exact_mau28_by_dimensions
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/messaging_system_derived/snippets_users_daily_v1/metadata.yaml

@@ -7,3 +7,11 @@ labels:
   incremental: true
 scheduling:
   dag_name: bqetl_messaging_system
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - release_channel

sql/moz-fx-data-shared-prod/messaging_system_derived/snippets_users_last_seen_v1/metadata.yaml

@@ -10,3 +10,11 @@ labels:
 scheduling:
   dag_name: bqetl_messaging_system
   depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - release_channel

sql/moz-fx-data-shared-prod/monitoring_derived/average_ping_sizes_v1/metadata.yaml

@@ -10,3 +10,8 @@ scheduling:
   referenced_tables:
     - ['moz-fx-data-shared-prod', 'monitoring_derived', 'stable_table_sizes_v1']
     - ['moz-fx-data-shared-prod', '*_stable', "*"]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_scheduled_queries_cost_v1/metadata.yaml

@@ -9,3 +9,8 @@ owners:
 scheduling:
   dag_name: bqetl_monitoring
   arguments: ["--date", "{{ ds }}"]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_scheduled_query_usage_v1/metadata.yaml

@@ -10,3 +10,8 @@ owners:
 scheduling:
   dag_name: bqetl_monitoring
   arguments: ["--date", "{{ ds }}"]
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_table_storage_timeline_daily_v1/metadata.yaml

@@ -10,6 +10,6 @@ scheduling:
   arguments: ["--date", "{{ ds }}"]
 bigquery:
   time_partitioning:
-    field: change_date
     type: day
+    field: change_date
     require_partition_filter: true

sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_tables_inventory_v1/metadata.yaml

@@ -10,5 +10,6 @@ scheduling:
   arguments: ["--date", "{{ ds }}"]
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
+    require_partition_filter: true

sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v1/metadata.yaml

@@ -10,6 +10,6 @@ scheduling:
   arguments: ["--date", "{{ ds }}"]
 bigquery:
   time_partitioning:
-    field: creation_date
     type: day
+    field: creation_date
     require_partition_filter: true

sql/moz-fx-data-shared-prod/monitoring_derived/column_size_v1/metadata.yaml

@@ -10,3 +10,8 @@ scheduling:
   arguments: ["--date", "{{ ds }}"]
   referenced_tables:
     - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/deletion_request_volume_v1/metadata.yaml

@@ -18,3 +18,8 @@ scheduling:
     - ['moz-fx-data-shared-prod',
        'monitoring',
        'payload_bytes_decoded_telemetry']
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v2/metadata.yaml

@@ -9,3 +9,14 @@ labels:
   incremental: true
 scheduling:
   dag_name: bqetl_monitoring
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - document_namespace
+      - document_type
+      - path
+      - job_name

sql/moz-fx-data-shared-prod/monitoring_derived/stable_and_derived_table_sizes_v1/metadata.yaml

@@ -15,3 +15,8 @@ scheduling:
     - ['moz-fx-data-shared-prod', '*_stable', '*']
     - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
     - ['moz-fx-data-shared-prod', '*_derived', '*']
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/stable_table_column_counts_v1/metadata.yaml

@@ -15,6 +15,6 @@ scheduling:
   parameters: ["submission_date:DATE:{{ds}}"]
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/structured_distinct_docids_v1/metadata.yaml

@@ -10,8 +10,8 @@ labels:
   incremental: true
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false
 scheduling:
   dag_name: bqetl_monitoring

sql/moz-fx-data-shared-prod/monitoring_derived/structured_missing_columns_v1/metadata.yaml

@@ -12,3 +12,8 @@ scheduling:
   arguments: ["--date", "{{ ds }}"]
   referenced_tables:
     - ['moz-fx-data-shared-prod', '*_stable', '*']
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_distinct_docids_v1/metadata.yaml

@@ -10,8 +10,8 @@ labels:
   incremental: true
 bigquery:
   time_partitioning:
-    field: submission_date
     type: day
+    field: submission_date
     require_partition_filter: false
 scheduling:
   dag_name: bqetl_monitoring

sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v3/metadata.yaml

@@ -12,3 +12,8 @@ scheduling:
   dag_name: bqetl_monitoring
   referenced_tables:
     - ['moz-fx-data-shared-prod', '*_stable', '*']
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/mozilla_vpn_derived/active_subscription_ids_v1/metadata.yaml

@@ -14,3 +14,8 @@ scheduling:
   # delay aggregates by 7 days, to ensure data is complete
   date_partition_offset: -7
   date_partition_parameter: date
+bigquery:
+  time_partitioning:
+    type: day
+    field: active_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/mozilla_vpn_derived/active_subscriptions_v1/metadata.yaml

@@ -11,3 +11,8 @@ scheduling:
   # delay aggregates by 7 days, to ensure data is complete
   date_partition_offset: -7
   date_partition_parameter: date
+bigquery:
+  time_partitioning:
+    type: day
+    field: active_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/metadata.yaml

@@ -8,3 +8,8 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_subplat
+bigquery:
+  time_partitioning:
+    type: day
+    field: timestamp
+    require_partition_filter: false