Enforce more yaml lint rules (#1878)
This commit is contained in:
Daniel Thorn
GitHub
Родитель
fd2f3d7613
Коммит
dfeea39ac5
|
|
@ -314,7 +314,7 @@ jobs:
|
|||
- run:
|
||||
name: Determine docker image name
|
||||
command:
|
||||
# yamllint disable-line rule:line-length
|
||||
# yamllint disable-line rule:line-length
|
||||
echo 'IMAGE="${CIRCLE_PROJECT_USERNAME+$CIRCLE_PROJECT_USERNAME/}${CIRCLE_PROJECT_REPONAME:-bigquery-etl}:${CIRCLE_TAG:-latest}"' > $BASH_ENV
|
||||
- run:
|
||||
name: Build docker image
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
---
|
||||
extends: default
|
||||
|
||||
rules:
|
||||
line-length:
|
||||
allow-non-breakable-words: true
|
||||
allow-non-breakable-inline-mappings: true
|
||||
|
||||
document-start: disable
|
||||
indentation:
|
||||
indent-sequences: consistent
|
||||
|
||||
ignore: |
|
||||
.git/
|
||||
venv/
|
||||
bigquery_etl/events_daily/
|
||||
|
|
|
|||
|
|
@ -14,3 +14,4 @@ scheduling:
|
|||
parameters: ["submission_date:DATE:{% raw %}{{ds}}{% endraw %}"]
|
||||
referenced_tables: [['moz-fx-data-shared-prod', '{{ app_id }}_derived',
|
||||
'event_types_history_v1']]
|
||||
|
||||
|
|
|
|||
|
|
@ -17,3 +17,4 @@ scheduling:
|
|||
'event_types_v1'
|
||||
],
|
||||
]
|
||||
|
||||
|
|
|
|||
11
dags.yaml
11
dags.yaml
|
|
@ -243,7 +243,7 @@ bqetl_public_data_json:
|
|||
|
||||
bqetl_internet_outages:
|
||||
schedule_interval: 0 3 * * *
|
||||
description: >
|
||||
description: >
|
||||
DAG for building the internet outages datasets.
|
||||
See [bug 1640204](https://bugzilla.mozilla.org/show_bug.cgi?id=1640204).
|
||||
default_args:
|
||||
|
|
@ -350,8 +350,8 @@ bqetl_search_dashboard:
|
|||
default_args:
|
||||
depends_on_past: false
|
||||
email:
|
||||
- telemetry-alerts@mozilla.com
|
||||
- ssuh@mozilla.com
|
||||
- telemetry-alerts@mozilla.com
|
||||
- ssuh@mozilla.com
|
||||
email_on_failure: true
|
||||
email_on_retry: true
|
||||
owner: ssuh@mozilla.com
|
||||
|
|
@ -381,8 +381,8 @@ bqetl_internal_tooling:
|
|||
default_args:
|
||||
depends_on_past: false
|
||||
email:
|
||||
- wlachance@mozilla.com
|
||||
- telemetry-alerts@mozilla.com
|
||||
- wlachance@mozilla.com
|
||||
- telemetry-alerts@mozilla.com
|
||||
email_on_failure: true
|
||||
email_on_retry: true
|
||||
end_date: null
|
||||
|
|
@ -391,4 +391,3 @@ bqetl_internal_tooling:
|
|||
retry_delay: 30m
|
||||
start_date: '2020-06-01'
|
||||
schedule_interval: 0 4 * * *
|
||||
|
||||
|
|
|
|||
|
|
@ -6,31 +6,31 @@ site_author: Mozilla Data Platform Team
|
|||
# Repository
|
||||
repo_name: mozilla/bigquery-etl
|
||||
repo_url: https://github.com/mozilla/bigquery-etl/
|
||||
edit_uri: ""
|
||||
edit_uri: ""
|
||||
|
||||
# Configuration
|
||||
theme:
|
||||
name: material
|
||||
features:
|
||||
- navigation.tabs
|
||||
palette:
|
||||
primary: teal
|
||||
favicon: favicon.png
|
||||
custom_dir: overrides
|
||||
name: material
|
||||
features:
|
||||
- navigation.tabs
|
||||
palette:
|
||||
primary: teal
|
||||
favicon: favicon.png
|
||||
custom_dir: overrides
|
||||
|
||||
google_analytics:
|
||||
- UA-104326577-3
|
||||
- auto
|
||||
|
||||
plugins:
|
||||
- search
|
||||
- awesome-pages
|
||||
- search
|
||||
- awesome-pages
|
||||
nav:
|
||||
- index.md
|
||||
- Cookbooks:
|
||||
- Common workflows: cookbooks/common_workflows.md
|
||||
- Creating a derived dataset: cookbooks/creating_a_derived_dataset.md
|
||||
- Datasets:
|
||||
- ... | mozdata/**.md
|
||||
- UDFs:
|
||||
- ... | mozfun/**.md
|
||||
- index.md
|
||||
- Cookbooks:
|
||||
- Common workflows: cookbooks/common_workflows.md
|
||||
- Creating a derived dataset: cookbooks/creating_a_derived_dataset.md
|
||||
- Datasets:
|
||||
- ... | mozdata/**.md
|
||||
- UDFs:
|
||||
- ... | mozfun/**.md
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
description: Cost of executed experiment analysis queries.
|
||||
description: Cost of executed experiment analysis queries.
|
||||
friendly_name: Experiment Analysis Query Costs
|
||||
labels:
|
||||
incremental: true
|
||||
|
|
|
|||
|
|
@ -7,4 +7,3 @@ owners:
|
|||
- bewu@mozilla.com
|
||||
scheduling:
|
||||
dag_name: bqetl_google_analytics_derived
|
||||
|
||||
|
|
|
|||
|
|
@ -17,11 +17,9 @@ scheduling:
|
|||
dag_name: bqetl_account_ecosystem
|
||||
depends_on_past: true
|
||||
referenced_tables:
|
||||
- [
|
||||
"moz-fx-data-shared-prod",
|
||||
"firefox_accounts_stable",
|
||||
"account_ecosystem_v1",
|
||||
]
|
||||
- - "moz-fx-data-shared-prod"
|
||||
- "firefox_accounts_stable"
|
||||
- "account_ecosystem_v1"
|
||||
# This is an unpartitioned table where the script adds rows via INSERT INTO,
|
||||
# thus the custom settings below.$
|
||||
date_partition_parameter: null
|
||||
|
|
|
|||
|
|
@ -14,13 +14,9 @@ scheduling:
|
|||
# We access a restricted table for getting an HMAC key, so cannot dry run
|
||||
# and must explicitly list referenced tables.
|
||||
referenced_tables:
|
||||
- [
|
||||
"moz-fx-data-shared-prod",
|
||||
"firefox_accounts_stable",
|
||||
"account_ecosystem_v1",
|
||||
]
|
||||
- [
|
||||
"moz-fx-data-shared-prod",
|
||||
"account_ecosystem_derived",
|
||||
"ecosystem_user_id_lookup_v1",
|
||||
]
|
||||
- - "moz-fx-data-shared-prod"
|
||||
- "firefox_accounts_stable"
|
||||
- "account_ecosystem_v1"
|
||||
- - "moz-fx-data-shared-prod"
|
||||
- "account_ecosystem_derived"
|
||||
- "ecosystem_user_id_lookup_v1"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
friendly_name: Internet Outages
|
||||
# yamllint disable rule:line-length
|
||||
description: |-
|
||||
description: |-
|
||||
This contains a set aggregated metrics that correlate to internet
|
||||
outages for different countries in the world.
|
||||
|
||||
|
|
@ -14,7 +14,7 @@ description: |-
|
|||
- `proportion_undefined`: the proportion of users who failed to send
|
||||
telemetry for a reason that was not listed in the other cases.
|
||||
- `proportion_timeout`: the proportion of users that had their connection
|
||||
timeout while uploading telemetry
|
||||
timeout while uploading telemetry
|
||||
([after 90s, in Firefox Desktop](https://searchfox.org/mozilla-central/rev/fa2df28a49883612bd7af4dacd80cdfedcccd2f6/toolkit/components/telemetry/app/TelemetrySend.jsm#81)).
|
||||
- `proportion_abort`: the proportion of users that had their connection
|
||||
terminated by the client (for example, terminating open connections before
|
||||
|
|
|
|||
|
|
@ -15,4 +15,4 @@ scheduling:
|
|||
date_partition_parameter: null
|
||||
parameters: ["submission_date:DATE:{{ds}}"]
|
||||
referenced_tables: [['moz-fx-data-shared-prod', 'messaging_system_derived',
|
||||
'event_types_history_v1']]
|
||||
'event_types_history_v1']]
|
||||
|
|
|
|||
|
|
@ -18,4 +18,4 @@ scheduling:
|
|||
'messaging_system_derived',
|
||||
'event_types_v1'
|
||||
],
|
||||
]
|
||||
]
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ scheduling:
|
|||
dag_name: bqetl_monitoring
|
||||
arguments: ["--date", "{{ ds }}"]
|
||||
referenced_tables:
|
||||
- ['moz-fx-data-shared-prod',
|
||||
'payload_bytes_decoded',
|
||||
- ['moz-fx-data-shared-prod',
|
||||
'payload_bytes_decoded',
|
||||
'structured_*']
|
||||
- ['moz-fx-data-shared-prod', 'payload_bytes_decoded', 'stub_installer_*']
|
||||
- ['moz-fx-data-shared-prod', '*_stable', '*']
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ labels:
|
|||
scheduling:
|
||||
dag_name: bqetl_monitoring
|
||||
referenced_tables:
|
||||
- ['moz-fx-data-shared-prod',
|
||||
'payload_bytes_decoded',
|
||||
- ['moz-fx-data-shared-prod',
|
||||
'payload_bytes_decoded',
|
||||
'telemetry_telemetry__*']
|
||||
- ['moz-fx-data-shared-prod', 'telemetry_stable', '*']
|
||||
- ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
|
||||
|
|
|
|||
|
|
@ -10,5 +10,5 @@ labels:
|
|||
incremental: false
|
||||
scheduling:
|
||||
dag_name: bqetl_monitoring
|
||||
referenced_tables:
|
||||
referenced_tables:
|
||||
- ['moz-fx-data-shared-prod', '*_stable', '*']
|
||||
|
|
|
|||
|
|
@ -15,13 +15,9 @@ scheduling:
|
|||
parameters:
|
||||
- "date:DATE:{{ds}}"
|
||||
referenced_tables:
|
||||
- [
|
||||
"moz-fx-data-shared-prod",
|
||||
"firefox_accounts_derived",
|
||||
"fxa_auth_events_v1",
|
||||
]
|
||||
- [
|
||||
"moz-fx-data-shared-prod",
|
||||
"firefox_accounts_derived",
|
||||
"fxa_content_events_v1",
|
||||
]
|
||||
- - "moz-fx-data-shared-prod"
|
||||
- "firefox_accounts_derived"
|
||||
- "fxa_auth_events_v1"
|
||||
- - "moz-fx-data-shared-prod"
|
||||
- "firefox_accounts_derived"
|
||||
- "fxa_content_events_v1"
|
||||
|
|
|
|||
|
|
@ -15,4 +15,4 @@ scheduling:
|
|||
date_partition_parameter: null
|
||||
parameters: ["submission_date:DATE:{{ds}}"]
|
||||
referenced_tables: [['moz-fx-data-shared-prod', 'org_mozilla_firefox_derived',
|
||||
'event_types_history_v1']]
|
||||
'event_types_history_v1']]
|
||||
|
|
|
|||
|
|
@ -18,4 +18,4 @@ scheduling:
|
|||
'org_mozilla_firefox_derived',
|
||||
'event_types_v1'
|
||||
],
|
||||
]
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
friendly_name: Clients Last Seen
|
||||
# yamllint disable rule:line-length
|
||||
description: |-
|
||||
description: |-
|
||||
Captures history of activity of each client in 28 day
|
||||
windows for each submission date.
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,6 @@ labels:
|
|||
schedule: daily
|
||||
public_json: true
|
||||
public_bigquery: true
|
||||
review_bugs:
|
||||
- 1624528
|
||||
review_bugs:
|
||||
- 1624528
|
||||
incremental_export: false
|
||||
|
|
|
|||
|
|
@ -15,4 +15,4 @@ scheduling:
|
|||
date_partition_parameter: null
|
||||
parameters: ["submission_date:DATE:{{ds}}"]
|
||||
referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_derived',
|
||||
'event_types_history_v1']]
|
||||
'event_types_history_v1']]
|
||||
|
|
|
|||
|
|
@ -18,4 +18,4 @@ scheduling:
|
|||
'telemetry_derived',
|
||||
'event_types_v1'
|
||||
],
|
||||
]
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
friendly_name: Experiment Enrollment Aggregates Live
|
||||
description: >
|
||||
description: >
|
||||
View for live experiment enrollment events.
|
||||
labels:
|
||||
incremental: false
|
||||
|
|
|
|||
|
|
@ -6,4 +6,3 @@ labels:
|
|||
incremental: false
|
||||
owners:
|
||||
- ascholtz@mozilla.com
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
friendly_name: Experiment Enrollment Overall
|
||||
description: >
|
||||
description: >
|
||||
Overall number of clients enrolled in experiments,
|
||||
clustered by experiment.
|
||||
labels:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
friendly_name: Experiment Unenrollment Overall
|
||||
description: >
|
||||
description: >
|
||||
Overall number of clients that unenrolled from experiments,
|
||||
clustered by experiment.
|
||||
labels:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
friendly_name: Italy's mid-pandemic internet outage (May 2020)
|
||||
# yamllint disable rule:line-length
|
||||
description: |-
|
||||
description: |-
|
||||
The released data includes aggregated Firefox Desktop data for Italy
|
||||
from “health” and some fields of the “main” pings that were created
|
||||
between January 1st, 2020 up until March 31st, 2020. The different types
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ labels:
|
|||
incremental: true
|
||||
public_json: true
|
||||
public_bigquery: true
|
||||
review_bugs:
|
||||
- 1414839
|
||||
review_bugs:
|
||||
- 1414839
|
||||
incremental_export: false
|
||||
scheduling:
|
||||
dag_name: bqetl_ssl_ratios
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
---
|
||||
description: Parses and labels the components of a telemetry desktop ping
|
||||
submission uri Per
|
||||
submission uri Per
|
||||
https://docs.telemetry.mozilla.org/concepts/pipeline/http_edge_spec.html#special-handling-for-firefox-desktop-telemetry
|
||||
the format is
|
||||
the format is
|
||||
/submit/telemetry/docId/docType/appName/appVersion/appUpdateChannel/appBuildID
|
||||
e.g.
|
||||
e.g.
|
||||
/submit/telemetry/ce39b608-f595-4c69-b6a6-f7a436604648/main/Firefox/61.0a1/nightly/20180328030202
|
||||
friendly_name: Parse Desktop Telemetry Uri
|
||||
|
|
|
|||
|
|
@ -1,3 +1,2 @@
|
|||
description: ""
|
||||
friendly_name: Map from array offsets
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ description: |
|
|||
Convert the Fenix client_info.app_build-format string to
|
||||
a DATETIME. May return NULL on failure.
|
||||
|
||||
Fenix originally used an
|
||||
Fenix originally used an
|
||||
[8-digit app_build format](
|
||||
https://github.com/mozilla-mobile/fenix/blob/c7283447/automation/gradle/versionCode.gradle#L12-L16)
|
||||
|
||||
|
|
@ -17,7 +17,7 @@ description: |
|
|||
|
||||
The last date seen with an 8-digit build ID is 2020-08-10.
|
||||
|
||||
Newer builds use a
|
||||
Newer builds use a
|
||||
[10-digit format](
|
||||
https://github.com/mozilla-mobile/fenix/blob/e6ee13dc/buildSrc/src/main/java/Config.kt#L55-L112)
|
||||
where the integer represents a pattern consisting of 32 bits.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ description: |
|
|||
Normalize an operating system string to one of the three major desktop
|
||||
platforms, one of the two major mobile platforms, or "Other".
|
||||
|
||||
This is a reimplementation of
|
||||
This is a reimplementation of
|
||||
[logic used in the data pipeline](
|
||||
https://github.com/mozilla/gcp-ingestion/blob/a6928fb089f1652856147c4605df715f327edfcd/ingestion-beam/src/main/java/com/mozilla/telemetry/transforms/NormalizeAttributes.java#L52-L74)
|
||||
to populate `normalized_os`.
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ labels:
|
|||
public_json: true
|
||||
incremental: true
|
||||
incremental_export: true
|
||||
review_bugs:
|
||||
review_bugs:
|
||||
- 123456
|
||||
scheduling:
|
||||
dag_name: "bqetl_events"
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ labels:
|
|||
public_json: true
|
||||
incremental: true
|
||||
incremental_export: false
|
||||
review_bugs:
|
||||
review_bugs:
|
||||
- 123456
|
||||
scheduling:
|
||||
dag_name: "bqetl_events"
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ labels:
|
|||
public_json: true
|
||||
incremental: true
|
||||
incremental_export: true
|
||||
review_bugs:
|
||||
- 123456
|
||||
review_bugs:
|
||||
- 123456
|
||||
scheduling:
|
||||
dag_name: "bqetl_events"
|
||||
depends_on_past: false
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ labels:
|
|||
public_json: false
|
||||
incremental: false
|
||||
incremental_export: false
|
||||
review_bugs:
|
||||
- 1999999
|
||||
review_bugs:
|
||||
- 1999999
|
||||
scheduling:
|
||||
dag_name: "bqetl_core"
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ labels:
|
|||
public_json: true
|
||||
incremental: false
|
||||
incremental_export: false
|
||||
review_bugs:
|
||||
- 1999999
|
||||
- 12121212
|
||||
review_bugs:
|
||||
- 1999999
|
||||
- 12121212
|
||||
scheduling:
|
||||
dag_name: "bqetl_core"
|
||||
depends_on_past: true
|
||||
|
|
|
|||
|
|
@ -53,7 +53,6 @@
|
|||
addon_version: 0.9.6
|
||||
total_uri_count: 1
|
||||
is_default_browser: false
|
||||
user_pref_browser_search_region: null
|
||||
user_pref_browser_search_region: DE
|
||||
- &client_a
|
||||
<<: *base
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче