diff --git a/bigquery_etl/cli/query.py b/bigquery_etl/cli/query.py index a174249e79..99836ef0a1 100644 --- a/bigquery_etl/cli/query.py +++ b/bigquery_etl/cli/query.py @@ -750,7 +750,7 @@ def _validate_schema(query_file): if not query_schema.compatible(table_schema): click.echo( - f"Schema for query in {query_file_path} " + f"ERROR: Schema for query in {query_file_path} " f"incompatible with schema deployed for " f"{project_name}.{dataset_name}.{table_name}", err=True, diff --git a/bigquery_etl/metadata/parse_metadata.py b/bigquery_etl/metadata/parse_metadata.py index 6209a72132..58293b78c2 100644 --- a/bigquery_etl/metadata/parse_metadata.py +++ b/bigquery_etl/metadata/parse_metadata.py @@ -207,8 +207,26 @@ class Metadata: if label_value == "": metadata_dict["labels"][label_key] = True + # Use literal yaml representation for descriptions + class Literal(str): + pass + + def literal_presenter(dumper, data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + yaml.add_representer(Literal, literal_presenter) + + if "description" in metadata_dict: + metadata_dict["description"] = Literal(metadata_dict["description"]) + converter = cattr.Converter() - file.write_text(yaml.dump(converter.unstructure(metadata_dict))) + file.write_text( + yaml.dump( + converter.unstructure(metadata_dict), + default_flow_style=False, + sort_keys=False, + ) + ) def is_public_bigquery(self): """Return true if the public_bigquery flag is set.""" diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_v6/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_v6/metadata.yaml index b213c51ee4..138405bcd0 100755 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_v6/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_v6/metadata.yaml @@ -1,35 +1,29 @@ -bigquery: - clustering: - fields: - - normalized_channel - - sample_id - time_partitioning: - field: submission_date - require_partition_filter: true - type: day -description: 'A daily aggregation over `main` pings from each Firefox - +friendly_name: Clients Daily +description: |- + A daily aggregation over `main` pings from each Firefox desktop client, partitioned by day. - It should normally be accessed through the user-facing view - `telemetry.clients_daily`. - Note that by end of Q1 2021, that view start referencing the downstream table - `clients_last_seen_joined_v1` which merges in fields based on the `event` - ping. - - See https://github.com/mozilla/bigquery-etl/issues/1761' -friendly_name: Clients Daily + See https://github.com/mozilla/bigquery-etl/issues/1761 +owners: +- dthorn@mozilla.com labels: application: firefox schedule: daily -owners: -- dthorn@mozilla.com scheduling: allow_field_addition_on_date: '2020-11-13' dag_name: bqetl_main_summary start_date: '2019-11-05' +bigquery: + time_partitioning: + field: submission_date + type: day + require_partition_filter: true + clustering: + fields: + - normalized_channel + - sample_id