Fix table name extraction and set write disposition for bigquery_etl_scheduled_queries_cost_v1

This commit is contained in:
Anna Scholtz 2020-08-26 10:04:21 -07:00
Родитель bd73aebb6f
Коммит d518b35575
1 изменённых файлов: 7 добавлений и 6 удалений

Просмотреть файл

@ -24,12 +24,13 @@ def create_query(query_paths, date):
return f"""
SELECT
'{date}' AS submission_date,
DATE('{date}') AS submission_date,
creation_time,
destination_table.dataset_id AS dataset,
destination_table.table_id AS table,
-- remove partition from table name
REGEXP_REPLACE(destination_table.table_id, r"\$.+", "") AS table,
total_bytes_processed,
total_bytes_processed / 1024 / 1024 / 1024 / 1024 * 5 AS cost
total_bytes_processed / 1024 / 1024 / 1024 / 1024 * 5 AS cost_usd
FROM `region-us`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
WHERE DATE(creation_time) = '{date}'
AND destination_table.dataset_id IN ({",".join(datasets)})
@ -49,10 +50,10 @@ def main():
query = create_query(query_paths, args.date)
partition = args.date.replace("-", "")
destination_table = (
f"{args.destination_dataset}.{args.destination_table}${partition}"
destination_table = f"{args.project}.{args.destination_dataset}.{args.destination_table}${partition}"
job_config = bigquery.QueryJobConfig(
destination=destination_table, write_disposition="WRITE_TRUNCATE"
)
job_config = bigquery.QueryJobConfig(destination=destination_table)
client.query(query, job_config=job_config).result()