Deng 6360 add arguments to metadata file (#6517)
* update logic and metadata to use the Cinder token secret * add trailing white space in jiinja var
This commit is contained in:
Родитель
ad51416036
Коммит
9fdf7611f3
|
@ -8,6 +8,7 @@ labels:
|
|||
owner1: example
|
||||
scheduling:
|
||||
dag_name: bqetl_addons
|
||||
arguments: ["--date", "{{ ds }}", "--cinder_bearer_token", "{{ var.value.CINDER_TOKEN }}"]
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
|
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from argparse import ArgumentParser
|
||||
from time import sleep
|
||||
|
@ -25,8 +24,6 @@ CSV_FIELDS = [
|
|||
"typed_metadata",
|
||||
]
|
||||
|
||||
CINDER_BEARER_TOKEN = os.environ.get("CINDER_TOKEN")
|
||||
|
||||
|
||||
def post_response(url, headers, data):
|
||||
"""POST response function."""
|
||||
|
@ -155,6 +152,7 @@ def main():
|
|||
parser.add_argument("--date", required=True)
|
||||
parser.add_argument("--project", default="moz-fx-data-shared-prod")
|
||||
parser.add_argument("--dataset", default="addon_moderations_derived")
|
||||
parser.add_argument("--cinder_bearer_token", required=True)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
@ -163,7 +161,7 @@ def main():
|
|||
table_name = "cinder_decisions_raw_v1"
|
||||
|
||||
date = args.date
|
||||
bearer_token = CINDER_BEARER_TOKEN
|
||||
bearer_token = args.cinder_bearer_token
|
||||
|
||||
data = []
|
||||
|
||||
|
|
Загрузка…
Ссылка в новой задаче