Deng 6360 add arguments to metadata file (#6517)
* update logic and metadata to use the Cinder token secret * add trailing white space in jiinja var
This commit is contained in:
Родитель
ad51416036
Коммит
9fdf7611f3
|
@ -8,6 +8,7 @@ labels:
|
||||||
owner1: example
|
owner1: example
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_addons
|
dag_name: bqetl_addons
|
||||||
|
arguments: ["--date", "{{ ds }}", "--cinder_bearer_token", "{{ var.value.CINDER_TOKEN }}"]
|
||||||
bigquery:
|
bigquery:
|
||||||
time_partitioning:
|
time_partitioning:
|
||||||
type: day
|
type: day
|
||||||
|
|
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
import csv
|
import csv
|
||||||
import json
|
import json
|
||||||
import os
|
|
||||||
import tempfile
|
import tempfile
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
@ -25,8 +24,6 @@ CSV_FIELDS = [
|
||||||
"typed_metadata",
|
"typed_metadata",
|
||||||
]
|
]
|
||||||
|
|
||||||
CINDER_BEARER_TOKEN = os.environ.get("CINDER_TOKEN")
|
|
||||||
|
|
||||||
|
|
||||||
def post_response(url, headers, data):
|
def post_response(url, headers, data):
|
||||||
"""POST response function."""
|
"""POST response function."""
|
||||||
|
@ -155,6 +152,7 @@ def main():
|
||||||
parser.add_argument("--date", required=True)
|
parser.add_argument("--date", required=True)
|
||||||
parser.add_argument("--project", default="moz-fx-data-shared-prod")
|
parser.add_argument("--project", default="moz-fx-data-shared-prod")
|
||||||
parser.add_argument("--dataset", default="addon_moderations_derived")
|
parser.add_argument("--dataset", default="addon_moderations_derived")
|
||||||
|
parser.add_argument("--cinder_bearer_token", required=True)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
@ -163,7 +161,7 @@ def main():
|
||||||
table_name = "cinder_decisions_raw_v1"
|
table_name = "cinder_decisions_raw_v1"
|
||||||
|
|
||||||
date = args.date
|
date = args.date
|
||||||
bearer_token = CINDER_BEARER_TOKEN
|
bearer_token = args.cinder_bearer_token
|
||||||
|
|
||||||
data = []
|
data = []
|
||||||
|
|
||||||
|
|
Загрузка…
Ссылка в новой задаче