if --parameter passed we set use_legacy_sql option to False by default and bq called with query by default if not explicitly passed in the bqetl query run command (#3922)

This commit is contained in:
kik-kik 2023-06-13 10:53:03 +02:00 коммит произвёл GitHub
Родитель d68cace7b0
Коммит b2a06b8779
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
2 изменённых файлов: 11 добавлений и 0 удалений

Просмотреть файл

@ -863,6 +863,15 @@ def _run_query(
# point to a public table it needs to be passed as parameter for the query
query_arguments.append("--destination_table={}".format(destination_table))
if bool(list(filter(lambda x: x.startswith("--parameter"), query_arguments))):
# need to do this as parameters are not supported with legacy sql
query_arguments.append("--use_legacy_sql=False")
# this assumed query command should always be passed inside query_arguments
# also dedup the query_arguments in case the same option is passed multiple times.
if "query" not in query_arguments:
query_arguments = ["query"] + list(set(query_arguments))
# write rendered query to a temporary file;
# query string cannot be passed directly to bq as SQL comments will be interpreted as CLI arguments
with tempfile.NamedTemporaryFile(mode="w+") as query_stream:

Просмотреть файл

@ -36,6 +36,7 @@ class TestRunQuery:
assert mock_call.call_args.args == (
[
"bq",
"query",
"--dataset_id=test",
"--destination_table=query_v1",
],
@ -71,6 +72,7 @@ class TestRunQuery:
assert mock_call.call_args.args == (
[
"bq",
"query",
"--dataset_id=test",
"--destination_table=mozilla-public-data:test.query_v1",
],