Refactoring
This commit is contained in:
Родитель
c5d727c336
Коммит
4d85266015
|
@ -263,9 +263,7 @@ class Task:
|
||||||
|
|
||||||
# the order of table dependencies changes between requests
|
# the order of table dependencies changes between requests
|
||||||
# sort to maintain same order between DAG generation runs
|
# sort to maintain same order between DAG generation runs
|
||||||
sorted_table_names = list(table_names)
|
return sorted(table_names)
|
||||||
sorted_table_names.sort()
|
|
||||||
return sorted_table_names
|
|
||||||
|
|
||||||
def with_dependencies(self, client, dag_collection):
|
def with_dependencies(self, client, dag_collection):
|
||||||
"""Perfom a dry_run to get upstream dependencies."""
|
"""Perfom a dry_run to get upstream dependencies."""
|
||||||
|
|
|
@ -143,7 +143,7 @@ bqetl_main_summary:
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
bqetl_experiments:
|
bqetl_experiments_daily:
|
||||||
schedule_interval: 0 1 * * *
|
schedule_interval: 0 1 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: ssuh@mozilla.com
|
owner: ssuh@mozilla.com
|
||||||
|
|
|
@ -17,7 +17,7 @@ default_args = {
|
||||||
}
|
}
|
||||||
|
|
||||||
with DAG(
|
with DAG(
|
||||||
"bqetl_experiments", default_args=default_args, schedule_interval="0 1 * * *"
|
"bqetl_experiments_daily", default_args=default_args, schedule_interval="0 1 * * *"
|
||||||
) as dag:
|
) as dag:
|
||||||
|
|
||||||
telemetry_derived__experiments_daily_active_clients__v1 = bigquery_etl_query(
|
telemetry_derived__experiments_daily_active_clients__v1 = bigquery_etl_query(
|
|
@ -7,4 +7,4 @@ labels:
|
||||||
application: experiments
|
application: experiments
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_experiments
|
dag_name: bqetl_experiments_daily
|
||||||
|
|
|
@ -415,7 +415,7 @@ class TestTask:
|
||||||
assert f"{temporary_dataset}__table2__v1" in tables
|
assert f"{temporary_dataset}__table2__v1" in tables
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
def test_muultipart_task_get_dependencies(
|
def test_multipart_task_get_dependencies(
|
||||||
self, tmp_path, bigquery_client, project_id, temporary_dataset
|
self, tmp_path, bigquery_client, project_id, temporary_dataset
|
||||||
):
|
):
|
||||||
query_file_path = tmp_path / "sql" / temporary_dataset / "query_v1"
|
query_file_path = tmp_path / "sql" / temporary_dataset / "query_v1"
|
||||||
|
|
Загрузка…
Ссылка в новой задаче