Limit Fivetran task concurrency so it only tries to run one sync per connector at a time. (#5083)
This commit is contained in:
Родитель
f5ebf44a85
Коммит
54f4a6df70
|
@ -383,6 +383,7 @@ with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None
|
|||
{{ fivetran_task.task_id }}_sync_start = FivetranOperator(
|
||||
connector_id='{% raw %}{{ var.value.{% endraw %}{{ fivetran_task.task_id }}{% raw %}_connector_id }}{% endraw %}',
|
||||
task_id='{{ fivetran_task.task_id }}_task',
|
||||
task_concurrency=1,
|
||||
)
|
||||
|
||||
{% do fivetran_seen.append(fivetran_task) %}
|
||||
|
|
|
@ -63,6 +63,7 @@ with DAG(
|
|||
fivetran_import_1_sync_start = FivetranOperator(
|
||||
connector_id="{{ var.value.fivetran_import_1_connector_id }}",
|
||||
task_id="fivetran_import_1_task",
|
||||
task_concurrency=1,
|
||||
)
|
||||
|
||||
test__non_incremental_query__v1.set_upstream(fivetran_import_1_sync_start)
|
||||
|
@ -70,6 +71,7 @@ with DAG(
|
|||
fivetran_import_2_sync_start = FivetranOperator(
|
||||
connector_id="{{ var.value.fivetran_import_2_connector_id }}",
|
||||
task_id="fivetran_import_2_task",
|
||||
task_concurrency=1,
|
||||
)
|
||||
|
||||
test__non_incremental_query__v1.set_upstream(fivetran_import_2_sync_start)
|
||||
|
|
Загрузка…
Ссылка в новой задаче