mozilla
/
bigquery-etl
зеркало из https://github.com/mozilla/bigquery-etl.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1713631 - Use bigquery_etl image from private GCR repository

This commit is contained in:
Arkadiusz Komarzewski 2021-06-08 14:26:28 +02:00 коммит произвёл akkomar
Родитель 907ef931be
Коммит 8c82b48e87
10 изменённых файлов: 25 добавлений и 25 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
bigquery_etl/query_scheduling/templates/airflow_dag.j2
Просмотреть файл

@ -38,7 +38,7 @@ with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None
'python',
'sql/{{ task.project }}/{{ task.dataset }}/{{ task.table }}_{{ task.version }}/query.py',
] + {{ task.arguments }},
docker_image='mozilla/bigquery-etl:latest',
docker_image='gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest',
owner='{{ task.owner }}',
email={{ task.email | sort }},
)

10
bigquery_etl/query_scheduling/templates/public_data_json_airflow_dag.j2
Просмотреть файл

@ -23,15 +23,15 @@ Built from bigquery-etl repo, [`dags/{{ name }}.py`](https://github.com/mozilla/
"""
default_args = {{
default_args.to_dict() |
default_args = {{
default_args.to_dict() |
format_attr("start_date", "format_date") |
format_attr("end_date", "format_date") |
format_attr("retry_delay", "format_timedelta")
format_attr("retry_delay", "format_timedelta")
}}
with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None -%}, schedule_interval={{ schedule_interval | format_timedelta | format_schedule_interval }}{%+ endif -%}, doc_md = docs) as dag:
docker_image = "mozilla/bigquery-etl:latest"
docker_image = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest"
{% for task in tasks | sort(attribute='task_name') %}
{{ task.task_name }} = GKEPodOperator(
task_id="{{ task.task_name }}",
@ -65,7 +65,7 @@ with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None
mode='reschedule',
pool='DATA_ENG_EXTERNALTASKSENSOR',
)
{{ task.task_name }}.set_upstream(wait_for_{{ dependency.task_id }})
{% endif -%}
{% endfor -%}

2
dags/bqetl_firefox_ios.py
Просмотреть файл

@ -45,7 +45,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/unified_metrics_v1/query.py",
]
+ [],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="amiyaguchi@mozilla.com",
email=["amiyaguchi@mozilla.com", "telemetry-alerts@mozilla.com"],
)

14
dags/bqetl_monitoring.py
Просмотреть файл

@ -47,7 +47,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/monitoring_derived/average_ping_sizes_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="ascholtz@mozilla.com",
email=["ascholtz@mozilla.com"],
)
@ -59,7 +59,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_scheduled_queries_cost_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="ascholtz@mozilla.com",
email=["ascholtz@mozilla.com"],
)
@ -71,7 +71,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_scheduled_query_usage_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="ascholtz@mozilla.com",
email=["ascholtz@mozilla.com"],
)
@ -83,7 +83,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/monitoring_derived/column_size_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="ascholtz@mozilla.com",
email=["ascholtz@mozilla.com"],
)
@ -107,7 +107,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/monitoring_derived/stable_table_sizes_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="ascholtz@mozilla.com",
email=["ascholtz@mozilla.com"],
)
@ -119,7 +119,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/monitoring_derived/structured_distinct_docids_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="bewu@mozilla.com",
email=["ascholtz@mozilla.com", "bewu@mozilla.com"],
)
@ -131,7 +131,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/monitoring_derived/structured_missing_columns_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="amiyaguchi@mozilla.com",
email=["amiyaguchi@mozilla.com", "ascholtz@mozilla.com"],
)

10
dags/bqetl_mozilla_vpn.py
Просмотреть файл

@ -171,7 +171,7 @@ with DAG(
"--destination_table",
"moz-fx-data-shared-prod.mozilla_vpn_derived.survey_cancellation_of_service_v1",
],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="amiyaguchi@mozilla.com",
email=[
"amiyaguchi@mozilla.com",
@ -198,7 +198,7 @@ with DAG(
"--destination_table",
"moz-fx-data-shared-prod.mozilla_vpn_derived.survey_intercept_q3_v1",
],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="amiyaguchi@mozilla.com",
email=[
"amiyaguchi@mozilla.com",
@ -225,7 +225,7 @@ with DAG(
"--destination_table",
"moz-fx-data-shared-prod.mozilla_vpn_derived.survey_market_fit_v1",
],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="amiyaguchi@mozilla.com",
email=[
"amiyaguchi@mozilla.com",
@ -252,7 +252,7 @@ with DAG(
"--destination_table",
"moz-fx-data-shared-prod.mozilla_vpn_derived.survey_product_quality_v1",
],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="amiyaguchi@mozilla.com",
email=[
"amiyaguchi@mozilla.com",
@ -279,7 +279,7 @@ with DAG(
"--destination_table",
"moz-fx-data-shared-prod.mozilla_vpn_derived.survey_recommend_v1",
],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="amiyaguchi@mozilla.com",
email=[
"amiyaguchi@mozilla.com",

2
dags/bqetl_pocket.py
Просмотреть файл

@ -62,7 +62,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/pocket_derived/rolling_monthly_active_user_counts_history_v1/query.py",
]
+ ["--date", "{{ ds }}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="jklukas@mozilla.com",
email=["jklukas@mozilla.com", "telemetry-alerts@mozilla.com"],
)

2
dags/bqetl_public_data_json.py
Просмотреть файл

@ -42,7 +42,7 @@ with DAG(
schedule_interval="0 5 * * *",
doc_md=docs,
) as dag:
docker_image = "mozilla/bigquery-etl:latest"
docker_image = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest"
export_public_data_json_mozregression_aggregates__v1 = GKEPodOperator(
task_id="export_public_data_json_mozregression_aggregates__v1",

4
dags/bqetl_releases.py
Просмотреть файл

@ -49,7 +49,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/releases_v1/query.py",
]
+ [],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="ascholtz@mozilla.com",
email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"],
)
@ -61,7 +61,7 @@ with DAG(
"sql/moz-fx-data-shared-prod/telemetry_derived/releases_v1/query.py",
]
+ [],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="ascholtz@mozilla.com",
email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"],
)

2
tests/data/dags/python_script_test_dag
Просмотреть файл

@ -39,7 +39,7 @@ with DAG(
"sql/moz-fx-data-test-project/test/python_script_query_v1/query.py",
]
+ ["--date", "{{ds}}"],
docker_image="mozilla/bigquery-etl:latest",
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
owner="test@example.com",
email=["test@example.com"],
)

2
tests/data/dags/test_public_data_json_dag
Просмотреть файл

@ -35,7 +35,7 @@ with DAG(
schedule_interval="@daily",
doc_md=docs,
) as dag:
docker_image = "mozilla/bigquery-etl:latest"
docker_image = "gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest"
export_public_data_json_test__non_incremental_query__v1 = GKEPodOperator(
task_id="export_public_data_json_test__non_incremental_query__v1",