зеркало из https://github.com/mozilla/docker-etl.git
few PR changes
This commit is contained in:
Родитель
a4f50f49b3
Коммит
62c1c3e124
|
@ -2,3 +2,4 @@
|
|||
venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.idea/
|
|
@ -1,5 +1,5 @@
|
|||
FROM python:3.7
|
||||
MAINTAINER REPLACE ME <replaceme@mozilla.com>
|
||||
MAINTAINER Perry McManis <pmcmanis@mozilla.com>
|
||||
|
||||
# https://github.com/mozilla-services/Dockerflow/blob/master/docs/building-container.md
|
||||
ARG USER_ID="10001"
|
||||
|
@ -14,7 +14,6 @@ WORKDIR ${HOME}
|
|||
|
||||
RUN apt install gcc
|
||||
RUN apt install g++
|
||||
RUN apt install gcloud
|
||||
|
||||
RUN pip install --upgrade pip
|
||||
|
||||
|
|
|
@ -1,36 +0,0 @@
|
|||
"""
|
||||
See [etl-graph in the docker-etl repository]
|
||||
(https://github.com/mozilla/docker-etl/blob/main/jobs/etl-graph).
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from airflow import DAG
|
||||
from operators.gcp_container_operator import GKEPodOperator
|
||||
from utils.tags import Tag
|
||||
|
||||
default_args = {
|
||||
"owner": "pmcmanis@mozilla.com",
|
||||
"email": ["pmcmanis@mozilla.com"],
|
||||
"depends_on_past": False,
|
||||
"start_date": datetime(2021, 2, 1),
|
||||
"email_on_failure": True,
|
||||
"email_on_retry": True,
|
||||
"retries": 2,
|
||||
"retry_delay": timedelta(minutes=30),
|
||||
}
|
||||
|
||||
tags = [Tag.ImpactTier.tier_3]
|
||||
|
||||
with DAG("etl_graph", default_args=default_args, schedule_interval="0 2 * * sat", doc_md=__doc__, tags=tags,) as dag:
|
||||
dataset_yamls = ["yaml/mobile.yaml", "yaml/desktop.yaml"]
|
||||
|
||||
experiment_enrollment_export = gke_command(
|
||||
task_id="experiment_enrollment_export",
|
||||
command=[
|
||||
"python", "kpi-forecasting/kpi-forecasting.py",
|
||||
"-c",
|
||||
] + dataset_yamls,
|
||||
docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/kpi-forecasting_docker_etl:latest",
|
||||
dag=dag,
|
||||
)
|
Загрузка…
Ссылка в новой задаче