Merge pull request #77 from mistercrunch/celery_param

Making concurency a conf param, stting CELERYD_PREFETCH_MULTIPLIER=1

airflow/airflow.cfg.template

@@ -23,6 +23,7 @@ SMTP_MAIL_FROM: 'airflow_alerts@mydomain.com'
 CELERY_APP_NAME: airflow.executors.celery_executor
 BROKER_URL = sqla+mysql://airflow:airflow@localhost:3306/airflow
 CELERY_RESULT_BACKEND = db+mysql://airflow:airflow@localhost:3306/airflow
+CELERYD_CONCURRENCY = 16
 WORKER_LOG_SERVER_PORT = 8793
 
 [hooks]

airflow/executors/celery_executor.py

@@ -20,6 +20,8 @@ class CeleryConfig(object):
     BROKER_URL = conf.get('celery', 'BROKER_URL')
     CELERY_RESULT_BACKEND = conf.get('celery', 'CELERY_RESULT_BACKEND')
     CELERY_ACCEPT_CONTENT = ['json', 'pickle']
+    CELERYD_PREFETCH_MULTIPLIER = 1
+    CELERYD_CONCURRENCY = int(conf.get('celery', 'CELERYD_CONCURRENCY'))
 
 app = Celery(
     conf.get('celery', 'CELERY_APP_NAME'),