From 18f4ef681a428c99b0aebf5bb849b39c9c9b322f Mon Sep 17 00:00:00 2001 From: Christopher Grebs Date: Mon, 1 Feb 2016 20:03:51 +0100 Subject: [PATCH] Update initialization of celery. * Update module path in supervisor configuration * Move celery configuration entirely to olympia.amo.celery * Fix paths in reindex management command --- docker/supervisor-celery.conf | 4 ++- src/olympia/amo/celery.py | 15 +++++++++- .../lib/es/management/commands/reindex.py | 10 ++++--- src/olympia/startup.py | 30 ------------------- 4 files changed, 23 insertions(+), 36 deletions(-) diff --git a/docker/supervisor-celery.conf b/docker/supervisor-celery.conf index 65cba49c67..41a63a7b33 100644 --- a/docker/supervisor-celery.conf +++ b/docker/supervisor-celery.conf @@ -8,7 +8,7 @@ logfile=/code/logs/supervisord-celery.log # Changes: # - Set concurrency (number of workers) back to the default # - The code auto-reloads for development -command=celery -A olympia worker --autoreload -E --loglevel=INFO +command=celery -A olympia.amo.celery:app worker --autoreload -E --loglevel=INFO directory=/code stopasgroup=true autostart=true @@ -16,6 +16,8 @@ redirect_stderr=true stdout_logfile=logs/docker-celery.log stdout_logfile_maxbytes=1MB stopsignal=KILL +environment=DJANGO_SETTINGS_MODULE='settings' + # The following sections enable supervisorctl. diff --git a/src/olympia/amo/celery.py b/src/olympia/amo/celery.py index bb0ad7ab33..00c9645a82 100644 --- a/src/olympia/amo/celery.py +++ b/src/olympia/amo/celery.py @@ -14,6 +14,8 @@ from celery.signals import task_failure, task_postrun, task_prerun from celery.task.sets import TaskSet from django_statsd.clients import statsd from post_request_task.task import PostRequestTask +from raven import Client +from raven.contrib.celery import register_signal, register_logger_signal from olympia.amo.utils import chunked, utc_millesecs_from_epoch @@ -27,7 +29,18 @@ task = app.task app.config_from_object('django.conf:settings') app.autodiscover_tasks(settings.INSTALLED_APPS) -# See olympia.startup::init_celery() for more configuration. +# Hook up Sentry in celery. +raven_client = Client(settings.SENTRY_DSN) + +# register a custom filter to filter out duplicate logs +register_logger_signal(raven_client) + +# hook into the Celery error handler +register_signal(raven_client) + +# After upgrading raven we can specify loglevel=logging.INFO to override +# the default (which is ERROR). +register_logger_signal(raven_client) @task_failure.connect diff --git a/src/olympia/lib/es/management/commands/reindex.py b/src/olympia/lib/es/management/commands/reindex.py index 97cf2dd3dc..446cd1211c 100644 --- a/src/olympia/lib/es/management/commands/reindex.py +++ b/src/olympia/lib/es/management/commands/reindex.py @@ -20,7 +20,8 @@ from olympia.lib.es.utils import ( timestamp_index) logger = logging.getLogger('z.elasticsearch') -time_limits = settings.CELERY_TIME_LIMITS['lib.es.management.commands.reindex'] +time_limits = settings.CELERY_TIME_LIMITS[ + 'olympia.lib.es.management.commands.reindex'] ES = get_es() @@ -231,9 +232,10 @@ class Command(BaseCommand): # the soft and hard time limits on the @task decorator. But we're not # using the @task decorator here, but a decorator from celery_tasktree. if not getattr(settings, 'CELERY_ALWAYS_EAGER', False): - control.time_limit('lib.es.management.commands.reindex.index_data', - soft=time_limits['soft'], - hard=time_limits['hard']) + control.time_limit( + 'olympia.lib.es.management.commands.reindex.index_data', + soft=time_limits['soft'], + hard=time_limits['hard']) try: tree.apply_async() diff --git a/src/olympia/startup.py b/src/olympia/startup.py index 571dd1d1bc..a4fa6cc243 100644 --- a/src/olympia/startup.py +++ b/src/olympia/startup.py @@ -71,35 +71,6 @@ def init_amo(): amo = __import__('olympia.amo') -def init_celery(): - """ - Initialize Celery, and make our app instance available as `celery_app` - for use by the `celery` command. - """ - from django.conf import settings - from raven import Client - from raven.contrib.celery import register_signal, register_logger_signal - from olympia.amo import celery - - # I think `manage.py celery` relies on this global? We typically don't run - # celery like that anymore though. - global celery_app - celery_app = celery.app - - # Hook up Sentry in celery. - client = Client(settings.SENTRY_DSN) - - # register a custom filter to filter out duplicate logs - register_logger_signal(client) - - # hook into the Celery error handler - register_signal(client) - - # After upgrading raven we can specify loglevel=logging.INFO to override - # the default (which is ERROR). - register_logger_signal(client) - - def configure_logging(): """Configure the `logging` module to route logging based on settings in our various settings modules and defaults in `lib.log_settings_base`.""" @@ -126,5 +97,4 @@ init_jinja2() init_amo() configure_logging() init_jingo() -init_celery() load_product_details()