Bug 1333079 - Switch to new style Celery setting names

Converted using:

celery upgrade settings treeherder/config/settings.py --django
celery upgrade settings tests/settings.py --django

As suggested by:
http://celery.readthedocs.io/en/latest/whatsnew-4.0.html#lowercase-setting-names

(I manually reverted the env variable name back to simplify deployment,
we can always clean these up later.)

The Django integration in celery.py has been cleaned up by following:
https://celery.readthedocs.io/en/latest/django/first-steps-with-django.html

docs/dataload.rst

@@ -2,12 +2,12 @@ Loading buildbot data
 =====================
 
 In order to start ingesting data, you need to turn on a celery worker with a '-B' option.
-In this way the worker can run some scheduled tasks that are defined in treeherder.config.settings.CELERYBEAT_SCHEDULE.
+In this way the worker can run some scheduled tasks that are defined in treeherder.config.settings.CELERY_BEAT_SCHEDULE.
 Here is a brief description of what each periodic task will do for you:
 
 *fetch-push-logs*
   Retrieves and store all the latest pushes (a.k.a. resultsets) from the available repositories.
-  You need to have this running before you can start ingestiong job data. No pushes, no jobs.
+  You need to have this running before you can start ingesting job data. No pushes, no jobs.
 
 *fetch-buildapi-pending*
   Retrieves and store buildbot pending jobs using `RelEng buildapi`_ service

lints/queuelint.py

@@ -20,7 +20,7 @@ settings_queues = set()
 
 queues_list = None
 for item in code.body:
-    if isinstance(item, ast.Assign) and item.targets[0].id == "CELERY_QUEUES":
+    if isinstance(item, ast.Assign) and item.targets[0].id == "CELERY_TASK_QUEUES":
         queues_list = item.value
 
 if queues_list is None:

runtests.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-echo "Checking CELERY_QUEUES matches Procfile"
+echo "Checking CELERY_TASK_QUEUES matches Procfile"
 ./lints/queuelint.py || { exit 1; }
 
 echo "Running flake8"

tests/conftest.py

@@ -156,7 +156,7 @@ def result_set_stored(test_repository, sample_resultset):
 @pytest.fixture
 def mock_message_broker(monkeypatch):
     from django.conf import settings
-    monkeypatch.setattr(settings, 'BROKER_URL', 'memory://')
+    monkeypatch.setattr(settings, 'CELERY_BROKER_URL', 'memory://')
 
 
 @pytest.fixture

tests/settings.py

@@ -5,11 +5,11 @@ DATABASES["default"]["TEST"] = {"NAME": "test_treeherder"}
 TREEHERDER_TEST_REPOSITORY_NAME = 'test_treeherder_jobs'
 
 # this makes celery calls synchronous, useful for unit testing
-CELERY_ALWAYS_EAGER = True
-CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
+CELERY_TASK_ALWAYS_EAGER = True
+CELERY_TASK_EAGER_PROPAGATES = True
 
 # Reconfigure pulse to operate on default vhost of rabbitmq
-PULSE_URI = BROKER_URL
+PULSE_URI = CELERY_BROKER_URL
 PULSE_EXCHANGE_NAMESPACE = 'test'
 
 # Set a fake api key for testing bug filing

treeherder/celery.py

@@ -3,14 +3,17 @@ from __future__ import absolute_import
 import os
 
 from celery import Celery
-from django.conf import settings
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'treeherder.config.settings')
 
 app = Celery('treeherder')
 
-# Using a string here means the worker will not have to
-# pickle the object when using Windows.
-app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()

treeherder/config/settings.py

@@ -223,7 +223,7 @@ if DEBUG:
         }
     }
 
-CELERY_QUEUES = [
+CELERY_TASK_QUEUES = [
     Queue('default', Exchange('default'), routing_key='default'),
     # queue for failed jobs/logs
     Queue('log_parser', Exchange('default'), routing_key='log_parser.normal'),
@@ -259,11 +259,11 @@ CELERY_TASK_SERIALIZER = 'json'
 CELERY_RESULT_SERIALIZER = 'json'
 
 # default value when no task routing info is specified
-CELERY_DEFAULT_QUEUE = 'default'
-CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
-CELERY_DEFAULT_ROUTING_KEY = 'default'
+CELERY_TASK_DEFAULT_QUEUE = 'default'
+CELERY_TASK_DEFAULT_EXCHANGE_TYPE = 'direct'
+CELERY_TASK_DEFAULT_ROUTING_KEY = 'default'
 
-CELERYBEAT_SCHEDULE = {
+CELERY_BEAT_SCHEDULE = {
     'fetch-push-logs-every-minute': {
         'task': 'fetch-push-logs',
         'schedule': timedelta(minutes=1),
@@ -578,13 +578,13 @@ CACHES = {
 KEY_PREFIX = TREEHERDER_MEMCACHED_KEY_PREFIX
 
 # Celery broker setup
-BROKER_URL = env('BROKER_URL')
+CELERY_BROKER_URL = env('BROKER_URL')
 
 # Force Celery to use TLS when appropriate (ie if not localhost),
-# rather than relying on `BROKER_URL` having `amqps://` or `?ssl=` set.
+# rather than relying on the broker URL string using `amqps://` or `?ssl=`.
 # This is required since CloudAMQP's automatically defined URL uses neither.
-if server_supports_tls(BROKER_URL):
-    BROKER_USE_SSL = True
+if server_supports_tls(CELERY_BROKER_URL):
+    CELERY_BROKER_USE_SSL = True
 
 # This code handles the memcachier service on heroku.
 # TODO: Stop special-casing Heroku and use newer best practices from:
@@ -602,7 +602,7 @@ if env.bool('IS_HEROKU', default=False):
         },
     })
 
-CELERY_IGNORE_RESULT = True
+CELERY_TASK_IGNORE_RESULT = True
 
 SWAGGER_SETTINGS = {
     'SECURITY_DEFINITIONS': {},

treeherder/etl/management/commands/ingest_push.py

@@ -63,7 +63,7 @@ class Command(BaseCommand):
             fetch_push_id = None
 
         # make sure all tasks are run synchronously / immediately
-        settings.CELERY_ALWAYS_EAGER = True
+        settings.CELERY_TASK_ALWAYS_EAGER = True
 
         # get hg pushlog
         pushlog_url = '%s/json-pushes/?full=1&version=2' % repo.url