Upgrade to Celery 4.4

2020-02-10 09:46:48 -07:00 · 2020-02-10 09:46:48 -07:00 · eb274de12b
--- a/.env-build
+++ b/.env-build
@ -1,9 +1,9 @@
 ALLOWED_HOSTS=*
-BROKER_URL=redis://redis:6437/4
+CELERY_BROKER_URL=redis://redis:6437/4
 CACHE_URL=dummy://
 REDIS_DEFAULT_URL=redis://redis:6437/1
 REDIS_HELPFULVOTES_URL=redis://redis:6437/2
-CELERY_ALWAYS_EAGER=True
+CELERY_TASK_ALWAYS_EAGER=True
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=sqlite://
 DATABASE_READ_ONLY_URL=sqlite://
--- a/.env-dist
+++ b/.env-dist
@ -1,9 +1,9 @@
 ALLOWED_HOSTS=*
-BROKER_URL=redis://redis:6379/4
+CELERY_BROKER_URL=redis://redis:6379/4
 REDIS_DEFAULT_URL=redis://redis:6379/1
 REDIS_HELPFULVOTES_URL=redis://redis:6379/2
 CACHE_URL=redis://redis:6379/3
-CELERY_ALWAYS_EAGER=True
+CELERY_TASK_ALWAYS_EAGER=True
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=mysql://root:kitsune@mariadb:3306/kitsune
 DATABASE_READ_ONLY_URL=mysql://root:kitsune@mariadb:3306/kitsune
--- a/.env-test
+++ b/.env-test
@ -2,12 +2,12 @@ DEBUG=False
 # Some cron jobs are skipped on stage.
 STAGE=False
 SESSION_COOKIE_SECURE=False
-CELERY_ALWAYS_EAGER=True
+CELERY_TASK_ALWAYS_EAGER=True
 # Make sure pipeline is enabled so it does not collectstatic on every test
 PIPELINE_ENABLED=True
 ES_LIVE_INDEXING=False
 ALLOWED_HOSTS=*
-BROKER_URL=redis://redis:6379/4
+CELERY_BROKER_URL=redis://redis:6379/4
 REDIS_DEFAULT_URL=redis://redis:6379/1
 REDIS_HELPFULVOTES_URL=redis://redis:6379/2
 CACHE_URL=redis://redis:6379/3
--- a/bin/run-celery-worker.sh
+++ b/bin/run-celery-worker.sh
@ -1,3 +1,3 @@
 #!/bin/bash

-exec newrelic-admin run-program celery -A kitsune worker --maxtasksperchild=${CELERY_MAX_TASKS_PER_CHILD:-25}
+exec newrelic-admin run-program celery -A kitsune worker --maxtasksperchild=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-25}
--- a/docs/celery.rst
+++ b/docs/celery.rst
@ -41,29 +41,28 @@ Configuring and Running

 We set some reasonable defaults for Celery in ``settings.py``. These can be
 overriden either in ``settings_local.py`` or via the command line when running
-``celery -A kitsune``.
+``celery -A kitsune worker``.

 In ``settings_local.py`` you should set at least this, if you want to use
 Celery::

-    CELERY_ALWAYS_EAGER = False
+    CELERY_TASK_ALWAYS_EAGER = False

 This defaults to ``True``, which causes all task processing to be done online.
 This lets you run Kitsune even if you don't have Redis or want to deal with
 running workers all the time.

-You can also configure the log level or concurrency. Here are the defaults::
+You can also configure the concurrency. Here is the default::

-    CELERYD_LOG_LEVEL = logging.INFO
-    CELERYD_CONCURRENCY = 4
+    CELERY_WORKER_CONCURRENCY = 4

 Then to start the Celery workers, you just need to run::

-    celery -A kitsune
+    celery -A kitsune worker

 This will start Celery with the default number of worker threads and the
 default logging level. You can change those with::

-    celery -A kitsune --log-level=DEBUG -c 10
+    celery -A kitsune worker --loglevel=DEBUG --concurrency=10

 This would start Celery with 10 worker threads and a log level of ``DEBUG``.
--- a/kitsune/celery.py
+++ b/kitsune/celery.py
@ -6,5 +6,5 @@ from celery import Celery  # noqa
 from django.conf import settings  # noqa

 app = Celery('kitsune')
-app.config_from_object('django.conf:settings')
+app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks(settings.INSTALLED_APPS)
--- a/kitsune/questions/tests/test_notifications.py
+++ b/kitsune/questions/tests/test_notifications.py
@ -24,7 +24,7 @@ from kitsune.users.tests import UserFactory
 # expect a locale in reverse()d URLs. When firing off a celery task outside the
 # scope of a request, expect none.
 #
-# In production, with CELERY_ALWAYS_EAGER=False, celery tasks run in a
+# In production, with CELERY_TASK_ALWAYS_EAGER=False, celery tasks run in a
 # different interpreter (with no access to the thread-local), so reverse() will
 # never prepend a locale code unless passed force_locale=True. Thus, these
 # test-emails with locale prefixes are not identical to the ones sent in
--- a/kitsune/settings.py
+++ b/kitsune/settings.py
@ -904,20 +904,26 @@ if EMAIL_LOGGING_REAL_BACKEND == 'django.core.mail.backends.smtp.EmailBackend':


 # Celery
-CELERY_IGNORE_RESULT = config('CELERY_IGNORE_RESULT', default=True, cast=bool)
-if not CELERY_IGNORE_RESULT:
+# TODO: Upgrade to task_protocol 2.
+CELERY_TASK_PROTOCOL = 1
+CELERY_TASK_SERIALIZER = config('CELERY_TASK_SERIALIZER', default='pickle')
+CELERY_RESULT_SERIALIZER = config('CELERY_RESULT_SERIALIZER', default='pickle')
+CELERY_ACCEPT_CONTENT = config('CELERY_ACCEPT_CONTENT', default='pickle',
+                               cast=lambda v: [s.strip() for s in v.split(',')])
+CELERY_TASK_IGNORE_RESULT = config('CELERY_TASK_IGNORE_RESULT', default=True, cast=bool)
+if not CELERY_TASK_IGNORE_RESULT:
    # E.g. redis://localhost:6479/1
    CELERY_RESULT_BACKEND = config('CELERY_RESULT_BACKEND')

-CELERY_ALWAYS_EAGER = config('CELERY_ALWAYS_EAGER', default=DEBUG, cast=bool)  # For tests. Set to False for use.
-if not CELERY_ALWAYS_EAGER:
-    BROKER_URL = config('BROKER_URL')
+CELERY_TASK_ALWAYS_EAGER = config('CELERY_TASK_ALWAYS_EAGER', default=DEBUG, cast=bool)  # For tests. Set to False for use.
+if not CELERY_TASK_ALWAYS_EAGER:
+    CELERY_BROKER_URL = config('CELERY_BROKER_URL')

-CELERY_SEND_TASK_ERROR_EMAILS = config('CELERY_SEND_TASK_ERROR_EMAILS', default=True, cast=bool)
-CELERYD_LOG_LEVEL = config('CELERYD_LOG_LEVEL', default='INFO', cast=lambda x: getattr(logging, x))
-CELERYD_CONCURRENCY = config('CELERYD_CONCURRENCY', default=4, cast=int)
-CELERY_EAGER_PROPAGATES_EXCEPTIONS = config('CELERY_EAGER_PROPAGATES_EXCEPTIONS', default=True, cast=bool)  # Explode loudly during tests.
-CELERYD_HIJACK_ROOT_LOGGER = config('CELERYD_HIJACK_ROOT_LOGGER', default=False, cast=bool)
+# TODO:PY3: Setting gone, use celery worker --loglevel flag.
+# CELERYD_LOG_LEVEL = config('CELERYD_LOG_LEVEL', default='INFO', cast=lambda x: getattr(logging, x))
+CELERY_WORKER_CONCURRENCY = config('CELERY_WORKER_CONCURRENCY', default=4, cast=int)
+CELERY_TASK_EAGER_PROPAGATES = config('CELERY_TASK_EAGER_PROPAGATES', default=True, cast=bool)  # Explode loudly during tests.
+CELERY_WORKER_HIJACK_ROOT_LOGGER = config('CELERY_WORKER_HIJACK_ROOT_LOGGER', default=False, cast=bool)

 # Wiki rebuild settings
 WIKI_REBUILD_TOKEN = 'sumo:wiki:full-rebuild'
--- a/kitsune/wiki/tasks.py
+++ b/kitsune/wiki/tasks.py
@ -141,7 +141,7 @@ def send_contributor_notification(based_on, revision, document, message):
 def schedule_rebuild_kb():
    """Try to schedule a KB rebuild, if we're allowed to."""
    if (not waffle.switch_is_active('wiki-rebuild-on-demand') or
-            settings.CELERY_ALWAYS_EAGER):
+            settings.CELERY_TASK_ALWAYS_EAGER):
        return

    if cache.get(settings.WIKI_REBUILD_TOKEN):
--- a/kitsune/wiki/tests/test_tasks.py
+++ b/kitsune/wiki/tests/test_tasks.py
@ -39,10 +39,10 @@ https://testserver/en-US/kb/%s/history
 """


-@override_settings(CELERY_ALWAYS_EAGER=True)
+@override_settings(CELERY_TASK_ALWAYS_EAGER=True)
 class RebuildTestCase(TestCase):
    rf = RequestFactory()
-    ALWAYS_EAGER = settings.CELERY_ALWAYS_EAGER
+    ALWAYS_EAGER = settings.CELERY_TASK_ALWAYS_EAGER

    def setUp(self):
        # create some random revisions.
@ -66,7 +66,7 @@ class RebuildTestCase(TestCase):

    @mock.patch.object(rebuild_kb, 'delay')
    @mock.patch.object(waffle, 'switch_is_active')
-    @override_settings(CELERY_ALWAYS_EAGER=False)
+    @override_settings(CELERY_TASK_ALWAYS_EAGER=False)
    def test_task_queue(self, switch_is_active, delay):
        switch_is_active.return_value = True
        schedule_rebuild_kb()
--- a/requirements/default.in
+++ b/requirements/default.in
@ -5,7 +5,7 @@ Werkzeug~=0.16.1
 babis~=0.2.1
 bleach~=2.1.4
 boto3>=1.2.3  # pip-compile does not handle extras properly (django-storages)
-celery~=3.1.25
+celery~=4.4.0
 commonware~=0.5.0
 cryptography>=1.3.4  # pip-compile does not handle extras properly (requests)
 dennis~=0.7
@ -14,7 +14,7 @@ django-activity-stream[jsonfield]~=0.8.0
 django-adminplus~=0.5
 django-allow-cidr~=0.3.1
 django-authority~=0.14
-django-cache-url~=2.0.0
+django-cache-url~=3.1.1
 django-cors-headers~=3.2.1
 django-enforce-host~=1.0.1
 django-extensions~=2.2.6
@ -63,7 +63,7 @@ pyquery~=1.2.9
 python-decouple~=3.0
 python-memcached~=1.59
 pytz~=2019.3
-redis~=3.3.11
+redis~=3.4.1
 requests[security]~=2.22.0
 sentry-sdk~=0.7.6
 simplejson~=3.7.3
--- a/requirements/default.txt
+++ b/requirements/default.txt
@ -1,16 +1,13 @@
-# SHA1:759cbdc9f7236c41a5b295bf04d09c087f2c0a2e
+# SHA1:ec2dee8fa95712b7d9ccd39d683fb63edbd8e302
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
 #
 #    pip-compile-multi
 #
-amqp==1.4.9 \
-    --hash=sha256:2dea4d16d073c902c3b89d9b96620fb6729ac0f7a923bbc777cb4ad827c0c61a \
-    --hash=sha256:e0ed0ce6b8ffe5690a2e856c7908dc557e0e605283d6885dd1361d79f2928908 \
-    # via kombu
-anyjson==0.3.3 \
-    --hash=sha256:37812d863c9ad3e35c0734c42e0bf0320ce8c3bed82cd20ad54cb34d158157ba \
+amqp==2.5.2 \
+    --hash=sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8 \
+    --hash=sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d \
    # via kombu
 apscheduler==3.5.3 \
    --hash=sha256:6599bc78901ee7e9be85cbd073d9cc155c42d2bc867c5cde4d4d1cc339ebfbeb \
@ -22,15 +19,9 @@ babel==2.8.0 \
 babis==0.2.2 \
    --hash=sha256:31095db6b412062b1dcdad9d94ff4c207299fb90135a950e430567863b20f071 \
    --hash=sha256:c5cba6b0170ed02020d2cdd999da9d9fa4f4d4e725833e3ed98dad1b898d311b
-billiard==3.3.0.23 \
-    --hash=sha256:204e75d390ef8f839c30a93b696bd842c3941916e15921745d05edc2a83868ab \
-    --hash=sha256:23cb71472712e96bff3e0d45763b7b8a99e5040385fffb96816028352c255682 \
-    --hash=sha256:692a2a5a55ee39a42bcb7557930e2541da85df9ea81c6e24827f63b80cd39d0b \
-    --hash=sha256:82041dbaa62f7fde1464d7ab449978618a38b241b40c0d31dafabb36446635dc \
-    --hash=sha256:958fc9f8fd5cc9b936b2cb9d96f02aa5ec3613ba13ee7f089c77ff0bcc368fac \
-    --hash=sha256:c0cbe8d45ba8d8213ad68ef9a1881002a151569c9424d551634195a18c3a4160 \
-    --hash=sha256:ccfe0419eb5e49f27ad35cf06e75360af903df6d576c66cb8073246d4e023e5c \
-    --hash=sha256:d4d2fed1a251ea58eed47b48db3778ebb92f5ff4407dc91869c6f41c3a9249d0 \
+billiard==3.6.2.0 \
+    --hash=sha256:26fd494dc3251f8ce1f5559744f18aeed427fdaf29a75d7baae26752a5d3816f \
+    --hash=sha256:f4e09366653aa3cb3ae8ed16423f9ba1665ff426f087bcdbbed86bf3664fe02c \
    # via celery
 bleach==2.1.4 \
    --hash=sha256:0ee95f6167129859c5dce9b1ca291ebdb5d8cd7e382ca0e237dfd0dad63f63d8 \
@ -46,9 +37,9 @@ cachetools==4.0.0 \
    --hash=sha256:9a52dd97a85f257f4e4127f15818e71a0c7899f121b34591fcc1173ea79a0198 \
    --hash=sha256:b304586d357c43221856be51d73387f93e2a961598a9b6b6670664746f3b6c6c \
    # via google-auth, premailer
-celery==3.1.26.post2 \
-    --hash=sha256:5493e172ae817b81ba7d09443ada114886765a8ce02f16a56e6fac68d953a9b2 \
-    --hash=sha256:60211897aee321266ff043fe2b33eaac825dfe9f46843cf964fc97507a186334
+celery==4.4.0 \
+    --hash=sha256:7c544f37a84a5eadc44cab1aa8c9580dff94636bb81978cdf9bf8012d9ea7d8f \
+    --hash=sha256:d3363bb5df72d74420986a435449f3c3979285941dff57d5d97ecba352a0e3e2
 certifi==2019.11.28 \
    --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \
    --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f \
@ -145,9 +136,9 @@ django-appconf==1.0.3 \
 django-authority==0.14 \
    --hash=sha256:20642871d146e9f989d764db28bd6ea441157ea152c817338db574632df7cd67 \
    --hash=sha256:f2e0b2adfa98cc5f0b4766f35d2666c7d040452d91e93fc47401e85f85bbecbd
-django-cache-url==2.0.0 \
-    --hash=sha256:d7e7d3f3fadaa38ab7c655afafc85c2a1b06a3ddc0002af4ae021c1948307f8b \
-    --hash=sha256:f0009452c4d63468714c7ff59266085947f3f3cc0947b3b2e87121bb27538d6c
+django-cache-url==3.1.1 \
+    --hash=sha256:04cac7f452ecbf4a88ce59fec8daaaa96fb5a81a0f2f6cbfa2fa9ce2de261c17 \
+    --hash=sha256:e23d22f62f27ce0384179f837251ec93a321b29ef825c1f35edbf74b5e990c69
 django-cors-headers==3.2.1 \
    --hash=sha256:a5960addecc04527ab26617e51b8ed42f0adab4594b24bb0f3c33e2bd3857c3f \
    --hash=sha256:a785b5f446f6635810776d9f5f5d23e6a2a2f728ea982648370afaf0dfdf2627
@ -282,6 +273,10 @@ httplib2==0.17.0 \
 idna==2.8 \
    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
+importlib-metadata==1.5.0 \
+    --hash=sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302 \
+    --hash=sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b \
+    # via kombu
 jinja2==2.11.1 \
    --hash=sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250 \
    --hash=sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49 \
@ -294,9 +289,9 @@ josepy==1.3.0 \
    --hash=sha256:c341ffa403399b18e9eae9012f804843045764d1390f9cb4648980a7569b1619 \
    --hash=sha256:e54882c64be12a2a76533f73d33cba9e331950fda9e2731e843490b774e7a01c \
    # via mozilla-django-oidc
-kombu==3.0.37 \
-    --hash=sha256:7ceab743e3e974f3e5736082e8cc514c009e254e646d6167342e0e192aee81a6 \
-    --hash=sha256:e064a00c66b4d1058cd2b0523fb8d98c82c18450244177b6c0f7913016642650 \
+kombu==4.6.7 \
+    --hash=sha256:2a9e7adff14d046c9996752b2c48b6d9185d0b992106d5160e1a179907a5d4ac \
+    --hash=sha256:67b32ccb6fea030f8799f8fd50dd08e03a4b99464ebc4952d71d8747b1a52ad1 \
    # via celery
 lxml==4.4.3 \
    --hash=sha256:0126dc8fe8c154f2cc7ec0f8f65beff3ae948b8f2a273c0933e92eb29011d595 \
@ -454,9 +449,9 @@ python-memcached==1.59 \
 pytz==2019.3 \
    --hash=sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d \
    --hash=sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be
-redis==3.3.11 \
-    --hash=sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62 \
-    --hash=sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2
+redis==3.4.1 \
+    --hash=sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f \
+    --hash=sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833
 requests-oauthlib==1.3.0 \
    --hash=sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d \
    --hash=sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a \
@ -509,6 +504,10 @@ urllib3==1.25.8 \
    --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \
    --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \
    # via botocore, elasticsearch, requests, sentry-sdk
+vine==1.3.0 \
+    --hash=sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87 \
+    --hash=sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af \
+    # via amqp, celery
 webencodings==0.5.1 \
    --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
    --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \
@ -521,6 +520,10 @@ whitenoise==3.3.1 \
    --hash=sha256:9d81515f2b5b27051910996e1e860b1332e354d9e7bcf30c98f21dcb6713e0dd
 zdesk==2.7.1 \
    --hash=sha256:7a9c11bee43973bcd9680d727d3c0359d35b434f5a0ff4514bbc5bd03e1afbe9
+zipp==2.2.0 \
+    --hash=sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50 \
+    --hash=sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e \
+    # via importlib-metadata

 # WARNING: The following packages were not pinned, but pip requires them to be
 # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
--- a/requirements/test.txt
+++ b/requirements/test.txt
@ -27,10 +27,6 @@ execnet==1.7.1 \
 flake8==3.7.9 \
    --hash=sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb \
    --hash=sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca
-importlib-metadata==1.5.0 \
-    --hash=sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302 \
-    --hash=sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b \
-    # via pluggy, pytest
 mccabe==0.6.1 \
    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
@ -106,10 +102,6 @@ wcwidth==0.1.8 \
    --hash=sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603 \
    --hash=sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8 \
    # via pytest
-zipp==2.2.0 \
-    --hash=sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50 \
-    --hash=sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e \
-    # via importlib-metadata

 # WARNING: The following packages were not pinned, but pip requires them to be
 # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
--- a/scripts/travis/setup.sh
+++ b/scripts/travis/setup.sh
@ -16,7 +16,7 @@ DATABASES['default']['NAME'] = 'kitsune'
 DATABASES['default']['HOST'] = 'localhost'
 DATABASES['default']['USER'] = 'travis'
 DATABASES['default']['CONN_MAX_AGE'] = 600
-CELERY_ALWAYS_EAGER = True
+CELERY_TASK_ALWAYS_EAGER = True
 ES_INDEX_PREFIX = 'sumo'
 ES_URLS = ['http://localhost:9200']
 SETTINGS