Remove lib/cache.py, simplify CACHES settings (#18706)

* Remove lib/cache.py, simplify CACHES settings

* Remove useless noqa comments

* Useless f prefix

settings.py

@@ -38,7 +38,8 @@ def insert_debug_toolbar_middleware(middlewares):
     for i, middleware in enumerate(ret_middleware):
         if 'GZipMiddleware' in middleware:
             ret_middleware.insert(
-                i + 1, 'debug_toolbar.middleware.DebugToolbarMiddleware')
+                i + 1, 'debug_toolbar.middleware.DebugToolbarMiddleware'
+            )
             break
 
     return tuple(ret_middleware)
@@ -53,15 +54,6 @@ DEBUG_TOOLBAR_CONFIG = {
 
 FILESYSTEM_CACHE_ROOT = os.path.join(TMP_PATH, 'cache')
 
-# We are setting memcached here to make sure our local setup is as close
-# to our production system as possible.
-CACHES = {
-    'default': {
-        'BACKEND': CACHE_BACKEND,
-        'LOCATION': os.environ.get('MEMCACHE_LOCATION', 'localhost:11211'),
-    },
-}
-
 # If you're not running on SSL you'll want this to be False.
 SESSION_COOKIE_SECURE = False
 SESSION_COOKIE_DOMAIN = None
@@ -81,8 +73,7 @@ EXTERNAL_SITE_URL = SITE_URL
 STATIC_URL = '%s/static/' % EXTERNAL_SITE_URL
 MEDIA_URL = '%s/user-media/' % EXTERNAL_SITE_URL
 
-CODE_MANAGER_URL = (
+CODE_MANAGER_URL = os.environ.get('CODE_MANAGER_URL') or 'http://olympia.test:5000'
-    os.environ.get('CODE_MANAGER_URL') or 'http://olympia.test:5000')
 
 ALLOWED_HOSTS = ALLOWED_HOSTS + [SERVICES_DOMAIN, 'nginx']
 
@@ -93,7 +84,8 @@ ALLOW_SELF_REVIEWS = True
 
 AES_KEYS = {
     'api_key:secret': os.path.join(
-        ROOT, 'src', 'olympia', 'api', 'tests', 'assets', 'test-api-key.txt'),
+        ROOT, 'src', 'olympia', 'api', 'tests', 'assets', 'test-api-key.txt'
+    ),
 }
 
 DATABASES = {
@@ -106,22 +98,25 @@ FXA_CONFIG = {
         'client_id': env('FXA_CLIENT_ID', default='a25796da7bc73ffa'),
         'client_secret': env(
             'FXA_CLIENT_SECRET',
-            default='4828af02f60a12738a79c7121b06d42b481f112dce1831440902a8412d2770c5'),  # noqa
+            default='4828af02f60a12738a79c7121b06d42b481f112dce1831440902a8412d2770c5',
+        ),
         # fxa redirects to http://olympia.test/api/auth/authenticate-callback/
     },
     'amo': {
         'client_id': env('FXA_CLIENT_ID', default='0f95f6474c24c1dc'),
         'client_secret': env(
             'FXA_CLIENT_SECRET',
-            default='ca45e503a1b4ec9e2a3d4855d79849e098da18b7dfe42b6bc76dfed420fc1d38'),  # noqa
+            default='ca45e503a1b4ec9e2a3d4855d79849e098da18b7dfe42b6bc76dfed420fc1d38',
+        ),
         # fxa redirects to http://localhost:3000/fxa-authenticate
     },
     'local': {
         'client_id': env('FXA_CLIENT_ID', default='4dce1adfa7901c08'),
         'client_secret': env(
             'FXA_CLIENT_SECRET',
-            default='d7d5f1148a35b12c067fb9eafafc29d35165a90f5d8b0032f1fcd37468ae49fe'),  # noqa
+            default='d7d5f1148a35b12c067fb9eafafc29d35165a90f5d8b0032f1fcd37468ae49fe',
-        # noqa fxa redirects to http://localhost:3000/api/auth/authenticate-callback/?config=local  #noqa
+        ),
+        # fxa redirects to http://localhost:3000/api/auth/authenticate-callback/?config=local  # noqa
     },
 }
 FXA_CONTENT_HOST = 'https://stable.dev.lcip.org'
@@ -168,7 +163,8 @@ except ImportError:
     import warnings
     import traceback
 
-    warnings.warn('Could not import local_settings module. {}'.format(
+    warnings.warn(
-        traceback.format_exc()))
+        'Could not import local_settings module. {}'.format(traceback.format_exc())
+    )
 
 SITEMAP_DEBUG_AVAILABLE = True

src/olympia/conf/dev/settings.py

@@ -52,12 +52,6 @@ SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
 
 REPLICA_DATABASES = ['replica']
 
-CACHES = {}
-CACHES['default'] = env.cache('CACHES_DEFAULT')
-CACHES['default']['TIMEOUT'] = 500
-CACHES['default']['BACKEND'] = CACHE_BACKEND
-CACHES['default']['KEY_PREFIX'] = CACHE_KEY_PREFIX
-
 # Celery
 CELERY_WORKER_PREFETCH_MULTIPLIER = 1
 

src/olympia/conf/prod/settings.py

@@ -44,12 +44,6 @@ SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
 
 REPLICA_DATABASES = ['replica']
 
-CACHES = {}
-CACHES['default'] = env.cache('CACHES_DEFAULT')
-CACHES['default']['TIMEOUT'] = 500
-CACHES['default']['BACKEND'] = CACHE_BACKEND
-CACHES['default']['KEY_PREFIX'] = CACHE_KEY_PREFIX
-
 # Celery
 CELERY_BROKER_CONNECTION_TIMEOUT = 0.5
 

src/olympia/conf/stage/settings.py

@@ -49,12 +49,6 @@ SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
 
 REPLICA_DATABASES = ['replica']
 
-CACHES = {}
-CACHES['default'] = env.cache('CACHES_DEFAULT')
-CACHES['default']['TIMEOUT'] = 500
-CACHES['default']['BACKEND'] = CACHE_BACKEND
-CACHES['default']['KEY_PREFIX'] = CACHE_KEY_PREFIX
-
 # Celery
 CELERY_WORKER_PREFETCH_MULTIPLIER = 1
 

src/olympia/lib/cache.py

@@ -1,62 +0,0 @@
-import hashlib
-import functools
-import itertools
-
-from django.core.cache.backends.base import DEFAULT_TIMEOUT
-from django.core.cache import cache
-from django.utils import translation
-from django.utils.encoding import force_bytes, force_str
-
-
-def make_key(key, with_locale=True, normalize=False):
-    """Generate the full key for ``k``, with a prefix."""
-    if with_locale:
-        key = f'{key}:{translation.get_language()}'
-
-    if normalize:
-        return force_str(hashlib.md5(force_bytes(key)).hexdigest())
-    return force_str(key)
-
-
-def memoize_key(prefix, *args, **kwargs):
-    """
-    For a prefix and arguments returns a key suitable for use in memcache.
-    Used by memoize.
-
-    :param prefix: a prefix for the key in memcache
-    :type param: string
-    :param args: arguments to be str()'d to form the key
-    :type args: list
-    :param kwargs: arguments to be str()'d to form the key
-    :type kwargs: list
-    """
-    key = hashlib.md5()
-    for arg in itertools.chain(args, sorted(kwargs.items())):
-        key.update(force_bytes(arg))
-    return f'memoize:{prefix}:{key.hexdigest()}'
-
-
-def memoize(prefix, timeout=60):
-    """
-    A simple decorator that caches into memcache, using a simple
-    key based on stringing args and kwargs.
-    Arguments to the method must be easily and consistently serializable
-    using str(..) otherwise the cache key will be inconsistent.
-    :param prefix: a prefix for the key in memcache
-    :type prefix: string
-    :param timeout: number of seconds to cache the key for, default 60 seconds
-    :type timeout: integer
-    """
-
-    def decorator(func):
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            def wrapped_func():
-                return func(*args, **kwargs)
-
-            key = memoize_key(prefix, *args, **kwargs)
-            return cache.get_or_set(key, wrapped_func, timeout=DEFAULT_TIMEOUT)
-
-        return wrapper
-
-    return decorator

src/olympia/lib/settings_base.py

@@ -917,29 +917,16 @@ MINIFY_BUNDLES = {
     },
 }
 
+CACHES = {
+    'default': env.cache(
+        'CACHES_DEFAULT',
+        'memcache://%s' % os.environ.get('MEMCACHE_LOCATION', 'localhost:11211'),
+    )
+}
+CACHES['default']['TIMEOUT'] = 500
+CACHES['default']['BACKEND'] = 'django.core.cache.backends.memcached.PyMemcacheCache'
 # Prefix for cache keys (will prevent collisions when running parallel copies)
-# This value is being used by `conf/settings/{dev,stage,prod}.py
+CACHES['default']['KEY_PREFIX'] = 'amo:%s:' % BUILD_ID
-CACHE_KEY_PREFIX = 'amo:%s:' % BUILD_ID
-
-CACHE_BACKEND = 'django.core.cache.backends.memcached.PyMemcacheCache'
-
-CACHE_MIDDLEWARE_KEY_PREFIX = CACHE_KEY_PREFIX
-FETCH_BY_ID = True
-
-# Number of seconds a count() query should be cached.  Keep it short because
-# it's not possible to invalidate these queries.
-CACHE_COUNT_TIMEOUT = 60
-
-# To enable pylibmc compression (in bytes)
-PYLIBMC_MIN_COMPRESS_LEN = 0  # disabled
-
-# External tools.
-JAVA_BIN = '/usr/bin/java'
-
-# URL paths
-# paths for images, e.g. mozcdn.com/amo or '/static'
-VAMO_URL = 'https://versioncheck.addons.mozilla.org'
-
 
 # Outgoing URL bouncer
 REDIRECT_URL = 'https://outgoing.prod.mozaws.net/v1/'

src/olympia/lib/tests/test_cache.py

@@ -1,50 +0,0 @@
-from django.utils import translation
-from django.core.cache import cache
-
-from olympia.lib.cache import memoize, memoize_key, make_key
-
-
-def test_make_key():
-    with translation.override('en-US'):
-        assert make_key('é@øel') == 'é@øel:en-us'
-
-    with translation.override('de'):
-        assert make_key('é@øel') == 'é@øel:de'
-
-    with translation.override('de'):
-        assert make_key('é@øel', with_locale=False) == 'é@øel'
-
-    with translation.override('en-US'):
-        assert make_key('é@øel', normalize=True) == '2798e65bbe384320c9da7930e93e63fb'
-
-    assert (
-        make_key('é@øel', with_locale=False, normalize=True)
-        == 'a83feada27737072d4ec741640368f07'
-    )
-
-    with translation.override('fr'):
-        assert make_key('é@øel', normalize=True) == 'bc5208e905c8dfcc521e4196e16cfa1a'
-
-
-def test_memoize_key():
-    assert memoize_key('foo', ['a', 'b'], {'c': 'e'}) == (
-        'memoize:foo:9666a2a48c17dc1c308fb327c2a6e3a8'
-    )
-
-
-def test_memoize():
-    @memoize('f')
-    def add(*args):
-        return sum(args)
-
-    cache_key = memoize_key('f', 1, 2)
-    assert add(1, 2) == cache.get(cache_key)
-
-
-def test_memcached_unicode():
-    """Regression test for
-
-    https://github.com/linsomniac/python-memcached/issues/79
-    """
-    cache.set('këy', 'Iñtërnâtiônàlizætiøn2')
-    assert cache.get('këy') == 'Iñtërnâtiônàlizætiøn2'

src/olympia/reviewers/serializers.py

@@ -1,3 +1,4 @@
+import hashlib
 import io
 import os
 import mimetypes
@@ -14,7 +15,7 @@ from rest_framework.reverse import reverse as drf_reverse
 from django.core.cache import cache
 from django.urls import reverse
 from django.utils.functional import cached_property
-from django.utils.encoding import force_str
+from django.utils.encoding import force_bytes, force_str
 from django.utils.translation import gettext
 
 from olympia import amo
@@ -34,7 +35,6 @@ from olympia.files.models import File, FileValidation
 from olympia.reviewers.models import CannedResponse
 from olympia.versions.models import Version
 from olympia.git.utils import AddonGitRepository, get_mime_type_for_blob
-from olympia.lib.cache import make_key
 from olympia.lib import unicodehelper
 
 
@@ -141,10 +141,13 @@ class FileEntriesMixin:
 
         # Normalize the key as we want to avoid that we exceed max
         # key lengh because of selected_file.
-        cache_key = make_key(
+        cache_key = force_str(
-            f'reviewers:fileentriesserializer:hashes' f':{commit.hex}:{selected_file}',
+            hashlib.sha256(
-            with_locale=False,
+                force_bytes(
-            normalize=True,
+                    'reviewers:fileentriesserializer:hashes'
+                    f':{commit.hex}:{selected_file}',
+                )
+            ).hexdigest()
         )
 
         def _calculate_hash():