Remove lib/cache.py, simplify CACHES settings (#18706)
* Remove lib/cache.py, simplify CACHES settings * Remove useless noqa comments * Useless f prefix
This commit is contained in:
Mathieu Pillard
GitHub
Родитель
7963068b9d
Коммит
71ea6b9fde
34
settings.py
34
settings.py
|
|
@ -38,7 +38,8 @@ def insert_debug_toolbar_middleware(middlewares):
|
|||
for i, middleware in enumerate(ret_middleware):
|
||||
if 'GZipMiddleware' in middleware:
|
||||
ret_middleware.insert(
|
||||
i + 1, 'debug_toolbar.middleware.DebugToolbarMiddleware')
|
||||
i + 1, 'debug_toolbar.middleware.DebugToolbarMiddleware'
|
||||
)
|
||||
break
|
||||
|
||||
return tuple(ret_middleware)
|
||||
|
|
@ -53,15 +54,6 @@ DEBUG_TOOLBAR_CONFIG = {
|
|||
|
||||
FILESYSTEM_CACHE_ROOT = os.path.join(TMP_PATH, 'cache')
|
||||
|
||||
# We are setting memcached here to make sure our local setup is as close
|
||||
# to our production system as possible.
|
||||
CACHES = {
|
||||
'default': {
|
||||
'BACKEND': CACHE_BACKEND,
|
||||
'LOCATION': os.environ.get('MEMCACHE_LOCATION', 'localhost:11211'),
|
||||
},
|
||||
}
|
||||
|
||||
# If you're not running on SSL you'll want this to be False.
|
||||
SESSION_COOKIE_SECURE = False
|
||||
SESSION_COOKIE_DOMAIN = None
|
||||
|
|
@ -81,8 +73,7 @@ EXTERNAL_SITE_URL = SITE_URL
|
|||
STATIC_URL = '%s/static/' % EXTERNAL_SITE_URL
|
||||
MEDIA_URL = '%s/user-media/' % EXTERNAL_SITE_URL
|
||||
|
||||
CODE_MANAGER_URL = (
|
||||
os.environ.get('CODE_MANAGER_URL') or 'http://olympia.test:5000')
|
||||
CODE_MANAGER_URL = os.environ.get('CODE_MANAGER_URL') or 'http://olympia.test:5000'
|
||||
|
||||
ALLOWED_HOSTS = ALLOWED_HOSTS + [SERVICES_DOMAIN, 'nginx']
|
||||
|
||||
|
|
@ -93,7 +84,8 @@ ALLOW_SELF_REVIEWS = True
|
|||
|
||||
AES_KEYS = {
|
||||
'api_key:secret': os.path.join(
|
||||
ROOT, 'src', 'olympia', 'api', 'tests', 'assets', 'test-api-key.txt'),
|
||||
ROOT, 'src', 'olympia', 'api', 'tests', 'assets', 'test-api-key.txt'
|
||||
),
|
||||
}
|
||||
|
||||
DATABASES = {
|
||||
|
|
@ -106,22 +98,25 @@ FXA_CONFIG = {
|
|||
'client_id': env('FXA_CLIENT_ID', default='a25796da7bc73ffa'),
|
||||
'client_secret': env(
|
||||
'FXA_CLIENT_SECRET',
|
||||
default='4828af02f60a12738a79c7121b06d42b481f112dce1831440902a8412d2770c5'), # noqa
|
||||
default='4828af02f60a12738a79c7121b06d42b481f112dce1831440902a8412d2770c5',
|
||||
),
|
||||
# fxa redirects to http://olympia.test/api/auth/authenticate-callback/
|
||||
},
|
||||
'amo': {
|
||||
'client_id': env('FXA_CLIENT_ID', default='0f95f6474c24c1dc'),
|
||||
'client_secret': env(
|
||||
'FXA_CLIENT_SECRET',
|
||||
default='ca45e503a1b4ec9e2a3d4855d79849e098da18b7dfe42b6bc76dfed420fc1d38'), # noqa
|
||||
default='ca45e503a1b4ec9e2a3d4855d79849e098da18b7dfe42b6bc76dfed420fc1d38',
|
||||
),
|
||||
# fxa redirects to http://localhost:3000/fxa-authenticate
|
||||
},
|
||||
'local': {
|
||||
'client_id': env('FXA_CLIENT_ID', default='4dce1adfa7901c08'),
|
||||
'client_secret': env(
|
||||
'FXA_CLIENT_SECRET',
|
||||
default='d7d5f1148a35b12c067fb9eafafc29d35165a90f5d8b0032f1fcd37468ae49fe'), # noqa
|
||||
# noqa fxa redirects to http://localhost:3000/api/auth/authenticate-callback/?config=local #noqa
|
||||
default='d7d5f1148a35b12c067fb9eafafc29d35165a90f5d8b0032f1fcd37468ae49fe',
|
||||
),
|
||||
# fxa redirects to http://localhost:3000/api/auth/authenticate-callback/?config=local # noqa
|
||||
},
|
||||
}
|
||||
FXA_CONTENT_HOST = 'https://stable.dev.lcip.org'
|
||||
|
|
@ -168,7 +163,8 @@ except ImportError:
|
|||
import warnings
|
||||
import traceback
|
||||
|
||||
warnings.warn('Could not import local_settings module. {}'.format(
|
||||
traceback.format_exc()))
|
||||
warnings.warn(
|
||||
'Could not import local_settings module. {}'.format(traceback.format_exc())
|
||||
)
|
||||
|
||||
SITEMAP_DEBUG_AVAILABLE = True
|
||||
|
|
|
|||
|
|
@ -52,12 +52,6 @@ SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
|
|||
|
||||
REPLICA_DATABASES = ['replica']
|
||||
|
||||
CACHES = {}
|
||||
CACHES['default'] = env.cache('CACHES_DEFAULT')
|
||||
CACHES['default']['TIMEOUT'] = 500
|
||||
CACHES['default']['BACKEND'] = CACHE_BACKEND
|
||||
CACHES['default']['KEY_PREFIX'] = CACHE_KEY_PREFIX
|
||||
|
||||
# Celery
|
||||
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
|
||||
|
||||
|
|
|
|||
|
|
@ -44,12 +44,6 @@ SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
|
|||
|
||||
REPLICA_DATABASES = ['replica']
|
||||
|
||||
CACHES = {}
|
||||
CACHES['default'] = env.cache('CACHES_DEFAULT')
|
||||
CACHES['default']['TIMEOUT'] = 500
|
||||
CACHES['default']['BACKEND'] = CACHE_BACKEND
|
||||
CACHES['default']['KEY_PREFIX'] = CACHE_KEY_PREFIX
|
||||
|
||||
# Celery
|
||||
CELERY_BROKER_CONNECTION_TIMEOUT = 0.5
|
||||
|
||||
|
|
|
|||
|
|
@ -49,12 +49,6 @@ SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
|
|||
|
||||
REPLICA_DATABASES = ['replica']
|
||||
|
||||
CACHES = {}
|
||||
CACHES['default'] = env.cache('CACHES_DEFAULT')
|
||||
CACHES['default']['TIMEOUT'] = 500
|
||||
CACHES['default']['BACKEND'] = CACHE_BACKEND
|
||||
CACHES['default']['KEY_PREFIX'] = CACHE_KEY_PREFIX
|
||||
|
||||
# Celery
|
||||
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
|
||||
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
import hashlib
|
||||
import functools
|
||||
import itertools
|
||||
|
||||
from django.core.cache.backends.base import DEFAULT_TIMEOUT
|
||||
from django.core.cache import cache
|
||||
from django.utils import translation
|
||||
from django.utils.encoding import force_bytes, force_str
|
||||
|
||||
|
||||
def make_key(key, with_locale=True, normalize=False):
|
||||
"""Generate the full key for ``k``, with a prefix."""
|
||||
if with_locale:
|
||||
key = f'{key}:{translation.get_language()}'
|
||||
|
||||
if normalize:
|
||||
return force_str(hashlib.md5(force_bytes(key)).hexdigest())
|
||||
return force_str(key)
|
||||
|
||||
|
||||
def memoize_key(prefix, *args, **kwargs):
|
||||
"""
|
||||
For a prefix and arguments returns a key suitable for use in memcache.
|
||||
Used by memoize.
|
||||
|
||||
:param prefix: a prefix for the key in memcache
|
||||
:type param: string
|
||||
:param args: arguments to be str()'d to form the key
|
||||
:type args: list
|
||||
:param kwargs: arguments to be str()'d to form the key
|
||||
:type kwargs: list
|
||||
"""
|
||||
key = hashlib.md5()
|
||||
for arg in itertools.chain(args, sorted(kwargs.items())):
|
||||
key.update(force_bytes(arg))
|
||||
return f'memoize:{prefix}:{key.hexdigest()}'
|
||||
|
||||
|
||||
def memoize(prefix, timeout=60):
|
||||
"""
|
||||
A simple decorator that caches into memcache, using a simple
|
||||
key based on stringing args and kwargs.
|
||||
Arguments to the method must be easily and consistently serializable
|
||||
using str(..) otherwise the cache key will be inconsistent.
|
||||
:param prefix: a prefix for the key in memcache
|
||||
:type prefix: string
|
||||
:param timeout: number of seconds to cache the key for, default 60 seconds
|
||||
:type timeout: integer
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
def wrapped_func():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
key = memoize_key(prefix, *args, **kwargs)
|
||||
return cache.get_or_set(key, wrapped_func, timeout=DEFAULT_TIMEOUT)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
|
@ -917,29 +917,16 @@ MINIFY_BUNDLES = {
|
|||
},
|
||||
}
|
||||
|
||||
CACHES = {
|
||||
'default': env.cache(
|
||||
'CACHES_DEFAULT',
|
||||
'memcache://%s' % os.environ.get('MEMCACHE_LOCATION', 'localhost:11211'),
|
||||
)
|
||||
}
|
||||
CACHES['default']['TIMEOUT'] = 500
|
||||
CACHES['default']['BACKEND'] = 'django.core.cache.backends.memcached.PyMemcacheCache'
|
||||
# Prefix for cache keys (will prevent collisions when running parallel copies)
|
||||
# This value is being used by `conf/settings/{dev,stage,prod}.py
|
||||
CACHE_KEY_PREFIX = 'amo:%s:' % BUILD_ID
|
||||
|
||||
CACHE_BACKEND = 'django.core.cache.backends.memcached.PyMemcacheCache'
|
||||
|
||||
CACHE_MIDDLEWARE_KEY_PREFIX = CACHE_KEY_PREFIX
|
||||
FETCH_BY_ID = True
|
||||
|
||||
# Number of seconds a count() query should be cached. Keep it short because
|
||||
# it's not possible to invalidate these queries.
|
||||
CACHE_COUNT_TIMEOUT = 60
|
||||
|
||||
# To enable pylibmc compression (in bytes)
|
||||
PYLIBMC_MIN_COMPRESS_LEN = 0 # disabled
|
||||
|
||||
# External tools.
|
||||
JAVA_BIN = '/usr/bin/java'
|
||||
|
||||
# URL paths
|
||||
# paths for images, e.g. mozcdn.com/amo or '/static'
|
||||
VAMO_URL = 'https://versioncheck.addons.mozilla.org'
|
||||
|
||||
CACHES['default']['KEY_PREFIX'] = 'amo:%s:' % BUILD_ID
|
||||
|
||||
# Outgoing URL bouncer
|
||||
REDIRECT_URL = 'https://outgoing.prod.mozaws.net/v1/'
|
||||
|
|
|
|||
|
|
@ -1,50 +0,0 @@
|
|||
from django.utils import translation
|
||||
from django.core.cache import cache
|
||||
|
||||
from olympia.lib.cache import memoize, memoize_key, make_key
|
||||
|
||||
|
||||
def test_make_key():
|
||||
with translation.override('en-US'):
|
||||
assert make_key('é@øel') == 'é@øel:en-us'
|
||||
|
||||
with translation.override('de'):
|
||||
assert make_key('é@øel') == 'é@øel:de'
|
||||
|
||||
with translation.override('de'):
|
||||
assert make_key('é@øel', with_locale=False) == 'é@øel'
|
||||
|
||||
with translation.override('en-US'):
|
||||
assert make_key('é@øel', normalize=True) == '2798e65bbe384320c9da7930e93e63fb'
|
||||
|
||||
assert (
|
||||
make_key('é@øel', with_locale=False, normalize=True)
|
||||
== 'a83feada27737072d4ec741640368f07'
|
||||
)
|
||||
|
||||
with translation.override('fr'):
|
||||
assert make_key('é@øel', normalize=True) == 'bc5208e905c8dfcc521e4196e16cfa1a'
|
||||
|
||||
|
||||
def test_memoize_key():
|
||||
assert memoize_key('foo', ['a', 'b'], {'c': 'e'}) == (
|
||||
'memoize:foo:9666a2a48c17dc1c308fb327c2a6e3a8'
|
||||
)
|
||||
|
||||
|
||||
def test_memoize():
|
||||
@memoize('f')
|
||||
def add(*args):
|
||||
return sum(args)
|
||||
|
||||
cache_key = memoize_key('f', 1, 2)
|
||||
assert add(1, 2) == cache.get(cache_key)
|
||||
|
||||
|
||||
def test_memcached_unicode():
|
||||
"""Regression test for
|
||||
|
||||
https://github.com/linsomniac/python-memcached/issues/79
|
||||
"""
|
||||
cache.set('këy', 'Iñtërnâtiônàlizætiøn2')
|
||||
assert cache.get('këy') == 'Iñtërnâtiônàlizætiøn2'
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import mimetypes
|
||||
|
|
@ -14,7 +15,7 @@ from rest_framework.reverse import reverse as drf_reverse
|
|||
from django.core.cache import cache
|
||||
from django.urls import reverse
|
||||
from django.utils.functional import cached_property
|
||||
from django.utils.encoding import force_str
|
||||
from django.utils.encoding import force_bytes, force_str
|
||||
from django.utils.translation import gettext
|
||||
|
||||
from olympia import amo
|
||||
|
|
@ -34,7 +35,6 @@ from olympia.files.models import File, FileValidation
|
|||
from olympia.reviewers.models import CannedResponse
|
||||
from olympia.versions.models import Version
|
||||
from olympia.git.utils import AddonGitRepository, get_mime_type_for_blob
|
||||
from olympia.lib.cache import make_key
|
||||
from olympia.lib import unicodehelper
|
||||
|
||||
|
||||
|
|
@ -141,10 +141,13 @@ class FileEntriesMixin:
|
|||
|
||||
# Normalize the key as we want to avoid that we exceed max
|
||||
# key lengh because of selected_file.
|
||||
cache_key = make_key(
|
||||
f'reviewers:fileentriesserializer:hashes' f':{commit.hex}:{selected_file}',
|
||||
with_locale=False,
|
||||
normalize=True,
|
||||
cache_key = force_str(
|
||||
hashlib.sha256(
|
||||
force_bytes(
|
||||
'reviewers:fileentriesserializer:hashes'
|
||||
f':{commit.hex}:{selected_file}',
|
||||
)
|
||||
).hexdigest()
|
||||
)
|
||||
|
||||
def _calculate_hash():
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче