mozilla
/
addons-server
зеркало из https://github.com/mozilla/addons-server.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

"Black" all the things! (#16123)

This commit is contained in:
William Durand 2020-12-09 19:24:56 +01:00 коммит произвёл GitHub
Родитель 8572b7906c
Коммит e61d8b0de7
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
490 изменённых файлов: 33737 добавлений и 26230 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
Makefile-docker
Просмотреть файл

@ -189,6 +189,7 @@ setup-codestyle:
.PHONY: lint
lint: ## lint the code
black --check src/ services/ tests/
flake8 src/ services/ tests/
$(shell npm $(NPM_ARGS) bin)/prettier --check '**'
curlylint src/
@ -255,6 +256,7 @@ watch_js_tests: ## Run+watch the JavaScript test suite (requires compiled/compre
.PHONY: format
format: ## Autoformat our codebase.
$(shell npm $(NPM_ARGS) bin)/prettier --write '**'
black src/ services/ tests/
.PHONY: help_submake
help_submake:

23
pyproject.toml Normal file
Просмотреть файл

@ -0,0 +1,23 @@
[tool.black]
line-length = 88
target-version = ['py38']
skip-string-normalization = true
exclude = '''
(
/(
docs
| node_modules
| build*.py
| media
| storage
| logs
| site-static
| static
| \.git
| \.npm
| \.tox
)/
| src/.*/migrations/.*\.py
| src/olympia/translations/tests/testapp/migrations/.*\.py
)
'''

82
requirements/codestyle.txt
Просмотреть файл

@ -45,3 +45,85 @@ curlylint==0.12.0 \
toml==0.10.2 \
--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
black==20.8b1 \
--hash=sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea
typing-extensions==3.7.4.3 \
--hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \
--hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \
--hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f
regex==2020.11.13 \
--hash=sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538 \
--hash=sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4 \
--hash=sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc \
--hash=sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa \
--hash=sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444 \
--hash=sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1 \
--hash=sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af \
--hash=sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8 \
--hash=sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9 \
--hash=sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88 \
--hash=sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba \
--hash=sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364 \
--hash=sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e \
--hash=sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7 \
--hash=sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0 \
--hash=sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31 \
--hash=sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683 \
--hash=sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee \
--hash=sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b \
--hash=sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884 \
--hash=sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c \
--hash=sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e \
--hash=sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562 \
--hash=sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85 \
--hash=sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c \
--hash=sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6 \
--hash=sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d \
--hash=sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b \
--hash=sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70 \
--hash=sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b \
--hash=sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b \
--hash=sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f \
--hash=sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0 \
--hash=sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5 \
--hash=sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5 \
--hash=sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f \
--hash=sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e \
--hash=sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512 \
--hash=sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d \
--hash=sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917 \
--hash=sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f
mypy-extensions==0.4.3 \
--hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \
--hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8
typed-ast==1.4.1 \
--hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \
--hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \
--hash=sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d \
--hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \
--hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \
--hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \
--hash=sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c \
--hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \
--hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \
--hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \
--hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \
--hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \
--hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \
--hash=sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d \
--hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \
--hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \
--hash=sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c \
--hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \
--hash=sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395 \
--hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \
--hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \
--hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \
--hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \
--hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \
--hash=sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072 \
--hash=sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298 \
--hash=sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91 \
--hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \
--hash=sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f \
--hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7

3
requirements/tests.txt
Просмотреть файл

@ -90,3 +90,6 @@ WebOb==1.8.6 \
wcwidth==0.2.5 \
--hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \
--hash=sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83
toml==0.10.2 \
--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f

105
services/update.py
Просмотреть файл

@ -6,7 +6,12 @@ from urllib.parse import parse_qsl
from time import time
from services.utils import (
get_cdn_url, log_configure, mypool, settings, PLATFORM_NAMES_TO_CONSTANTS)
get_cdn_url,
log_configure,
mypool,
settings,
PLATFORM_NAMES_TO_CONSTANTS,
)
# This has to be imported after the settings so statsd knows where to log to.
from django_statsd.clients import statsd
@ -27,7 +32,6 @@ log = olympia.core.logger.getLogger('z.services')
class Update(object):
def __init__(self, data, compat_mode='strict'):
self.conn, self.cursor = None, None
self.data = data.copy()
@ -60,11 +64,14 @@ class Update(object):
inactive = 0 AND
status NOT IN (%(STATUS_DELETED)s, %(STATUS_DISABLED)s)
LIMIT 1;"""
self.cursor.execute(sql, {
'guid': self.data['id'],
'STATUS_DELETED': base.STATUS_DELETED,
'STATUS_DISABLED': base.STATUS_DISABLED,
})
self.cursor.execute(
sql,
{
'guid': self.data['id'],
'STATUS_DELETED': base.STATUS_DELETED,
'STATUS_DISABLED': base.STATUS_DISABLED,
},
)
result = self.cursor.fetchone()
if result is None:
return False
@ -88,7 +95,8 @@ class Update(object):
data['STATUS_APPROVED'] = base.STATUS_APPROVED
data['RELEASE_CHANNEL_LISTED'] = base.RELEASE_CHANNEL_LISTED
sql = ["""
sql = [
"""
SELECT
addons.guid as guid, addons.addontype_id as type,
addons.inactive as disabled_by_user, appmin.version as min,
@ -111,11 +119,13 @@ class Update(object):
AND appmax.application_id = %(app_id)s
INNER JOIN files
ON files.version_id = versions.id AND (files.platform_id = 1
"""]
"""
]
if data.get('appOS'):
sql.append(' OR files.platform_id = %(appOS)s')
sql.append("""
sql.append(
"""
)
-- Find a reference to the user's current version, if it exists.
-- These should never be inner joins. We need results even if we
@ -128,7 +138,8 @@ class Update(object):
versions.deleted = 0 AND
versions.channel = %(RELEASE_CHANNEL_LISTED)s AND
files.status = %(STATUS_APPROVED)s
""")
"""
)
sql.append('AND appmin.version_int <= %(version_int)s ')
@ -138,11 +149,13 @@ class Update(object):
elif self.compat_mode == 'normal':
# When file has strict_compatibility enabled, or file has binary
# components, default to compatible is disabled.
sql.append("""AND
sql.append(
"""AND
CASE WHEN files.strict_compatibility = 1 OR
files.binary_components = 1
THEN appmax.version_int >= %(version_int)s ELSE 1 END
""")
"""
)
# Filter out versions that don't have the minimum maxVersion
# requirement to qualify for default-to-compatible.
d2c_min = applications.D2C_MIN_VERSIONS.get(data['app_id'])
@ -158,12 +171,27 @@ class Update(object):
result = self.cursor.fetchone()
if result:
row = dict(zip([
'guid', 'type', 'disabled_by_user', 'min', 'max',
'file_id', 'file_status', 'hash', 'filename', 'version_id',
'datestatuschanged', 'strict_compat', 'releasenotes',
'version'],
list(result)))
row = dict(
zip(
[
'guid',
'type',
'disabled_by_user',
'min',
'max',
'file_id',
'file_status',
'hash',
'filename',
'version_id',
'datestatuschanged',
'strict_compat',
'releasenotes',
'version',
],
list(result),
)
)
row['type'] = base.ADDON_SLUGS_UPDATE[row['type']]
row['url'] = get_cdn_url(data['id'], row)
row['appguid'] = applications.APPS_ALL[data['app_id']].guid
@ -189,24 +217,14 @@ class Update(object):
return {}
def get_no_updates_output(self):
return {
'addons': {
self.data['guid']: {
'updates': []
}
}
}
return {'addons': {self.data['guid']: {'updates': []}}}
def get_success_output(self):
data = self.data['row']
update = {
'version': data['version'],
'update_link': data['url'],
'applications': {
'gecko': {
'strict_min_version': data['min']
}
}
'applications': {'gecko': {'strict_min_version': data['min']}},
}
if data['strict_compat']:
update['applications']['gecko']['strict_max_version'] = data['max']
@ -214,25 +232,24 @@ class Update(object):
update['update_hash'] = data['hash']
if data['releasenotes']:
update['update_info_url'] = '%s%s%s/%%APP_LOCALE%%/' % (
settings.SITE_URL, '/versions/updateInfo/', data['version_id'])
return {
'addons': {
self.data['guid']: {
'updates': [update]
}
}
}
settings.SITE_URL,
'/versions/updateInfo/',
data['version_id'],
)
return {'addons': {self.data['guid']: {'updates': [update]}}}
def format_date(self, secs):
return '%s GMT' % formatdate(time() + secs)[:25]
def get_headers(self, length):
content_type = 'application/json'
return [('Content-Type', content_type),
('Cache-Control', 'public, max-age=3600'),
('Last-Modified', self.format_date(0)),
('Expires', self.format_date(3600)),
('Content-Length', str(length))]
return [
('Content-Type', content_type),
('Cache-Control', 'public, max-age=3600'),
('Last-Modified', self.format_date(0)),
('Expires', self.format_date(3600)),
('Content-Length', str(length)),
]
def application(environ, start_response):

21
services/utils.py
Просмотреть файл

@ -55,7 +55,8 @@ PLATFORM_NAMES_TO_CONSTANTS = {
}
version_re = re.compile(r"""(?P<major>\d+) # major (x in x.y)
version_re = re.compile(
r"""(?P<major>\d+) # major (x in x.y)
\.(?P<minor1>\d+) # minor1 (y in x.y)
\.?(?P<minor2>\d+|\*)? # minor2 (z in x.y.z)
\.?(?P<minor3>\d+|\*)? # minor3 (w in x.y.z.w)
@ -63,7 +64,9 @@ version_re = re.compile(r"""(?P<major>\d+) # major (x in x.y)
(?P<alpha_ver>\d*) # alpha/beta version
(?P<pre>pre)? # pre release
(?P<pre_ver>\d)? # pre release version
""", re.VERBOSE)
""",
re.VERBOSE,
)
def get_cdn_url(id, row):
@ -75,9 +78,13 @@ def get_cdn_url(id, row):
def getconn():
db = settings.SERVICES_DATABASE
return mysql.connect(host=db['HOST'], user=db['USER'],
passwd=db['PASSWORD'], db=db['NAME'],
charset=db['OPTIONS']['charset'])
return mysql.connect(
host=db['HOST'],
user=db['USER'],
passwd=db['PASSWORD'],
db=db['NAME'],
charset=db['OPTIONS']['charset'],
)
mypool = pool.QueuePool(getconn, max_overflow=10, pool_size=5, recycle=300)
@ -92,13 +99,13 @@ def log_configure():
'mozlog': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'json'
'formatter': 'json',
},
},
'formatters': {
'json': {
'()': olympia.core.logger.JsonFormatter,
'logger_name': 'http_app_addons'
'logger_name': 'http_app_addons',
},
},
}

3
services/wsgi/pfs.py
Просмотреть файл

@ -4,8 +4,7 @@ import site
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local")
wsgidir = os.path.dirname(__file__)
for path in ['../', '../..',
'../../apps']:
for path in ['../', '../..', '../../apps']:
site.addsitedir(os.path.abspath(os.path.join(wsgidir, path)))
from ..pfs import application # noqa

4
services/wsgi/theme_update.py
Просмотреть файл

@ -4,9 +4,7 @@ import site
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local")
wsgidir = os.path.dirname(__file__)
for path in ['../',
'../..',
'../../apps']:
for path in ['../', '../..', '../../apps']:
site.addsitedir(os.path.abspath(os.path.join(wsgidir, path)))
from ..theme_update import application # noqa

5
services/wsgi/versioncheck.py
Просмотреть файл

@ -4,10 +4,7 @@ import site
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local")
wsgidir = os.path.dirname(__file__)
for path in ['../',
'../..',
'../../..',
'../../apps']:
for path in ['../', '../..', '../../..', '../../apps']:
site.addsitedir(os.path.abspath(os.path.join(wsgidir, path)))
from update import application # noqa

7
setup.cfg
Просмотреть файл

@ -23,7 +23,10 @@ filterwarnings =
ignore::ResourceWarning
[flake8]
ignore = F999,F405,W504
# E203: https://github.com/psf/black/issues/315
# W503: https://www.flake8rules.com/rules/W503.html
ignore = F999,F405,W503,E203
max-line-length = 88
exclude =
src/olympia/wsgi.py,
docs,
@ -49,5 +52,5 @@ default_section = THIRDPARTY
include_trailing_comma = false
known_django = django
known_olympia = olympia
line_length = 79
line_length = 88
sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,OLYMPIA,FIRSTPARTY,LOCALFOLDER

185
src/olympia/abuse/admin.py
Просмотреть файл

@ -51,8 +51,7 @@ class AbuseReportTypeFilter(admin.SimpleListFilter):
if self.value() == 'user':
return queryset.filter(user__isnull=False)
elif self.value() == 'addon':
return queryset.filter(Q(addon__isnull=False) |
Q(guid__isnull=False))
return queryset.filter(Q(addon__isnull=False) | Q(guid__isnull=False))
return queryset
@ -69,8 +68,10 @@ class FakeChoicesMixin(object):
"""
# Grab search query parts and filter query parts as tuples of tuples.
search_query_parts = (
((admin.views.main.SEARCH_VAR, changelist.query),)
) if changelist.query else ()
(((admin.views.main.SEARCH_VAR, changelist.query),))
if changelist.query
else ()
)
filters_query_parts = tuple(
(k, v)
for k, v in changelist.get_filters_params().items()
@ -95,6 +96,7 @@ class MinimumReportsCountFilter(FakeChoicesMixin, admin.SimpleListFilter):
Original idea:
https://hakibenita.com/how-to-add-a-text-filter-to-django-admin
"""
template = 'admin/abuse/abusereport/minimum_reports_count_filter.html'
title = ugettext('minimum reports count (grouped by guid)')
parameter_name = 'minimum_reports_count'
@ -122,24 +124,33 @@ class DateRangeFilter(FakeChoicesMixin, DateRangeFilterBase):
Needs FakeChoicesMixin for the fake choices the template will be using (the
upstream implementation depends on JavaScript for this).
"""
template = 'admin/abuse/abusereport/date_range_filter.html'
title = ugettext('creation date')
def _get_form_fields(self):
return OrderedDict((
(self.lookup_kwarg_gte, forms.DateField(
label='From',
widget=HTML5DateInput(),
localize=True,
required=False
)),
(self.lookup_kwarg_lte, forms.DateField(
label='To',
widget=HTML5DateInput(),
localize=True,
required=False
)),
))
return OrderedDict(
(
(
self.lookup_kwarg_gte,
forms.DateField(
label='From',
widget=HTML5DateInput(),
localize=True,
required=False,
),
),
(
self.lookup_kwarg_lte,
forms.DateField(
label='To',
widget=HTML5DateInput(),
localize=True,
required=False,
),
),
)
)
def choices(self, changelist):
# We want a fake 'All' choice as per FakeChoicesMixin, but as of 0.3.15
@ -152,14 +163,20 @@ class DateRangeFilter(FakeChoicesMixin, DateRangeFilterBase):
class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
class Media:
css = {
'all': ('css/admin/abuse_reports.css',)
}
css = {'all': ('css/admin/abuse_reports.css',)}
actions = ('delete_selected', 'mark_as_valid', 'mark_as_suspicious')
date_hierarchy = 'modified'
list_display = ('target_name', 'guid', 'type', 'state', 'distribution',
'reason', 'message_excerpt', 'created')
list_display = (
'target_name',
'guid',
'type',
'state',
'distribution',
'reason',
'message_excerpt',
'created',
)
list_filter = (
AbuseReportTypeFilter,
'state',
@ -204,25 +221,30 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
ADDON_METADATA_FIELDSET = 'Add-on metadata'
fieldsets = (
(None, {'fields': ('state', 'reason', 'message')}),
(None, {'fields': (
'created',
'modified',
'reporter',
'country_code',
'client_id',
'addon_signature',
'application',
'application_version',
'application_locale',
'operating_system',
'operating_system_version',
'install_date',
'addon_install_origin',
'addon_install_method',
'addon_install_source',
'addon_install_source_url',
'report_entry_point'
)})
(
None,
{
'fields': (
'created',
'modified',
'reporter',
'country_code',
'client_id',
'addon_signature',
'application',
'application_version',
'application_locale',
'operating_system',
'operating_system_version',
'install_date',
'addon_install_origin',
'addon_install_method',
'addon_install_source',
'addon_install_source_url',
'report_entry_point',
)
},
),
)
# The first fieldset is going to be dynamically added through
# get_fieldsets() depending on the target (add-on, user or unknown add-on),
@ -247,7 +269,10 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
extra_context = extra_context or {}
extra_context['show_save_and_continue'] = False # Don't need this.
return super().change_view(
request, object_id, form_url, extra_context=extra_context,
request,
object_id,
form_url,
extra_context=extra_context,
)
def delete_queryset(self, request, queryset):
@ -269,12 +294,19 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
type_ = request.GET.get('type')
if type_ == 'addon':
search_fields = (
'addon__name__localized_string', 'addon__slug', 'addon_name',
'=guid', 'message', '=addon__id',
'addon__name__localized_string',
'addon__slug',
'addon_name',
'=guid',
'message',
'=addon__id',
)
elif type_ == 'user':
search_fields = (
'message', '=user__id', '^user__username', '^user__email',
'message',
'=user__id',
'^user__username',
'^user__email',
)
else:
search_fields = ()
@ -304,11 +336,13 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
# filtering is actually done here, because it needs to happen after
# all other filters have been applied in order for the aggregate
# queryset to be correct.
guids = (qs.values_list('guid', flat=True)
.filter(guid__isnull=False)
.annotate(Count('guid'))
.filter(guid__count__gte=minimum_reports_count)
.order_by())
guids = (
qs.values_list('guid', flat=True)
.filter(guid__isnull=False)
.annotate(Count('guid'))
.filter(guid__count__gte=minimum_reports_count)
.order_by()
)
qs = qs.filter(guid__in=list(guids))
qs, use_distinct = super().get_search_results(request, qs, search_term)
return qs, use_distinct
@ -322,8 +356,7 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
# through prefetch_related() + only_translations() (we don't care about
# the other transforms).
return qs.prefetch_related(
Prefetch(
'addon', queryset=Addon.unfiltered.all().only_translations()),
Prefetch('addon', queryset=Addon.unfiltered.all().only_translations()),
)
def get_fieldsets(self, request, obj=None):
@ -333,14 +366,13 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
target = 'user'
else:
target = 'guid'
dynamic_fieldset = (
(None, {'fields': self.dynamic_fieldset_fields[target]}),
)
dynamic_fieldset = ((None, {'fields': self.dynamic_fieldset_fields[target]}),)
return dynamic_fieldset + self.fieldsets
def target_name(self, obj):
name = obj.target.name if obj.target else obj.addon_name
return '%s %s' % (name, obj.addon_version or '')
target_name.short_description = ugettext('User / Add-on')
def addon_card(self, obj):
@ -359,28 +391,40 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
'addon_name': addon.name,
'approvals_info': approvals_info,
'reports': Paginator(
(AbuseReport.objects
.filter(Q(addon=addon) | Q(user__in=developers))
.order_by('-created')), 5).page(1),
(
AbuseReport.objects.filter(
Q(addon=addon) | Q(user__in=developers)
).order_by('-created')
),
5,
).page(1),
'user_ratings': Paginator(
(Rating.without_replies
.filter(addon=addon, rating__lte=3, body__isnull=False)
.order_by('-created')), 5).page(1),
(
Rating.without_replies.filter(
addon=addon, rating__lte=3, body__isnull=False
).order_by('-created')
),
5,
).page(1),
'version': addon.current_version,
}
return template.render(context)
addon_card.short_description = ''
def distribution(self, obj):
return obj.get_addon_signature_display() if obj.addon_signature else ''
distribution.short_description = ugettext('Distribution')
def reporter_country(self, obj):
return obj.country_code
reporter_country.short_description = ugettext("Reporter's country")
def message_excerpt(self, obj):
return truncate_text(obj.message, 140)[0] if obj.message else ''
message_excerpt.short_description = ugettext('Message excerpt')
def mark_as_valid(self, request, qs):
@ -389,8 +433,10 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
self.message_user(
request,
ugettext(
'The %d selected reports have been marked as valid.' % (
qs.count())))
'The %d selected reports have been marked as valid.' % (qs.count())
),
)
mark_as_valid.short_description = 'Mark selected abuse reports as valid'
def mark_as_suspicious(self, request, qs):
@ -399,10 +445,13 @@ class AbuseReportAdmin(CommaSearchInAdminMixin, admin.ModelAdmin):
self.message_user(
request,
ugettext(
'The %d selected reports have been marked as suspicious.' % (
qs.count())))
mark_as_suspicious.short_description = (
ugettext('Mark selected abuse reports as suspicious'))
'The %d selected reports have been marked as suspicious.' % (qs.count())
),
)
mark_as_suspicious.short_description = ugettext(
'Mark selected abuse reports as suspicious'
)
admin.site.register(AbuseReport, AbuseReportAdmin)

92
src/olympia/abuse/models.py
Просмотреть файл

@ -50,8 +50,11 @@ class AbuseReport(ModelBase):
REASONS = APIChoicesWithNone(
('DAMAGE', 1, 'Damages computer and/or data'),
('SPAM', 2, 'Creates spam or advertising'),
('SETTINGS', 3, 'Changes search / homepage / new tab page '
'without informing user'),
(
'SETTINGS',
3,
'Changes search / homepage / new tab page ' 'without informing user',
),
# `4` was previously 'New tab takeover' but has been merged into the
# previous one. We avoid re-using the value.
('BROKEN', 5, "Doesn’t work, breaks websites, or slows Firefox down"),
@ -128,73 +131,90 @@ class AbuseReport(ModelBase):
# NULL if the reporter is anonymous.
reporter = models.ForeignKey(
UserProfile, null=True, blank=True, related_name='abuse_reported',
on_delete=models.SET_NULL)
country_code = models.CharField(
max_length=2, default=None, null=True)
UserProfile,
null=True,
blank=True,
related_name='abuse_reported',
on_delete=models.SET_NULL,
)
country_code = models.CharField(max_length=2, default=None, null=True)
# An abuse report can be for an addon or a user.
# If user is non-null then both addon and guid should be null.
# If user is null then addon should be non-null if guid was in our DB,
# otherwise addon will be null also.
# If both addon and user is null guid should be set.
addon = models.ForeignKey(
Addon, null=True, related_name='abuse_reports',
on_delete=models.CASCADE)
Addon, null=True, related_name='abuse_reports', on_delete=models.CASCADE
)
guid = models.CharField(max_length=255, null=True)
user = models.ForeignKey(
UserProfile, null=True, related_name='abuse_reports',
on_delete=models.SET_NULL)
UserProfile, null=True, related_name='abuse_reports', on_delete=models.SET_NULL
)
message = models.TextField(blank=True)
state = models.PositiveSmallIntegerField(
default=STATES.UNTRIAGED, choices=STATES.choices)
default=STATES.UNTRIAGED, choices=STATES.choices
)
# Extra optional fields for more information, giving some context that is
# meant to be extracted automatically by the client (i.e. Firefox) and
# submitted via the API.
client_id = models.CharField(
default=None, max_length=64, blank=True, null=True)
addon_name = models.CharField(
default=None, max_length=255, blank=True, null=True)
client_id = models.CharField(default=None, max_length=64, blank=True, null=True)
addon_name = models.CharField(default=None, max_length=255, blank=True, null=True)
addon_summary = models.CharField(
default=None, max_length=255, blank=True, null=True)
default=None, max_length=255, blank=True, null=True
)
addon_version = models.CharField(
default=None, max_length=255, blank=True, null=True)
default=None, max_length=255, blank=True, null=True
)
addon_signature = models.PositiveSmallIntegerField(
default=None, choices=ADDON_SIGNATURES.choices, blank=True, null=True)
default=None, choices=ADDON_SIGNATURES.choices, blank=True, null=True
)
application = models.PositiveSmallIntegerField(
default=amo.FIREFOX.id, choices=amo.APPS_CHOICES, blank=True,
null=True)
default=amo.FIREFOX.id, choices=amo.APPS_CHOICES, blank=True, null=True
)
application_version = models.CharField(
default=None, max_length=255, blank=True, null=True)
default=None, max_length=255, blank=True, null=True
)
application_locale = models.CharField(
default=None, max_length=255, blank=True, null=True)
default=None, max_length=255, blank=True, null=True
)
operating_system = models.CharField(
default=None, max_length=255, blank=True, null=True)
default=None, max_length=255, blank=True, null=True
)
operating_system_version = models.CharField(
default=None, max_length=255, blank=True, null=True)
install_date = models.DateTimeField(
default=None, blank=True, null=True)
default=None, max_length=255, blank=True, null=True
)
install_date = models.DateTimeField(default=None, blank=True, null=True)
reason = models.PositiveSmallIntegerField(
default=None, choices=REASONS.choices, blank=True, null=True)
default=None, choices=REASONS.choices, blank=True, null=True
)
addon_install_origin = models.CharField(
# Supposed to be an URL, but the scheme could be moz-foo: or something
# like that, and it's potentially truncated, so use a CharField and not
# a URLField. We also don't want to automatically turn this into a
# clickable link in the admin in case it's dangerous.
default=None, max_length=255, blank=True, null=True)
default=None,
max_length=255,
blank=True,
null=True,
)
addon_install_method = models.PositiveSmallIntegerField(
default=None, choices=ADDON_INSTALL_METHODS.choices, blank=True,
null=True)
default=None, choices=ADDON_INSTALL_METHODS.choices, blank=True, null=True
)
addon_install_source = models.PositiveSmallIntegerField(
default=None, choices=ADDON_INSTALL_SOURCES.choices, blank=True,
null=True)
default=None, choices=ADDON_INSTALL_SOURCES.choices, blank=True, null=True
)
addon_install_source_url = models.CharField(
# See addon_install_origin above as for why it's not an URLField.
default=None, max_length=255, blank=True, null=True)
default=None,
max_length=255,
blank=True,
null=True,
)
report_entry_point = models.PositiveSmallIntegerField(
default=None, choices=REPORT_ENTRY_POINTS.choices, blank=True,
null=True)
default=None, choices=REPORT_ENTRY_POINTS.choices, blank=True, null=True
)
unfiltered = AbuseReportManager(include_deleted=True)
objects = AbuseReportManager()
@ -238,7 +258,7 @@ class AbuseReport(ModelBase):
def type(self):
with translation.override(settings.LANGUAGE_CODE):
if self.addon and self.addon.type in amo.ADDON_TYPE:
type_ = (translation.ugettext(amo.ADDON_TYPE[self.addon.type]))
type_ = translation.ugettext(amo.ADDON_TYPE[self.addon.type])
elif self.user:
type_ = 'User'
else:

93
src/olympia/abuse/serializers.py
Просмотреть файл

@ -23,15 +23,14 @@ class BaseAbuseReportSerializer(serializers.ModelSerializer):
def validate_target(self, data, target_name):
if target_name not in data:
msg = serializers.Field.default_error_messages['required']
raise serializers.ValidationError({
target_name: [msg]
})
raise serializers.ValidationError({target_name: [msg]})
def to_internal_value(self, data):
output = super(BaseAbuseReportSerializer, self).to_internal_value(data)
request = self.context['request']
output['country_code'] = AbuseReport.lookup_country_code_from_ip(
request.META.get('REMOTE_ADDR'))
request.META.get('REMOTE_ADDR')
)
if request.user.is_authenticated:
output['reporter'] = request.user
return output
@ -40,40 +39,54 @@ class BaseAbuseReportSerializer(serializers.ModelSerializer):
class AddonAbuseReportSerializer(BaseAbuseReportSerializer):
error_messages = {
'max_length': _(
'Please ensure this field has no more than {max_length} '
'characters.'
'Please ensure this field has no more than {max_length} ' 'characters.'
)
}
addon = serializers.SerializerMethodField()
reason = ReverseChoiceField(
choices=list(AbuseReport.REASONS.api_choices), required=False,
allow_null=True)
choices=list(AbuseReport.REASONS.api_choices), required=False, allow_null=True
)
# 'message' has custom validation rules below depending on whether 'reason'
# was provided or not. We need to not set it as required and allow blank at
# the field level to make that work.
message = serializers.CharField(
required=False, allow_blank=True, max_length=10000,
error_messages=error_messages)
required=False,
allow_blank=True,
max_length=10000,
error_messages=error_messages,
)
app = ReverseChoiceField(
choices=list((v.id, k) for k, v in amo.APPS.items()), required=False,
source='application')
choices=list((v.id, k) for k, v in amo.APPS.items()),
required=False,
source='application',
)
appversion = serializers.CharField(
required=False, source='application_version', max_length=255)
required=False, source='application_version', max_length=255
)
lang = serializers.CharField(
required=False, source='application_locale', max_length=255)
required=False, source='application_locale', max_length=255
)
report_entry_point = ReverseChoiceField(
choices=list(AbuseReport.REPORT_ENTRY_POINTS.api_choices),
required=False, allow_null=True)
required=False,
allow_null=True,
)
addon_install_method = ReverseChoiceField(
choices=list(AbuseReport.ADDON_INSTALL_METHODS.api_choices),
required=False, allow_null=True)
required=False,
allow_null=True,
)
addon_install_source = ReverseChoiceField(
choices=list(AbuseReport.ADDON_INSTALL_SOURCES.api_choices),
required=False, allow_null=True)
required=False,
allow_null=True,
)
addon_signature = ReverseChoiceField(
choices=list(AbuseReport.ADDON_SIGNATURES.api_choices),
required=False, allow_null=True)
required=False,
allow_null=True,
)
class Meta:
model = AbuseReport
@ -95,7 +108,7 @@ class AddonAbuseReportSerializer(BaseAbuseReportSerializer):
'operating_system',
'operating_system_version',
'reason',
'report_entry_point'
'report_entry_point',
)
def validate(self, data):
@ -110,17 +123,18 @@ class AddonAbuseReportSerializer(BaseAbuseReportSerializer):
msg = serializers.Field.default_error_messages['null']
else:
msg = serializers.CharField.default_error_messages['blank']
raise serializers.ValidationError({
'message': [msg]
})
raise serializers.ValidationError({'message': [msg]})
return data
def handle_unknown_install_method_or_source(self, data, field_name):
reversed_choices = self.fields[field_name].reversed_choices
value = data[field_name]
if value not in reversed_choices:
log.warning('Unknown abuse report %s value submitted: %s',
field_name, str(data[field_name])[:255])
log.warning(
'Unknown abuse report %s value submitted: %s',
field_name,
str(data[field_name])[:255],
)
value = 'other'
return value
@ -130,22 +144,20 @@ class AddonAbuseReportSerializer(BaseAbuseReportSerializer):
# do it in a custom validation method because validation would be
# skipped entirely if the value is not a valid choice.
if 'addon_install_method' in data:
data['addon_install_method'] = (
self.handle_unknown_install_method_or_source(
data, 'addon_install_method'))
data['addon_install_method'] = self.handle_unknown_install_method_or_source(
data, 'addon_install_method'
)
if 'addon_install_source' in data:
data['addon_install_source'] = (
self.handle_unknown_install_method_or_source(
data, 'addon_install_source'))
data['addon_install_source'] = self.handle_unknown_install_method_or_source(
data, 'addon_install_source'
)
self.validate_target(data, 'addon')
view = self.context.get('view')
output = view.get_guid_and_addon()
# Pop 'addon' from data before passing that data to super(), we already
# have it in the output value.
data.pop('addon')
output.update(
super(AddonAbuseReportSerializer, self).to_internal_value(data)
)
output.update(super(AddonAbuseReportSerializer, self).to_internal_value(data))
return output
def get_addon(self, obj):
@ -163,25 +175,18 @@ class UserAbuseReportSerializer(BaseAbuseReportSerializer):
user = BaseUserSerializer(required=False) # We validate it ourselves.
# Unlike add-on reports, for user reports we don't have a 'reason' field so
# the message is always required and can't be blank.
message = serializers.CharField(
required=True, allow_blank=False, max_length=10000)
message = serializers.CharField(required=True, allow_blank=False, max_length=10000)
class Meta:
model = AbuseReport
fields = BaseAbuseReportSerializer.Meta.fields + (
'user',
)
fields = BaseAbuseReportSerializer.Meta.fields + ('user',)
def to_internal_value(self, data):
view = self.context.get('view')
self.validate_target(data, 'user')
output = {
'user': view.get_user_object()
}
output = {'user': view.get_user_object()}
# Pop 'user' before passing it to super(), we already have the
# output value and did the validation above.
data.pop('user')
output.update(
super(UserAbuseReportSerializer, self).to_internal_value(data)
)
output.update(super(UserAbuseReportSerializer, self).to_internal_value(data))
return output

200
src/olympia/abuse/tests/test_admin.py
Просмотреть файл

@ -4,8 +4,7 @@ from urllib.parse import parse_qsl, urlparse
from django.conf import settings
from django.contrib import admin
from django.contrib.messages.storage import (
default_storage as default_messages_storage)
from django.contrib.messages.storage import default_storage as default_messages_storage
from django.test import RequestFactory
from django.urls import reverse
@ -16,7 +15,11 @@ from olympia.abuse.admin import AbuseReportAdmin
from olympia.abuse.models import AbuseReport
from olympia.addons.models import AddonApprovalsCounter
from olympia.amo.tests import (
addon_factory, days_ago, grant_permission, TestCase, user_factory
addon_factory,
days_ago,
grant_permission,
TestCase,
user_factory,
)
from olympia.ratings.models import Rating
from olympia.reviewers.models import AutoApprovalSummary
@ -24,37 +27,48 @@ from olympia.versions.models import VersionPreview
class TestAbuse(TestCase):
@classmethod
def setUpTestData(cls):
cls.addon1 = addon_factory(guid='@guid1', name='Neo')
cls.addon1.name.__class__.objects.create(
id=cls.addon1.name_id, locale='fr', localized_string='Elu')
id=cls.addon1.name_id, locale='fr', localized_string='Elu'
)
cls.addon2 = addon_factory(guid='@guid2', name='Two')
cls.addon3 = addon_factory(guid='@guid3', name='Three')
cls.user = user_factory(email='someone@mozilla.com')
grant_permission(cls.user, 'AbuseReports:Edit', 'Abuse Report Triage')
# Create a few abuse reports.
cls.report1 = AbuseReport.objects.create(
addon=cls.addon1, guid='@guid1', message='Foo',
addon=cls.addon1,
guid='@guid1',
message='Foo',
state=AbuseReport.STATES.VALID,
created=days_ago(98))
created=days_ago(98),
)
AbuseReport.objects.create(
addon=cls.addon2, guid='@guid2', message='Bar',
state=AbuseReport.STATES.VALID)
addon=cls.addon2,
guid='@guid2',
message='Bar',
state=AbuseReport.STATES.VALID,
)
AbuseReport.objects.create(
addon=cls.addon3, guid='@guid3', message='Soap',
addon=cls.addon3,
guid='@guid3',
message='Soap',
reason=AbuseReport.REASONS.OTHER,
created=days_ago(100))
created=days_ago(100),
)
AbuseReport.objects.create(
addon=cls.addon1, guid='@guid1', message='',
reporter=user_factory())
addon=cls.addon1, guid='@guid1', message='', reporter=user_factory()
)
# This is a report for an addon not in the database.
cls.report2 = AbuseReport.objects.create(
guid='@unknown_guid', addon_name='Mysterious Addon', message='Doo')
guid='@unknown_guid', addon_name='Mysterious Addon', message='Doo'
)
# This is one against a user.
cls.report3 = AbuseReport.objects.create(
user=user_factory(username='malicious_user'), message='Ehehehehe')
user=user_factory(username='malicious_user'), message='Ehehehehe'
)
def setUp(self):
self.client.login(email=self.user.email)
@ -83,15 +97,13 @@ class TestAbuse(TestCase):
assert doc('#result_list tbody tr').length == expected_length
def test_list_filter_by_type(self):
response = self.client.get(
self.list_url, {'type': 'addon'}, follow=True)
response = self.client.get(self.list_url, {'type': 'addon'}, follow=True)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#result_list tbody tr').length == 5
assert 'Ehehehehe' not in doc('#result_list').text()
response = self.client.get(
self.list_url, {'type': 'user'}, follow=True)
response = self.client.get(self.list_url, {'type': 'user'}, follow=True)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#result_list tbody tr').length == 1
@ -103,8 +115,7 @@ class TestAbuse(TestCase):
assert lis.text().split() == ['Users', 'All', 'All', 'All', 'All']
def test_search_deactivated_if_not_filtering_by_type(self):
response = self.client.get(
self.list_url, {'q': 'Mysterious'}, follow=True)
response = self.client.get(self.list_url, {'q': 'Mysterious'}, follow=True)
assert response.status_code == 200
doc = pq(response.content)
assert not doc('#changelist-search')
@ -112,7 +123,8 @@ class TestAbuse(TestCase):
def test_search_user(self):
response = self.client.get(
self.list_url, {'q': 'Ehe', 'type': 'user'}, follow=True)
self.list_url, {'q': 'Ehe', 'type': 'user'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -121,8 +133,8 @@ class TestAbuse(TestCase):
user = AbuseReport.objects.get(message='Ehehehehe').user
response = self.client.get(
self.list_url, {'q': user.username[:4], 'type': 'user'},
follow=True)
self.list_url, {'q': user.username[:4], 'type': 'user'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -131,8 +143,8 @@ class TestAbuse(TestCase):
user = AbuseReport.objects.get(message='Ehehehehe').user
response = self.client.get(
self.list_url, {'q': user.email[:3], 'type': 'user'},
follow=True)
self.list_url, {'q': user.email[:3], 'type': 'user'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -140,8 +152,8 @@ class TestAbuse(TestCase):
assert 'Ehehehehe' in doc('#result_list').text()
response = self.client.get(
self.list_url, {'q': str(user.pk), 'type': 'user'},
follow=True)
self.list_url, {'q': str(user.pk), 'type': 'user'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -149,8 +161,8 @@ class TestAbuse(TestCase):
assert 'Ehehehehe' in doc('#result_list').text()
response = self.client.get(
self.list_url, {'q': 'NotGoingToFindAnything', 'type': 'user'},
follow=True)
self.list_url, {'q': 'NotGoingToFindAnything', 'type': 'user'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -159,7 +171,8 @@ class TestAbuse(TestCase):
def test_search_addon(self):
response = self.client.get(
self.list_url, {'q': 'sterious', 'type': 'addon'}, follow=True)
self.list_url, {'q': 'sterious', 'type': 'addon'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -167,8 +180,8 @@ class TestAbuse(TestCase):
assert 'Mysterious' in doc('#result_list').text()
response = self.client.get(
self.list_url, {'q': str(self.addon1.pk), 'type': 'addon'},
follow=True)
self.list_url, {'q': str(self.addon1.pk), 'type': 'addon'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -176,8 +189,8 @@ class TestAbuse(TestCase):
assert 'Neo' in doc('#result_list').text()
response = self.client.get(
self.list_url, {'q': 'NotGoingToFindAnything', 'type': 'addon'},
follow=True)
self.list_url, {'q': 'NotGoingToFindAnything', 'type': 'addon'}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -189,7 +202,8 @@ class TestAbuse(TestCase):
response = self.client.get(
self.list_url,
{'q': '%s,%s' % (self.addon1.pk, self.addon2.pk), 'type': 'addon'},
follow=True)
follow=True,
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -199,13 +213,13 @@ class TestAbuse(TestCase):
def test_search_multiple_users(self):
user1 = AbuseReport.objects.get(message='Ehehehehe').user
user2 = user_factory(username='second_user')
AbuseReport.objects.create(
user=user2, message='One more')
AbuseReport.objects.create(user=user2, message='One more')
response = self.client.get(
self.list_url,
{'q': '%s,%s' % (user1.pk, user2.pk), 'type': 'user'},
follow=True)
follow=True,
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#changelist-search')
@ -213,8 +227,8 @@ class TestAbuse(TestCase):
def test_filter_by_state(self):
response = self.client.get(
self.list_url, {'state__exact': AbuseReport.STATES.VALID},
follow=True)
self.list_url, {'state__exact': AbuseReport.STATES.VALID}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#result_list tbody tr').length == 2
@ -229,8 +243,8 @@ class TestAbuse(TestCase):
def test_filter_by_reason(self):
response = self.client.get(
self.list_url, {'reason__exact': AbuseReport.REASONS.OTHER},
follow=True)
self.list_url, {'reason__exact': AbuseReport.REASONS.OTHER}, follow=True
)
assert response.status_code == 200
doc = pq(response.content)
assert doc('#result_list tbody tr').length == 1
@ -263,9 +277,7 @@ class TestAbuse(TestCase):
# (because of the "All" default choice) but since 'created' is actually
# 2 fields, and we have submitted both, we now have 6 expected items.
assert len(lis) == 6
assert lis.text().split() == [
'All', 'All', 'All', 'From:', 'To:', 'All'
]
assert lis.text().split() == ['All', 'All', 'All', 'From:', 'To:', 'All']
elm = lis.eq(3).find('#id_created__range__gte')
assert elm
assert elm.attr('name') == 'created__range__gte'
@ -292,9 +304,7 @@ class TestAbuse(TestCase):
lis = doc('#changelist-filter li.selected')
# We've got 5 filters.
assert len(lis) == 5
assert lis.text().split() == [
'All', 'All', 'All', 'From:', 'All'
]
assert lis.text().split() == ['All', 'All', 'All', 'From:', 'All']
elm = lis.eq(3).find('#id_created__range__gte')
assert elm
assert elm.attr('name') == 'created__range__gte'
@ -317,18 +327,14 @@ class TestAbuse(TestCase):
lis = doc('#changelist-filter li.selected')
# We've got 5 filters.
assert len(lis) == 5
assert lis.text().split() == [
'All', 'All', 'All', 'To:', 'All'
]
assert lis.text().split() == ['All', 'All', 'All', 'To:', 'All']
elm = lis.eq(3).find('#id_created__range__lte')
assert elm
assert elm.attr('name') == 'created__range__lte'
assert elm.attr('value') == some_time_ago.isoformat()
def test_filter_by_minimum_reports_count_for_guid(self):
data = {
'minimum_reports_count': '2'
}
data = {'minimum_reports_count': '2'}
response = self.client.get(self.list_url, data, follow=True)
assert response.status_code == 200
doc = pq(response.content)
@ -344,7 +350,10 @@ class TestAbuse(TestCase):
# There is no label for minimum reports count, so despite having 5 lis
# we only have 4 things in .text().
assert lis.text().split() == [
'All', 'All', 'All', 'All',
'All',
'All',
'All',
'All',
]
# The 4th item should contain the input though.
elm = lis.eq(4).find('#id_minimum_reports_count')
@ -372,14 +381,20 @@ class TestAbuse(TestCase):
# Also, the forms we used for the 'created' filters should contain all
# active filters and search query so that we can combine them.
forms = doc('#changelist-filter form')
inputs = [(elm.name, elm.value) for elm in forms.find('input')
if elm.name and elm.value != '']
inputs = [
(elm.name, elm.value)
for elm in forms.find('input')
if elm.name and elm.value != ''
]
assert set(inputs) == set(data.items())
# Same for the 'search' form
form = doc('#changelist-filter form')
inputs = [(elm.name, elm.value) for elm in form.find('input')
if elm.name and elm.value != '']
inputs = [
(elm.name, elm.value)
for elm in form.find('input')
if elm.name and elm.value != ''
]
assert set(inputs) == set(data.items())
# Gather selected filters.
@ -389,9 +404,7 @@ class TestAbuse(TestCase):
# (because of the "All" default choice) but since 'created' is actually
# 2 fields, and we have submitted both, we now have 6 expected items.
assert len(lis) == 6
assert lis.text().split() == [
'Addons', 'All', 'Other', 'From:', 'To:', 'All'
]
assert lis.text().split() == ['Addons', 'All', 'Other', 'From:', 'To:', 'All']
assert lis.eq(3).find('#id_created__range__gte')
assert lis.eq(4).find('#id_created__range__lte')
@ -412,18 +425,19 @@ class TestAbuse(TestCase):
# self.user has AbuseReports:Edit
request.user = self.user
assert list(
abuse_report_admin.get_actions(request).keys()) == [
'mark_as_valid', 'mark_as_suspicious'
assert list(abuse_report_admin.get_actions(request).keys()) == [
'mark_as_valid',
'mark_as_suspicious',
]
# Advanced admins can also delete.
request.user = user_factory()
self.grant_permission(request.user, 'AbuseReports:Edit')
self.grant_permission(request.user, 'Admin:Advanced')
assert list(
abuse_report_admin.get_actions(request).keys()) == [
'delete_selected', 'mark_as_valid', 'mark_as_suspicious'
assert list(abuse_report_admin.get_actions(request).keys()) == [
'delete_selected',
'mark_as_valid',
'mark_as_suspicious',
]
def test_action_mark_multiple_as_valid(self):
@ -431,16 +445,14 @@ class TestAbuse(TestCase):
request = RequestFactory().post('/')
request.user = self.user
request._messages = default_messages_storage(request)
reports = AbuseReport.objects.filter(
guid__in=('@guid3', '@unknown_guid'))
reports = AbuseReport.objects.filter(guid__in=('@guid3', '@unknown_guid'))
assert reports.count() == 2
for report in reports.all():
assert report.state == AbuseReport.STATES.UNTRIAGED
other_report = AbuseReport.objects.get(guid='@guid1', message='')
assert other_report.state == AbuseReport.STATES.UNTRIAGED
action_callback = abuse_report_admin.get_actions(
request)['mark_as_valid'][0]
action_callback = abuse_report_admin.get_actions(request)['mark_as_valid'][0]
rval = action_callback(abuse_report_admin, request, reports)
assert rval is None # successful actions return None
for report in reports.all():
@ -454,16 +466,16 @@ class TestAbuse(TestCase):
request = RequestFactory().post('/')
request.user = self.user
request._messages = default_messages_storage(request)
reports = AbuseReport.objects.filter(
guid__in=('@guid3', '@unknown_guid'))
reports = AbuseReport.objects.filter(guid__in=('@guid3', '@unknown_guid'))
assert reports.count() == 2
for report in reports.all():
assert report.state == AbuseReport.STATES.UNTRIAGED
other_report = AbuseReport.objects.get(guid='@guid1', message='')
assert other_report.state == AbuseReport.STATES.UNTRIAGED
action_callback = abuse_report_admin.get_actions(
request)['mark_as_suspicious'][0]
action_callback = abuse_report_admin.get_actions(request)['mark_as_suspicious'][
0
]
rval = action_callback(abuse_report_admin, request, reports)
assert rval is None # successful actions return None
for report in reports.all():
@ -479,14 +491,12 @@ class TestAbuse(TestCase):
self.grant_permission(request.user, 'AbuseReports:Edit')
self.grant_permission(request.user, 'Admin:Advanced')
request._messages = default_messages_storage(request)
reports = AbuseReport.objects.filter(
guid__in=('@guid3', '@unknown_guid'))
reports = AbuseReport.objects.filter(guid__in=('@guid3', '@unknown_guid'))
assert reports.count() == 2
assert AbuseReport.objects.count() == 6
assert AbuseReport.unfiltered.count() == 6
action_callback = abuse_report_admin.get_actions(
request)['delete_selected'][0]
action_callback = abuse_report_admin.get_actions(request)['delete_selected'][0]
rval = action_callback(abuse_report_admin, request, reports)
assert rval is None # successful actions return None
assert reports.count() == 0 # All should have been soft-deleted.
@ -495,14 +505,17 @@ class TestAbuse(TestCase):
def test_detail_addon_report(self):
AddonApprovalsCounter.objects.create(
addon=self.addon1, last_human_review=datetime.now())
addon=self.addon1, last_human_review=datetime.now()
)
Rating.objects.create(
addon=self.addon1, rating=2.0, body='Badd-on', user=user_factory())
addon=self.addon1, rating=2.0, body='Badd-on', user=user_factory()
)
AutoApprovalSummary.objects.create(
version=self.addon1.current_version,
verdict=amo.AUTO_APPROVED)
version=self.addon1.current_version, verdict=amo.AUTO_APPROVED
)
self.detail_url = reverse(
'admin:abuse_abusereport_change', args=(self.report1.pk,))
'admin:abuse_abusereport_change', args=(self.report1.pk,)
)
response = self.client.get(self.detail_url, follow=True)
assert response.status_code == 200
doc = pq(response.content)
@ -511,13 +524,12 @@ class TestAbuse(TestCase):
assert 'Neo' in doc('.addon-info-and-previews h2').text()
assert doc('.addon-info-and-previews .meta-abuse td').text() == '2'
assert doc('.addon-info-and-previews .meta-rating td').text() == (
'Rated 2 out of 5 stars 1 review')
'Rated 2 out of 5 stars 1 review'
)
assert doc('.addon-info-and-previews .last-approval-date td').text()
assert doc('.reports-and-ratings')
assert doc('.reports-and-ratings h3').eq(0).text() == (
'Abuse Reports (2)')
assert doc('.reports-and-ratings h3').eq(1).text() == (
'Bad User Ratings (1)')
assert doc('.reports-and-ratings h3').eq(0).text() == ('Abuse Reports (2)')
assert doc('.reports-and-ratings h3').eq(1).text() == ('Bad User Ratings (1)')
# 'addon-info-and-previews' and 'reports-and-ratings' are coming from a
# reviewer tools template and shouldn't contain any admin-specific
# links. It also means that all links in it should be external, in
@ -539,7 +551,8 @@ class TestAbuse(TestCase):
def test_detail_guid_report(self):
self.detail_url = reverse(
'admin:abuse_abusereport_change', args=(self.report2.pk,))
'admin:abuse_abusereport_change', args=(self.report2.pk,)
)
response = self.client.get(self.detail_url, follow=True)
assert response.status_code == 200
doc = pq(response.content)
@ -549,7 +562,8 @@ class TestAbuse(TestCase):
def test_detail_user_report(self):
self.detail_url = reverse(
'admin:abuse_abusereport_change', args=(self.report3.pk,))
'admin:abuse_abusereport_change', args=(self.report3.pk,)
)
response = self.client.get(self.detail_url, follow=True)
assert response.status_code == 200
doc = pq(response.content)

11
src/olympia/abuse/tests/test_models.py
Просмотреть файл

@ -45,8 +45,7 @@ class TestAbuse(TestCase):
(None, 'None'),
(1, 'Damages computer and/or data'),
(2, 'Creates spam or advertising'),
(3, 'Changes search / homepage / new tab page without informing '
'user'),
(3, 'Changes search / homepage / new tab page without informing ' 'user'),
(5, "Doesn’t work, breaks websites, or slows Firefox down"),
(6, 'Hateful, violent, or illegal content'),
(7, "Pretends to be something it’s not"),
@ -81,7 +80,7 @@ class TestAbuse(TestCase):
(12, 'Temporary Add-on'),
(13, 'Sync'),
(14, 'URL'),
(127, 'Other')
(127, 'Other'),
)
assert AbuseReport.ADDON_INSTALL_METHODS.api_choices == (
@ -100,7 +99,7 @@ class TestAbuse(TestCase):
(12, 'temporary_addon'),
(13, 'sync'),
(14, 'url'),
(127, 'other')
(127, 'other'),
)
assert AbuseReport.ADDON_INSTALL_SOURCES.choices == (
@ -123,7 +122,7 @@ class TestAbuse(TestCase):
(16, 'System Add-on'),
(17, 'Temporary Add-on'),
(18, 'Unknown'),
(127, 'Other')
(127, 'Other'),
)
assert AbuseReport.ADDON_INSTALL_SOURCES.api_choices == (
@ -146,7 +145,7 @@ class TestAbuse(TestCase):
(16, 'system_addon'),
(17, 'temporary_addon'),
(18, 'unknown'),
(127, 'other')
(127, 'other'),
)
assert AbuseReport.REPORT_ENTRY_POINTS.choices == (

23
src/olympia/abuse/tests/test_serializers.py
Просмотреть файл

@ -9,13 +9,14 @@ from django.test.client import RequestFactory
from olympia import amo
from olympia.abuse.models import AbuseReport
from olympia.abuse.serializers import (
AddonAbuseReportSerializer, UserAbuseReportSerializer)
AddonAbuseReportSerializer,
UserAbuseReportSerializer,
)
from olympia.accounts.serializers import BaseUserSerializer
from olympia.amo.tests import TestCase, addon_factory, user_factory
class TestAddonAbuseReportSerializer(TestCase):
def serialize(self, report, **extra_context):
return AddonAbuseReportSerializer(report, context=extra_context).data
@ -25,11 +26,7 @@ class TestAddonAbuseReportSerializer(TestCase):
serialized = self.serialize(report)
assert serialized == {
'reporter': None,
'addon': {
'guid': addon.guid,
'id': addon.id,
'slug': addon.slug
},
'addon': {'guid': addon.guid, 'id': addon.id, 'slug': addon.slug},
'message': 'bad stuff',
'addon_install_method': None,
'addon_install_origin': None,
@ -55,11 +52,7 @@ class TestAddonAbuseReportSerializer(TestCase):
serialized = self.serialize(report)
assert serialized == {
'reporter': None,
'addon': {
'guid': '@guid',
'id': None,
'slug': None
},
'addon': {'guid': '@guid', 'id': None, 'slug': None},
'message': 'bad stuff',
'addon_install_method': None,
'addon_install_origin': None,
@ -115,7 +108,8 @@ class TestAddonAbuseReportSerializer(TestCase):
'report_entry_point': 'uninstall',
}
result = AddonAbuseReportSerializer(
data, context=extra_context).to_internal_value(data)
data, context=extra_context
).to_internal_value(data)
expected = {
'addon': None,
'addon_install_method': AbuseReport.ADDON_INSTALL_METHODS.URL,
@ -143,7 +137,6 @@ class TestAddonAbuseReportSerializer(TestCase):
class TestUserAbuseReportSerializer(TestCase):
def serialize(self, report, **extra_context):
return UserAbuseReportSerializer(report, context=extra_context).data
@ -155,5 +148,5 @@ class TestUserAbuseReportSerializer(TestCase):
assert serialized == {
'reporter': None,
'user': serialized_user,
'message': 'bad stuff'
'message': 'bad stuff',
}

198
src/olympia/abuse/tests/test_views.py
Просмотреть файл

@ -7,7 +7,12 @@ from unittest import mock
from olympia import amo
from olympia.abuse.models import AbuseReport
from olympia.amo.tests import (
APITestClient, TestCase, addon_factory, reverse_ns, user_factory)
APITestClient,
TestCase,
addon_factory,
reverse_ns,
user_factory,
)
class AddonAbuseViewSetTestBase(object):
@ -33,14 +38,14 @@ class AddonAbuseViewSetTestBase(object):
response = self.client.post(
self.url,
data={'addon': str(addon.id), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(addon_id=addon.id).exists()
report = AbuseReport.objects.get(addon_id=addon.id)
assert report.guid == addon.guid
self.check_report(report,
u'[Extension] Abuse Report for %s' % addon.name)
self.check_report(report, u'[Extension] Abuse Report for %s' % addon.name)
assert report.message == 'abuse!'
def test_report_addon_by_slug(self):
@ -48,28 +53,28 @@ class AddonAbuseViewSetTestBase(object):
response = self.client.post(
self.url,
data={'addon': addon.slug, 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(addon_id=addon.id).exists()
report = AbuseReport.objects.get(addon_id=addon.id)
assert report.guid == addon.guid
self.check_report(report,
u'[Extension] Abuse Report for %s' % addon.name)
self.check_report(report, u'[Extension] Abuse Report for %s' % addon.name)
def test_report_addon_by_guid(self):
addon = addon_factory(guid='@badman')
response = self.client.post(
self.url,
data={'addon': addon.guid, 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(addon_id=addon.id).exists()
report = AbuseReport.objects.get(addon_id=addon.id)
assert report.guid == addon.guid
self.check_report(report,
u'[Extension] Abuse Report for %s' % addon.name)
self.check_report(report, u'[Extension] Abuse Report for %s' % addon.name)
assert report.message == 'abuse!'
def test_report_addon_guid_not_on_amo(self):
@ -77,20 +82,20 @@ class AddonAbuseViewSetTestBase(object):
response = self.client.post(
self.url,
data={'addon': guid, 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(guid=guid).exists()
report = AbuseReport.objects.get(guid=guid)
assert not report.addon
self.check_report(report,
u'[Addon] Abuse Report for %s' % guid)
self.check_report(report, u'[Addon] Abuse Report for %s' % guid)
assert report.message == 'abuse!'
def test_report_addon_invalid_identifier(self):
response = self.client.post(
self.url,
data={'addon': 'randomnotguid', 'message': 'abuse!'})
self.url, data={'addon': 'randomnotguid', 'message': 'abuse!'}
)
assert response.status_code == 404
def test_addon_not_public(self):
@ -98,89 +103,78 @@ class AddonAbuseViewSetTestBase(object):
response = self.client.post(
self.url,
data={'addon': str(addon.id), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(addon_id=addon.id).exists()
report = AbuseReport.objects.get(addon_id=addon.id)
self.check_report(report,
u'[Extension] Abuse Report for %s' % addon.name)
self.check_report(report, u'[Extension] Abuse Report for %s' % addon.name)
assert report.message == 'abuse!'
def test_no_addon_fails(self):
response = self.client.post(
self.url,
data={'message': 'abuse!'})
response = self.client.post(self.url, data={'message': 'abuse!'})
assert response.status_code == 400
assert json.loads(response.content) == {
'addon': ['This field is required.']}
assert json.loads(response.content) == {'addon': ['This field is required.']}
def test_message_required_empty(self):
addon = addon_factory()
response = self.client.post(
self.url,
data={'addon': str(addon.id),
'message': ''})
self.url, data={'addon': str(addon.id), 'message': ''}
)
assert response.status_code == 400
assert json.loads(response.content) == {
'message': ['This field may not be blank.']}
'message': ['This field may not be blank.']
}
def test_message_required_missing(self):
addon = addon_factory()
response = self.client.post(
self.url,
data={'addon': str(addon.id)})
response = self.client.post(self.url, data={'addon': str(addon.id)})
assert response.status_code == 400
assert json.loads(response.content) == {
'message': ['This field is required.']}
assert json.loads(response.content) == {'message': ['This field is required.']}
def test_message_not_required_if_reason_is_provided(self):
addon = addon_factory()
response = self.client.post(
self.url,
data={'addon': str(addon.id), 'reason': 'broken'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(addon_id=addon.id).exists()
report = AbuseReport.objects.get(addon_id=addon.id)
self.check_report(report,
u'[Extension] Abuse Report for %s' % addon.name)
self.check_report(report, u'[Extension] Abuse Report for %s' % addon.name)
assert report.message == ''
def test_message_can_be_blank_if_reason_is_provided(self):
addon = addon_factory()
response = self.client.post(
self.url,
data={'addon': str(addon.id), 'reason': 'broken',
'message': ''},
REMOTE_ADDR='123.45.67.89')
data={'addon': str(addon.id), 'reason': 'broken', 'message': ''},
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(addon_id=addon.id).exists()
report = AbuseReport.objects.get(addon_id=addon.id)
self.check_report(report,
u'[Extension] Abuse Report for %s' % addon.name)
self.check_report(report, u'[Extension] Abuse Report for %s' % addon.name)
assert report.message == ''
def test_message_length_limited(self):
addon = addon_factory()
response = self.client.post(
self.url,
data={'addon': str(addon.id),
'message': 'a' * 10000})
self.url, data={'addon': str(addon.id), 'message': 'a' * 10000}
)
assert response.status_code == 201
response = self.client.post(
self.url,
data={'addon': str(addon.id),
'message': 'a' * 10001})
self.url, data={'addon': str(addon.id), 'message': 'a' * 10001}
)
assert response.status_code == 400
assert json.loads(response.content) == {
'message': [
'Please ensure this field has no more than 10000 characters.'
]
'message': ['Please ensure this field has no more than 10000 characters.']
}
def test_throttle(self):
@ -189,13 +183,15 @@ class AddonAbuseViewSetTestBase(object):
response = self.client.post(
self.url,
data={'addon': str(addon.id), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201, x
response = self.client.post(
self.url,
data={'addon': str(addon.id), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 429
def test_optional_fields(self):
@ -217,21 +213,23 @@ class AddonAbuseViewSetTestBase(object):
'addon_install_method': 'url',
'report_entry_point': None,
}
response = self.client.post(
self.url,
data=data,
REMOTE_ADDR='123.45.67.89')
response = self.client.post(self.url, data=data, REMOTE_ADDR='123.45.67.89')
assert response.status_code == 201, response.content
assert AbuseReport.objects.filter(guid=data['addon']).exists()
report = AbuseReport.objects.get(guid=data['addon'])
self.check_report(
report, u'[Addon] Abuse Report for %s' % data['addon'])
self.check_report(report, u'[Addon] Abuse Report for %s' % data['addon'])
assert not report.addon # Not an add-on in database, that's ok.
# Straightforward comparisons:
for field in ('message', 'client_id', 'addon_name', 'addon_summary',
'addon_version', 'operating_system',
'addon_install_origin'):
for field in (
'message',
'client_id',
'addon_name',
'addon_summary',
'addon_version',
'operating_system',
'addon_install_origin',
):
assert getattr(report, field) == data[field], field
# More complex comparisons:
assert report.addon_signature is None
@ -240,8 +238,7 @@ class AddonAbuseViewSetTestBase(object):
assert report.application_locale == data['lang']
assert report.install_date == datetime(2004, 8, 15, 16, 23, 42)
assert report.reason == 2 # Spam / Advertising
assert report.addon_install_method == (
AbuseReport.ADDON_INSTALL_METHODS.URL)
assert report.addon_install_method == (AbuseReport.ADDON_INSTALL_METHODS.URL)
assert report.addon_install_source is None
assert report.addon_install_source_url is None
assert report.report_entry_point is None
@ -267,33 +264,32 @@ class AddonAbuseViewSetTestBase(object):
'addon_install_source_url': 'http://%s' % 'a' * 249,
'report_entry_point': 'Something not in entrypoint choices',
}
response = self.client.post(
self.url,
data=data,
REMOTE_ADDR='123.45.67.89')
response = self.client.post(self.url, data=data, REMOTE_ADDR='123.45.67.89')
assert response.status_code == 400
expected_max_length_message = (
'Ensure this field has no more than %d characters.')
'Ensure this field has no more than %d characters.'
)
expected_choices_message = '"%s" is not a valid choice.'
assert response.json() == {
'client_id': [expected_max_length_message % 64],
'addon_name': [expected_max_length_message % 255],
'addon_summary': [expected_max_length_message % 255],
'addon_version': [expected_max_length_message % 255],
'addon_signature': [
expected_choices_message % data['addon_signature']],
'addon_signature': [expected_choices_message % data['addon_signature']],
'app': [expected_choices_message % data['app']],
'appversion': [expected_max_length_message % 255],
'lang': [expected_max_length_message % 255],
'operating_system': [expected_max_length_message % 255],
'install_date': [
'Datetime has wrong format. Use one of these formats '
'instead: YYYY-MM-DDThh:mm[:ss[.uuuuuu]][+HH:MM|-HH:MM|Z].'],
'instead: YYYY-MM-DDThh:mm[:ss[.uuuuuu]][+HH:MM|-HH:MM|Z].'
],
'reason': [expected_choices_message % data['reason']],
'addon_install_origin': [expected_max_length_message % 255],
'addon_install_source_url': [expected_max_length_message % 255],
'report_entry_point': [
expected_choices_message % data['report_entry_point']],
expected_choices_message % data['report_entry_point']
],
}
# Note: addon_install_method and addon_install_source silently convert
# unknown values to "other", so the values submitted here, despite not
@ -312,13 +308,10 @@ class AddonAbuseViewSetTestBase(object):
assert AbuseReport.objects.filter(guid=data['addon']).exists()
report = AbuseReport.objects.get(guid=data['addon'])
self.check_report(
report, u'[Addon] Abuse Report for %s' % data['addon'])
self.check_report(report, u'[Addon] Abuse Report for %s' % data['addon'])
assert not report.addon # Not an add-on in database, that's ok.
assert report.addon_install_method == (
AbuseReport.ADDON_INSTALL_METHODS.OTHER)
assert report.addon_install_source == (
AbuseReport.ADDON_INSTALL_SOURCES.OTHER)
assert report.addon_install_method == (AbuseReport.ADDON_INSTALL_METHODS.OTHER)
assert report.addon_install_source == (AbuseReport.ADDON_INSTALL_SOURCES.OTHER)
def test_addon_unknown_install_source_and_method_not_string(self):
addon = addon_factory()
@ -333,13 +326,10 @@ class AddonAbuseViewSetTestBase(object):
assert AbuseReport.objects.filter(guid=addon.guid).exists()
report = AbuseReport.objects.get(addon=addon)
self.check_report(
report, u'[Extension] Abuse Report for %s' % addon.name)
self.check_report(report, u'[Extension] Abuse Report for %s' % addon.name)
assert report.addon == addon
assert report.addon_install_method == (
AbuseReport.ADDON_INSTALL_METHODS.OTHER)
assert report.addon_install_source == (
AbuseReport.ADDON_INSTALL_SOURCES.OTHER)
assert report.addon_install_method == (AbuseReport.ADDON_INSTALL_METHODS.OTHER)
assert report.addon_install_source == (AbuseReport.ADDON_INSTALL_SOURCES.OTHER)
class TestAddonAbuseViewSetLoggedOut(AddonAbuseViewSetTestBase, TestCase):
@ -380,67 +370,63 @@ class UserAbuseViewSetTestBase(object):
response = self.client.post(
self.url,
data={'user': str(user.id), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(user_id=user.id).exists()
report = AbuseReport.objects.get(user_id=user.id)
self.check_report(report,
u'[User] Abuse Report for %s' % user.name)
self.check_report(report, u'[User] Abuse Report for %s' % user.name)
def test_report_user_username(self):
user = user_factory()
response = self.client.post(
self.url,
data={'user': str(user.username), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201
assert AbuseReport.objects.filter(user_id=user.id).exists()
report = AbuseReport.objects.get(user_id=user.id)
self.check_report(report,
u'[User] Abuse Report for %s' % user.name)
self.check_report(report, u'[User] Abuse Report for %s' % user.name)
def test_no_user_fails(self):
response = self.client.post(
self.url,
data={'message': 'abuse!'})
response = self.client.post(self.url, data={'message': 'abuse!'})
assert response.status_code == 400
assert json.loads(response.content) == {
'user': ['This field is required.']}
assert json.loads(response.content) == {'user': ['This field is required.']}
def test_message_required_empty(self):
user = user_factory()
response = self.client.post(
self.url,
data={'user': str(user.username), 'message': ''})
self.url, data={'user': str(user.username), 'message': ''}
)
assert response.status_code == 400
assert json.loads(response.content) == {
'message': ['This field may not be blank.']}
'message': ['This field may not be blank.']
}
def test_message_required_missing(self):
user = user_factory()
response = self.client.post(
self.url,
data={'user': str(user.username)})
response = self.client.post(self.url, data={'user': str(user.username)})
assert response.status_code == 400
assert json.loads(response.content) == {
'message': ['This field is required.']}
assert json.loads(response.content) == {'message': ['This field is required.']}
def test_throttle(self):
user = user_factory()
for x in range(20):
response = self.client.post(
self.url,
data={'user': str(
user.username), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
data={'user': str(user.username), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 201, x
response = self.client.post(
self.url,
data={'user': str(user.username), 'message': 'abuse!'},
REMOTE_ADDR='123.45.67.89')
REMOTE_ADDR='123.45.67.89',
)
assert response.status_code == 429

6
src/olympia/abuse/urls.py
Просмотреть файл

@ -5,10 +5,8 @@ from .views import AddonAbuseViewSet, UserAbuseViewSet
reporting = SimpleRouter()
reporting.register(r'addon', AddonAbuseViewSet,
basename='abusereportaddon')
reporting.register(r'user', UserAbuseViewSet,
basename='abusereportuser')
reporting.register(r'addon', AddonAbuseViewSet, basename='abusereportaddon')
reporting.register(r'user', UserAbuseViewSet, basename='abusereportuser')
urlpatterns = [
re_path(r'report/', include(reporting.urls)),

31
src/olympia/abuse/views.py
Просмотреть файл

@ -4,7 +4,9 @@ from rest_framework.mixins import CreateModelMixin
from rest_framework.viewsets import GenericViewSet
from olympia.abuse.serializers import (
AddonAbuseReportSerializer, UserAbuseReportSerializer)
AddonAbuseReportSerializer,
UserAbuseReportSerializer,
)
from olympia.accounts.views import AccountViewSet
from olympia.addons.views import AddonViewSet
from olympia.api.throttling import GranularUserRateThrottle
@ -25,13 +27,15 @@ class AddonAbuseViewSet(CreateModelMixin, GenericViewSet):
return self.addon_viewset
if 'addon_pk' not in self.kwargs:
self.kwargs['addon_pk'] = (
self.request.data.get('addon') or
self.request.GET.get('addon'))
self.kwargs['addon_pk'] = self.request.data.get(
'addon'
) or self.request.GET.get('addon')
self.addon_viewset = AddonViewSet(
request=self.request, permission_classes=[],
request=self.request,
permission_classes=[],
kwargs={'pk': self.kwargs['addon_pk']},
action='retrieve_from_related')
action='retrieve_from_related',
)
return self.addon_viewset
def get_addon_object(self):
@ -48,8 +52,7 @@ class AddonAbuseViewSet(CreateModelMixin, GenericViewSet):
}
# See if the addon input is guid-like first. It doesn't have to exist
# in our database.
if self.get_addon_viewset().get_lookup_field(
self.kwargs['addon_pk']) == 'guid':
if self.get_addon_viewset().get_lookup_field(self.kwargs['addon_pk']) == 'guid':
data['guid'] = self.kwargs['addon_pk']
try:
# But see if it's also in our database.
@ -79,10 +82,12 @@ class UserAbuseViewSet(CreateModelMixin, GenericViewSet):
return self.user_object
if 'user_pk' not in self.kwargs:
self.kwargs['user_pk'] = (
self.request.data.get('user') or
self.request.GET.get('user'))
self.kwargs['user_pk'] = self.request.data.get(
'user'
) or self.request.GET.get('user')
return AccountViewSet(
request=self.request, permission_classes=[],
kwargs={'pk': self.kwargs['user_pk']}).get_object()
request=self.request,
permission_classes=[],
kwargs={'pk': self.kwargs['user_pk']},
).get_object()

71
src/olympia/access/acl.py
Просмотреть файл

@ -42,7 +42,8 @@ def action_allowed_user(user, permission):
assert permission in amo.permissions.PERMISSIONS_LIST # constants only.
return any(
match_rules(group.rules, permission.app, permission.action)
for group in user.groups_list)
for group in user.groups_list
)
def experiments_submission_allowed(user, parsed_addon_data):
@ -51,9 +52,9 @@ def experiments_submission_allowed(user, parsed_addon_data):
See bug 1220097.
"""
return (
not parsed_addon_data.get('is_experiment', False) or
action_allowed_user(user, amo.permissions.EXPERIMENTS_SUBMIT))
return not parsed_addon_data.get('is_experiment', False) or action_allowed_user(
user, amo.permissions.EXPERIMENTS_SUBMIT
)
def langpack_submission_allowed(user, parsed_addon_data):
@ -63,9 +64,9 @@ def langpack_submission_allowed(user, parsed_addon_data):
See https://github.com/mozilla/addons-server/issues/11788 and
https://github.com/mozilla/addons-server/issues/11793
"""
return (
not parsed_addon_data.get('type') == amo.ADDON_LPAPP or
action_allowed_user(user, amo.permissions.LANGPACK_SUBMIT))
return not parsed_addon_data.get('type') == amo.ADDON_LPAPP or action_allowed_user(
user, amo.permissions.LANGPACK_SUBMIT
)
def system_addon_submission_allowed(user, parsed_addon_data):
@ -73,31 +74,40 @@ def system_addon_submission_allowed(user, parsed_addon_data):
by people with the right permission.
"""
guid = parsed_addon_data.get('guid') or ''
return (
not guid.lower().endswith(amo.SYSTEM_ADDON_GUIDS) or
action_allowed_user(user, amo.permissions.SYSTEM_ADDON_SUBMIT))
return not guid.lower().endswith(amo.SYSTEM_ADDON_GUIDS) or action_allowed_user(
user, amo.permissions.SYSTEM_ADDON_SUBMIT
)
def mozilla_signed_extension_submission_allowed(user, parsed_addon_data):
"""Add-ons already signed with mozilla internal certificate can only be
submitted by people with the right permission.
"""
return (
not parsed_addon_data.get('is_mozilla_signed_extension') or
action_allowed_user(user, amo.permissions.SYSTEM_ADDON_SUBMIT))
return not parsed_addon_data.get(
'is_mozilla_signed_extension'
) or action_allowed_user(user, amo.permissions.SYSTEM_ADDON_SUBMIT)
def check_ownership(request, obj, require_owner=False, require_author=False,
ignore_disabled=False, admin=True):
def check_ownership(
request,
obj,
require_owner=False,
require_author=False,
ignore_disabled=False,
admin=True,
):
"""
A convenience function. Check if request.user has permissions
for the object.
"""
if hasattr(obj, 'check_ownership'):
return obj.check_ownership(request, require_owner=require_owner,
require_author=require_author,
ignore_disabled=ignore_disabled,
admin=admin)
return obj.check_ownership(
request,
require_owner=require_owner,
require_author=require_author,
ignore_disabled=ignore_disabled,
admin=admin,
)
return False
@ -108,19 +118,21 @@ def check_collection_ownership(request, collection, require_owner=False):
if request.user.id == collection.author_id:
return True
elif collection.author_id == settings.TASK_USER_ID and action_allowed_user(
request.user, amo.permissions.ADMIN_CURATION):
request.user, amo.permissions.ADMIN_CURATION
):
return True
elif not require_owner:
return (
collection.pk == settings.COLLECTION_FEATURED_THEMES_ID and
action_allowed_user(
request.user, amo.permissions.COLLECTIONS_CONTRIBUTE))
collection.pk == settings.COLLECTION_FEATURED_THEMES_ID
and action_allowed_user(
request.user, amo.permissions.COLLECTIONS_CONTRIBUTE
)
)
else:
return False
def check_addon_ownership(request, addon, dev=False, admin=True,
ignore_disabled=False):
def check_addon_ownership(request, addon, dev=False, admin=True, ignore_disabled=False):
"""
Check request.user's permissions for the addon.
@ -145,15 +157,13 @@ def check_addon_ownership(request, addon, dev=False, admin=True,
if dev:
roles += (amo.AUTHOR_ROLE_DEV,)
return addon.addonuser_set.filter(
user=request.user, role__in=roles
).exists()
return addon.addonuser_set.filter(user=request.user, role__in=roles).exists()
def check_addons_reviewer(request, allow_content_reviewers=True):
permissions = [
amo.permissions.ADDONS_REVIEW,
amo.permissions.ADDONS_RECOMMENDED_REVIEW
amo.permissions.ADDONS_RECOMMENDED_REVIEW,
]
if allow_content_reviewers:
permissions.append(amo.permissions.ADDONS_CONTENT_REVIEW)
@ -180,7 +190,8 @@ def is_reviewer(request, addon, allow_content_reviewers=True):
if addon.type == amo.ADDON_STATICTHEME:
return check_static_theme_reviewer(request)
return check_addons_reviewer(
request, allow_content_reviewers=allow_content_reviewers)
request, allow_content_reviewers=allow_content_reviewers
)
def is_user_any_kind_of_reviewer(user, allow_viewers=False):

4
src/olympia/access/admin.py
Просмотреть файл

@ -14,9 +14,11 @@ class GroupUserInline(admin.TabularInline):
if obj.pk:
return format_html(
'<a href="{}">Admin User Profile</a>',
reverse('admin:users_userprofile_change', args=(obj.user.pk,)))
reverse('admin:users_userprofile_change', args=(obj.user.pk,)),
)
else:
return ''
user_profile_link.short_description = 'User Profile'

1
src/olympia/access/middleware.py
Просмотреть файл

@ -15,7 +15,6 @@ log = olympia.core.logger.getLogger('z.access')
class UserAndAddrMiddleware(MiddlewareMixin):
def process_request(self, request):
"""Attach authentication/permission helpers to request, and persist
user and remote addr in current thread."""

22
src/olympia/access/models.py
Просмотреть файл

@ -18,7 +18,8 @@ class Group(ModelBase):
name = models.CharField(max_length=255, default='')
rules = models.TextField()
users = models.ManyToManyField(
'users.UserProfile', through='GroupUser', related_name='groups')
'users.UserProfile', through='GroupUser', related_name='groups'
)
notes = models.TextField(blank=True)
class Meta:
@ -36,8 +37,7 @@ class GroupUser(models.Model):
class Meta:
db_table = 'groups_users'
constraints = [
models.UniqueConstraint(fields=('group', 'user'),
name='group_id'),
models.UniqueConstraint(fields=('group', 'user'), name='group_id'),
]
def invalidate_groups_list(self):
@ -52,25 +52,25 @@ class GroupUser(models.Model):
pass
@dispatch.receiver(signals.post_save, sender=GroupUser,
dispatch_uid='groupuser.post_save')
@dispatch.receiver(
signals.post_save, sender=GroupUser, dispatch_uid='groupuser.post_save'
)
def groupuser_post_save(sender, instance, **kw):
if kw.get('raw'):
return
activity.log_create(amo.LOG.GROUP_USER_ADDED, instance.group,
instance.user)
activity.log_create(amo.LOG.GROUP_USER_ADDED, instance.group, instance.user)
log.info('Added %s to %s' % (instance.user, instance.group))
instance.invalidate_groups_list()
@dispatch.receiver(signals.post_delete, sender=GroupUser,
dispatch_uid='groupuser.post_delete')
@dispatch.receiver(
signals.post_delete, sender=GroupUser, dispatch_uid='groupuser.post_delete'
)
def groupuser_post_delete(sender, instance, **kw):
if kw.get('raw'):
return
activity.log_create(amo.LOG.GROUP_USER_REMOVED, instance.group,
instance.user)
activity.log_create(amo.LOG.GROUP_USER_REMOVED, instance.group, instance.user)
log.info('Removed %s from %s' % (instance.user, instance.group))
instance.invalidate_groups_list()

16
src/olympia/access/templatetags/jinja_helpers.py
Просмотреть файл

@ -7,12 +7,16 @@ from .. import acl
@library.global_function
@jinja2.contextfunction
def check_ownership(context, obj, require_owner=False,
require_author=False, ignore_disabled=True):
return acl.check_ownership(context['request'], obj,
require_owner=require_owner,
require_author=require_author,
ignore_disabled=ignore_disabled)
def check_ownership(
context, obj, require_owner=False, require_author=False, ignore_disabled=True
):
return acl.check_ownership(
context['request'],
obj,
require_owner=require_owner,
require_author=require_author,
ignore_disabled=ignore_disabled,
)
@library.global_function

78
src/olympia/access/tests.py
Просмотреть файл

@ -6,16 +6,21 @@ from django.contrib.auth.models import AnonymousUser
from olympia import amo
from olympia.access.models import Group, GroupUser
from olympia.addons.models import Addon, AddonUser
from olympia.amo.tests import (
addon_factory, TestCase, req_factory_factory, user_factory)
from olympia.amo.tests import addon_factory, TestCase, req_factory_factory, user_factory
from olympia.users.models import UserProfile
from .acl import (
action_allowed, check_addon_ownership, check_addons_reviewer,
check_ownership, check_static_theme_reviewer,
action_allowed,
check_addon_ownership,
check_addons_reviewer,
check_ownership,
check_static_theme_reviewer,
check_unlisted_addons_reviewer,
is_reviewer, is_user_any_kind_of_reviewer, match_rules,
system_addon_submission_allowed)
is_reviewer,
is_user_any_kind_of_reviewer,
match_rules,
system_addon_submission_allowed,
)
pytestmark = pytest.mark.django_db
@ -58,8 +63,9 @@ def test_match_rules():
)
for rule in rules:
assert not match_rules(rule, 'Admin', '%'), \
assert not match_rules(rule, 'Admin', '%'), (
"%s == Admin:%% and shouldn't" % rule
)
def test_anonymous_user():
@ -69,13 +75,15 @@ def test_anonymous_user():
class ACLTestCase(TestCase):
"""Test some basic ACLs by going to various locked pages on AMO."""
fixtures = ['access/login.json']
def test_admin_login_anon(self):
# Login form for anonymous user on the admin page.
url = '/en-US/admin/models/'
self.assert3xx(self.client.get(url),
'/admin/models/login/?next=/en-US/admin/models/')
self.assert3xx(
self.client.get(url), '/admin/models/login/?next=/en-US/admin/models/'
)
class TestHasPerm(TestCase):
@ -117,8 +125,7 @@ class TestHasPerm(TestCase):
self.request = self.fake_request_with_user(self.login_admin())
assert check_ownership(self.request, self.addon, require_author=False)
assert not check_ownership(self.request, self.addon,
require_author=True)
assert not check_ownership(self.request, self.addon, require_author=True)
def test_disabled(self):
self.addon.update(status=amo.STATUS_DISABLED)
@ -133,8 +140,7 @@ class TestHasPerm(TestCase):
def test_ignore_disabled(self):
self.addon.update(status=amo.STATUS_DISABLED)
assert check_addon_ownership(self.request, self.addon,
ignore_disabled=True)
assert check_addon_ownership(self.request, self.addon, ignore_disabled=True)
def test_owner(self):
assert check_addon_ownership(self.request, self.addon)
@ -172,9 +178,7 @@ class TestHasPerm(TestCase):
# At this point, `user_dev` is an owner of `addon_for_user_dev`.
# Let's add `user_dev` as a developer of `self.addon`.
self.addon.addonuser_set.create(
user=user_dev, role=amo.AUTHOR_ROLE_DEV
)
self.addon.addonuser_set.create(user=user_dev, role=amo.AUTHOR_ROLE_DEV)
# Now, let's make sure `user_dev` is not an owner.
self.request = self.fake_request_with_user(user_dev)
@ -199,8 +203,7 @@ class TestCheckReviewer(TestCase):
assert not check_static_theme_reviewer(request)
assert not is_user_any_kind_of_reviewer(request.user)
assert not is_reviewer(request, self.addon)
assert not is_reviewer(
request, self.addon, allow_content_reviewers=False)
assert not is_reviewer(request, self.addon, allow_content_reviewers=False)
assert not is_reviewer(request, self.statictheme)
def test_perm_addons(self):
@ -240,11 +243,9 @@ class TestCheckReviewer(TestCase):
self.grant_permission(self.user, 'Addons:ThemeReview')
request = req_factory_factory('noop', user=self.user)
assert not is_reviewer(request, self.addon)
assert not is_reviewer(
request, self.addon, allow_content_reviewers=False)
assert not is_reviewer(request, self.addon, allow_content_reviewers=False)
assert is_reviewer(request, self.statictheme)
assert is_reviewer(
request, self.statictheme, allow_content_reviewers=False)
assert is_reviewer(request, self.statictheme, allow_content_reviewers=False)
assert check_static_theme_reviewer(request)
assert is_user_any_kind_of_reviewer(request.user)
@ -256,13 +257,11 @@ class TestCheckReviewer(TestCase):
assert not check_unlisted_addons_reviewer(request)
assert not check_static_theme_reviewer(request)
assert not is_reviewer(request, self.statictheme)
assert not is_reviewer(
request, self.statictheme, allow_content_reviewers=False)
assert not is_reviewer(request, self.statictheme, allow_content_reviewers=False)
assert check_addons_reviewer(request)
assert is_reviewer(request, self.addon)
assert not is_reviewer(
request, self.addon, allow_content_reviewers=False)
assert not is_reviewer(request, self.addon, allow_content_reviewers=False)
def test_perm_reviewertools_view(self):
self.grant_permission(self.user, 'ReviewerTools:View')
@ -274,17 +273,26 @@ class TestCheckReviewer(TestCase):
assert not is_reviewer(request, self.statictheme)
assert not check_addons_reviewer(request)
assert not is_reviewer(request, self.addon)
assert not is_reviewer(
request, self.addon, allow_content_reviewers=False)
assert not is_reviewer(request, self.addon, allow_content_reviewers=False)
system_guids = pytest.mark.parametrize('guid', [
'foø@mozilla.org', 'baa@shield.mozilla.org', 'moo@pioneer.mozilla.org',
'blâh@mozilla.com', 'foø@Mozilla.Org', 'addon@shield.moZilla.com',
'baa@ShielD.MozillA.OrG', 'moo@PIONEER.mozilla.org', 'blâh@MOZILLA.COM',
'flop@search.mozilla.org', 'user@mozillaonline.com',
'tester@MoZiLlAoNlInE.CoM'
])
system_guids = pytest.mark.parametrize(
'guid',
[
'foø@mozilla.org',
'baa@shield.mozilla.org',
'moo@pioneer.mozilla.org',
'blâh@mozilla.com',
'foø@Mozilla.Org',
'addon@shield.moZilla.com',
'baa@ShielD.MozillA.OrG',
'moo@PIONEER.mozilla.org',
'blâh@MOZILLA.COM',
'flop@search.mozilla.org',
'user@mozillaonline.com',
'tester@MoZiLlAoNlInE.CoM',
],
)
@system_guids

4
src/olympia/accounts/management/commands/monitor_fxa_sqs.py
Просмотреть файл

@ -11,6 +11,7 @@ class Command(BaseCommand):
processing each as it is found. It polls indefinitely and does not return;
to interrupt execution you'll need to e.g. SIGINT the process.
"""
help = 'Monitor the AWS SQS queue for FxA events.'
def add_arguments(self, parser):
@ -20,7 +21,8 @@ class Command(BaseCommand):
action='store',
dest='queue_url',
default=settings.FXA_SQS_AWS_QUEUE_URL,
help='Monitor specified SQS queue, rather than default.')
help='Monitor specified SQS queue, rather than default.',
)
def handle(self, *args, **options):
queue_url = options['queue_url']

168
src/olympia/accounts/serializers.py
Просмотреть файл

@ -11,8 +11,13 @@ from olympia.access import acl
from olympia.access.models import Group
from olympia.amo.templatetags.jinja_helpers import absolutify
from olympia.amo.utils import (
clean_nl, has_links, ImageCheck,
subscribe_newsletter, unsubscribe_newsletter, urlparams)
clean_nl,
has_links,
ImageCheck,
subscribe_newsletter,
unsubscribe_newsletter,
urlparams,
)
from olympia.api.serializers import SiteStatusSerializer
from olympia.api.utils import is_gate_active
from olympia.api.validators import OneOrMorePrintableCharacterAPIValidator
@ -33,8 +38,7 @@ class BaseUserSerializer(serializers.ModelSerializer):
def get_url(self, obj):
def is_adminish(user):
return (user and
acl.action_allowed_user(user, amo.permissions.USERS_EDIT))
return user and acl.action_allowed_user(user, amo.permissions.USERS_EDIT)
request = self.context.get('request', None)
current_user = getattr(request, 'user', None) if request else None
@ -44,8 +48,7 @@ class BaseUserSerializer(serializers.ModelSerializer):
# Used in subclasses.
def get_permissions(self, obj):
out = {perm for group in obj.groups_list
for perm in group.rules.split(',')}
out = {perm for group in obj.groups_list for perm in group.rules.split(',')}
return sorted(out)
# Used in subclasses.
@ -61,10 +64,19 @@ class PublicUserProfileSerializer(BaseUserSerializer):
class Meta(BaseUserSerializer.Meta):
fields = BaseUserSerializer.Meta.fields + (
'average_addon_rating', 'created', 'biography',
'has_anonymous_display_name', 'has_anonymous_username', 'homepage',
'is_addon_developer', 'is_artist', 'location', 'occupation',
'num_addons_listed', 'picture_type', 'picture_url',
'average_addon_rating',
'created',
'biography',
'has_anonymous_display_name',
'has_anonymous_username',
'homepage',
'is_addon_developer',
'is_artist',
'location',
'occupation',
'num_addons_listed',
'picture_type',
'picture_url',
)
# This serializer should never be used for updates but just to be sure.
read_only_fields = fields
@ -72,90 +84,111 @@ class PublicUserProfileSerializer(BaseUserSerializer):
class UserProfileSerializer(PublicUserProfileSerializer):
display_name = serializers.CharField(
min_length=2, max_length=50,
validators=[OneOrMorePrintableCharacterAPIValidator()])
min_length=2,
max_length=50,
validators=[OneOrMorePrintableCharacterAPIValidator()],
)
picture_upload = serializers.ImageField(use_url=True, write_only=True)
permissions = serializers.SerializerMethodField()
fxa_edit_email_url = serializers.SerializerMethodField()
reviewer_name = serializers.CharField(
min_length=2, max_length=50, allow_blank=True,
validators=[OneOrMorePrintableCharacterAPIValidator()])
min_length=2,
max_length=50,
allow_blank=True,
validators=[OneOrMorePrintableCharacterAPIValidator()],
)
# Just Need to specify any field for the source - '*' is the entire obj.
site_status = SiteStatusSerializer(source='*')
class Meta(PublicUserProfileSerializer.Meta):
fields = PublicUserProfileSerializer.Meta.fields + (
'deleted', 'display_name', 'email', 'fxa_edit_email_url',
'last_login', 'last_login_ip', 'permissions', 'picture_upload',
'read_dev_agreement', 'reviewer_name', 'site_status', 'username'
'deleted',
'display_name',
'email',
'fxa_edit_email_url',
'last_login',
'last_login_ip',
'permissions',
'picture_upload',
'read_dev_agreement',
'reviewer_name',
'site_status',
'username',
)
writeable_fields = (
'biography', 'display_name', 'homepage', 'location', 'occupation',
'picture_upload', 'reviewer_name',
'biography',
'display_name',
'homepage',
'location',
'occupation',
'picture_upload',
'reviewer_name',
)
read_only_fields = tuple(set(fields) - set(writeable_fields))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if (not self.instance or
not acl.is_user_any_kind_of_reviewer(self.instance)):
if not self.instance or not acl.is_user_any_kind_of_reviewer(self.instance):
self.fields.pop('reviewer_name', None)
def get_fxa_edit_email_url(self, user):
base_url = '{}/settings'.format(
settings.FXA_CONTENT_HOST
base_url = '{}/settings'.format(settings.FXA_CONTENT_HOST)
return urlparams(
base_url, uid=user.fxa_id, email=user.email, entrypoint='addons'
)
return urlparams(base_url, uid=user.fxa_id, email=user.email,
entrypoint='addons')
def validate_biography(self, value):
if has_links(clean_nl(str(value))):
# There's some links, we don't want them.
raise serializers.ValidationError(
ugettext(u'No links are allowed.'))
raise serializers.ValidationError(ugettext(u'No links are allowed.'))
return value
def validate_display_name(self, value):
if DeniedName.blocked(value):
raise serializers.ValidationError(
ugettext(u'This display name cannot be used.'))
ugettext(u'This display name cannot be used.')
)
return value
def validate_reviewer_name(self, value):
if DeniedName.blocked(value):
raise serializers.ValidationError(
ugettext(u'This reviewer name cannot be used.'))
ugettext(u'This reviewer name cannot be used.')
)
return value
def validate_homepage(self, value):
if settings.DOMAIN.lower() in value.lower():
raise serializers.ValidationError(
ugettext(u'The homepage field can only be used to link to '
u'external websites.')
ugettext(
u'The homepage field can only be used to link to '
u'external websites.'
)
)
return value
def validate_picture_upload(self, value):
image_check = ImageCheck(value)
if (value.content_type not in amo.IMG_TYPES or
not image_check.is_image()):
if value.content_type not in amo.IMG_TYPES or not image_check.is_image():
raise serializers.ValidationError(
ugettext(u'Images must be either PNG or JPG.'))
ugettext(u'Images must be either PNG or JPG.')
)
if image_check.is_animated():
raise serializers.ValidationError(
ugettext(u'Images cannot be animated.'))
raise serializers.ValidationError(ugettext(u'Images cannot be animated.'))
if value.size > settings.MAX_PHOTO_UPLOAD_SIZE:
raise serializers.ValidationError(
ugettext(u'Please use images smaller than %dMB.' %
(settings.MAX_PHOTO_UPLOAD_SIZE / 1024 / 1024)))
ugettext(
u'Please use images smaller than %dMB.'
% (settings.MAX_PHOTO_UPLOAD_SIZE / 1024 / 1024)
)
)
return value
def update(self, instance, validated_data):
instance = super(UserProfileSerializer, self).update(
instance, validated_data)
instance = super(UserProfileSerializer, self).update(instance, validated_data)
photo = validated_data.get('picture_upload')
if photo:
@ -166,16 +199,17 @@ class UserProfileSerializer(PublicUserProfileSerializer):
temp_file.write(chunk)
instance.update(picture_type=photo.content_type)
resize_photo.delay(
tmp_destination, instance.picture_path,
set_modified_on=instance.serializable_reference())
tmp_destination,
instance.picture_path,
set_modified_on=instance.serializable_reference(),
)
return instance
def to_representation(self, obj):
data = super(UserProfileSerializer, self).to_representation(obj)
request = self.context.get('request', None)
if request and is_gate_active(request,
'del-accounts-fxa-edit-email-url'):
if request and is_gate_active(request, 'del-accounts-fxa-edit-email-url'):
data.pop('fxa_edit_email_url', None)
return data
@ -190,19 +224,20 @@ class AccountSuperCreateSerializer(serializers.Serializer):
username = serializers.CharField(required=False)
email = serializers.EmailField(required=False)
fxa_id = serializers.CharField(required=False)
group = serializers.ChoiceField(choices=list(group_rules.items()),
required=False)
group = serializers.ChoiceField(choices=list(group_rules.items()), required=False)
def validate_email(self, email):
if email and UserProfile.objects.filter(email=email).exists():
raise serializers.ValidationError(
'Someone with this email already exists in the system')
'Someone with this email already exists in the system'
)
return email
def validate_username(self, username):
if username and UserProfile.objects.filter(username=username).exists():
raise serializers.ValidationError(
'Someone with this username already exists in the system')
'Someone with this username already exists in the system'
)
return username
def validate_group(self, group):
@ -214,12 +249,13 @@ class AccountSuperCreateSerializer(serializers.Serializer):
qs = Group.objects.filter(rules=rule)
count = qs.count()
if count != 1:
log.info(u'Super creation: looking for group with '
u'permissions {} {} (count: {})'
.format(group, rule, count))
log.info(
u'Super creation: looking for group with '
u'permissions {} {} (count: {})'.format(group, rule, count)
)
raise serializers.ValidationError(
'Could not find a permissions group with the exact '
'rules needed.')
'Could not find a permissions group with the exact ' 'rules needed.'
)
group = qs.get()
return group
@ -233,26 +269,25 @@ class UserNotificationSerializer(serializers.Serializer):
if instance.notification.mandatory:
raise serializers.ValidationError(
'Attempting to set [%s] to %s. Mandatory notifications can\'t '
'be modified' %
(instance.notification.short, validated_data.get('enabled')))
'be modified'
% (instance.notification.short, validated_data.get('enabled'))
)
enabled = validated_data['enabled']
request = self.context['request']
current_user = request.user
remote_by_id = {
ntfn.id: ntfn for ntfn in notifications.REMOTE_NOTIFICATIONS}
remote_by_id = {ntfn.id: ntfn for ntfn in notifications.REMOTE_NOTIFICATIONS}
if instance.notification_id in remote_by_id:
notification = remote_by_id[instance.notification_id]
if not enabled:
unsubscribe_newsletter(
current_user, notification.basket_newsletter_id)
unsubscribe_newsletter(current_user, notification.basket_newsletter_id)
elif enabled:
subscribe_newsletter(
current_user, notification.basket_newsletter_id,
request=request)
current_user, notification.basket_newsletter_id, request=request
)
elif 'enabled' in validated_data:
# Only save if non-mandatory and 'enabled' is set.
# Ignore other fields.
@ -265,6 +300,13 @@ class UserNotificationSerializer(serializers.Serializer):
class UserProfileBasketSyncSerializer(UserProfileSerializer):
class Meta(UserProfileSerializer.Meta):
model = UserProfile
fields = ('id', 'deleted', 'display_name', 'homepage', 'fxa_id',
'last_login', 'location')
fields = (
'id',
'deleted',
'display_name',
'homepage',
'fxa_id',
'last_login',
'location',
)
read_only_fields = fields

21
src/olympia/accounts/tasks.py
Просмотреть файл

@ -26,6 +26,7 @@ def user_profile_from_uid(f):
log.warning('Multiple profile matches for FxA id %s' % uid)
except UserProfile.DoesNotExist:
log.info('No profile match for FxA id %s' % uid)
return wrapper
@ -34,15 +35,17 @@ def user_profile_from_uid(f):
@user_profile_from_uid
def primary_email_change_event(profile, changed_date, email):
"""Process the primaryEmailChangedEvent."""
if (not profile.email_changed or
profile.email_changed < changed_date):
if not profile.email_changed or profile.email_changed < changed_date:
profile.update(email=email, email_changed=changed_date)
log.info(
'Account pk [%s] email [%s] changed from FxA on %s' % (
profile.id, email, changed_date))
'Account pk [%s] email [%s] changed from FxA on %s'
% (profile.id, email, changed_date)
)
else:
log.warning('Account pk [%s] email updated ignored, %s > %s' %
(profile.id, profile.email_changed, changed_date))
log.warning(
'Account pk [%s] email updated ignored, %s > %s'
% (profile.id, profile.email_changed, changed_date)
)
@task
@ -52,9 +55,9 @@ def delete_user_event(user, deleted_date):
"""Process the delete user event."""
if switch_is_active('fxa-account-delete'):
user.delete(addon_msg='Deleted via FxA account deletion')
log.info(
'Account pk [%s] deleted from FxA on %s' % (user.id, deleted_date))
log.info('Account pk [%s] deleted from FxA on %s' % (user.id, deleted_date))
else:
log.info(
f'Skipping deletion from FxA for account [{user.id}] because '
'waffle inactive')
'waffle inactive'
)

15
src/olympia/accounts/tests/test_helpers.py
Просмотреть файл

@ -7,18 +7,17 @@ from olympia.accounts.templatetags import jinja_helpers
@mock.patch(
'olympia.accounts.templatetags.jinja_helpers.utils.default_fxa_login_url',
lambda c: 'http://auth.ca')
lambda c: 'http://auth.ca',
)
def test_login_link():
request = RequestFactory().get('/en-US/firefox/addons')
assert jinja_helpers.login_link({'request': request}) == (
'http://auth.ca')
assert jinja_helpers.login_link({'request': request}) == ('http://auth.ca')
@mock.patch(
'olympia.accounts.templatetags.jinja_helpers.utils.'
'default_fxa_register_url',
lambda c: 'http://auth.ca')
'olympia.accounts.templatetags.jinja_helpers.utils.' 'default_fxa_register_url',
lambda c: 'http://auth.ca',
)
def test_register_link():
request = RequestFactory().get('/en-US/firefox/addons')
assert jinja_helpers.register_link({'request': request}) == (
'http://auth.ca')
assert jinja_helpers.register_link({'request': request}) == ('http://auth.ca')

90
src/olympia/accounts/tests/test_serializers.py
Просмотреть файл

@ -7,9 +7,12 @@ from rest_framework.test import APIRequestFactory
from olympia import amo
from olympia.access.models import Group, GroupUser
from olympia.accounts.serializers import (
BaseUserSerializer, PublicUserProfileSerializer,
UserNotificationSerializer, UserProfileBasketSyncSerializer,
UserProfileSerializer)
BaseUserSerializer,
PublicUserProfileSerializer,
UserNotificationSerializer,
UserProfileBasketSyncSerializer,
UserProfileSerializer,
)
from olympia.amo.templatetags.jinja_helpers import absolutify
from olympia.amo.tests import TestCase, addon_factory, days_ago, user_factory
from olympia.amo.utils import urlparams
@ -22,8 +25,7 @@ class BaseTestUserMixin(object):
def serialize(self):
# Manually reload the user first to clear any cached properties.
self.user = UserProfile.objects.get(pk=self.user.pk)
serializer = self.serializer_class(
self.user, context={'request': self.request})
serializer = self.serializer_class(self.user, context={'request': self.request})
return serializer.to_representation(self.user)
def test_basic(self):
@ -69,8 +71,10 @@ class TestPublicUserProfileSerializer(TestCase):
serializer = PublicUserProfileSerializer
user_kwargs = {
'username': 'amo',
'biography': 'stuff', 'homepage': 'http://mozilla.org/',
'location': 'everywhere', 'occupation': 'job',
'biography': 'stuff',
'homepage': 'http://mozilla.org/',
'location': 'everywhere',
'occupation': 'job',
}
user_private_kwargs = {
'reviewer_name': 'batman',
@ -78,12 +82,12 @@ class TestPublicUserProfileSerializer(TestCase):
def setUp(self):
self.request = APIRequestFactory().get('/')
self.user = user_factory(
**self.user_kwargs, **self.user_private_kwargs)
self.user = user_factory(**self.user_kwargs, **self.user_private_kwargs)
def serialize(self):
return (self.serializer(self.user, context={'request': self.request})
.to_representation(self.user))
return self.serializer(
self.user, context={'request': self.request}
).to_representation(self.user)
def test_picture(self):
serial = self.serialize()
@ -134,8 +138,7 @@ class TestPublicUserProfileSerializer(TestCase):
assert result['url'] == absolutify(self.user.get_url_path())
def test_anonymous_username_display_name(self):
self.user = user_factory(
username='anonymous-bb4f3cbd422e504080e32f2d9bbfcee0')
self.user = user_factory(username='anonymous-bb4f3cbd422e504080e32f2d9bbfcee0')
data = self.serialize()
assert self.user.has_anonymous_username is True
assert data['has_anonymous_username'] is True
@ -165,48 +168,54 @@ class PermissionsTestMixin(object):
# Single permission
group = Group.objects.create(name='a', rules='Addons:Review')
GroupUser.objects.create(group=group, user=self.user)
assert self.serializer(self.user).data['permissions'] == [
'Addons:Review']
assert self.serializer(self.user).data['permissions'] == ['Addons:Review']
# Multiple permissions
group.update(rules='Addons:Review,Addons:Edit')
del self.user.groups_list
assert self.serializer(self.user).data['permissions'] == [
'Addons:Edit', 'Addons:Review']
'Addons:Edit',
'Addons:Review',
]
# Change order to test sort
group.update(rules='Addons:Edit,Addons:Review')
del self.user.groups_list
assert self.serializer(self.user).data['permissions'] == [
'Addons:Edit', 'Addons:Review']
'Addons:Edit',
'Addons:Review',
]
# Add a second group membership to test duplicates
group2 = Group.objects.create(name='b', rules='Foo:Bar,Addons:Edit')
GroupUser.objects.create(group=group2, user=self.user)
assert self.serializer(self.user).data['permissions'] == [
'Addons:Edit', 'Addons:Review', 'Foo:Bar']
'Addons:Edit',
'Addons:Review',
'Foo:Bar',
]
class TestUserProfileSerializer(TestPublicUserProfileSerializer,
PermissionsTestMixin):
class TestUserProfileSerializer(TestPublicUserProfileSerializer, PermissionsTestMixin):
serializer = UserProfileSerializer
def setUp(self):
self.now = days_ago(0)
self.user_email = u'a@m.o'
self.user_kwargs.update({
'email': self.user_email,
'display_name': u'This is my náme',
'last_login_ip': '123.45.67.89',
})
self.user_kwargs.update(
{
'email': self.user_email,
'display_name': u'This is my náme',
'last_login_ip': '123.45.67.89',
}
)
super(TestUserProfileSerializer, self).setUp()
def test_basic(self):
# Have to update these separately as dates as tricky. As are bools.
self.user.update(last_login=self.now, read_dev_agreement=self.now)
data = super(TestUserProfileSerializer, self).test_basic()
assert data['last_login'] == (
self.now.replace(microsecond=0).isoformat() + 'Z')
assert data['last_login'] == (self.now.replace(microsecond=0).isoformat() + 'Z')
assert data['read_dev_agreement'] == data['last_login']
def test_is_reviewer(self):
@ -224,9 +233,12 @@ class TestUserProfileSerializer(TestPublicUserProfileSerializer,
self.user.update(fxa_id=user_fxa_id)
with override_settings(FXA_CONTENT_HOST=fxa_host):
expected_url = urlparams('{}/settings'.format(fxa_host),
uid=user_fxa_id, email=self.user_email,
entrypoint='addons')
expected_url = urlparams(
'{}/settings'.format(fxa_host),
uid=user_fxa_id,
email=self.user_email,
entrypoint='addons',
)
data = super(TestUserProfileSerializer, self).test_basic()
assert data['fxa_edit_email_url'] == expected_url
@ -281,8 +293,8 @@ class TestUserProfileSerializer(TestPublicUserProfileSerializer,
class TestUserProfileBasketSyncSerializer(TestCase):
def setUp(self):
self.user = user_factory(
display_name=None, last_login=self.days_ago(1),
fxa_id='qsdfghjklmù')
display_name=None, last_login=self.days_ago(1), fxa_id='qsdfghjklmù'
)
def test_basic(self):
serializer = UserProfileBasketSyncSerializer(self.user)
@ -292,9 +304,8 @@ class TestUserProfileBasketSyncSerializer(TestCase):
'fxa_id': self.user.fxa_id,
'homepage': '',
'id': self.user.pk,
'last_login': self.user.last_login.replace(
microsecond=0).isoformat() + 'Z',
'location': ''
'last_login': self.user.last_login.replace(microsecond=0).isoformat() + 'Z',
'location': '',
}
self.user.update(display_name='Dîsplay Mé!')
@ -310,21 +321,20 @@ class TestUserProfileBasketSyncSerializer(TestCase):
'fxa_id': self.user.fxa_id,
'homepage': '',
'id': self.user.pk,
'last_login': self.user.last_login.replace(
microsecond=0).isoformat() + 'Z',
'location': ''
'last_login': self.user.last_login.replace(microsecond=0).isoformat() + 'Z',
'location': '',
}
class TestUserNotificationSerializer(TestCase):
def setUp(self):
self.user = user_factory()
def test_basic(self):
notification = NOTIFICATIONS_BY_SHORT['upgrade_fail']
user_notification = UserNotification.objects.create(
user=self.user, notification_id=notification.id, enabled=True)
user=self.user, notification_id=notification.id, enabled=True
)
data = UserNotificationSerializer(user_notification).data
assert data['name'] == user_notification.notification.short
assert data['enabled'] == user_notification.enabled

50
src/olympia/accounts/tests/test_tasks.py
Просмотреть файл

@ -4,11 +4,9 @@ from unittest import mock
from waffle.testutils import override_switch
from olympia import amo
from olympia.accounts.tasks import (
delete_user_event, primary_email_change_event)
from olympia.accounts.tasks import delete_user_event, primary_email_change_event
from olympia.accounts.tests.test_utils import totimestamp
from olympia.amo.tests import (
addon_factory, collection_factory, TestCase, user_factory)
from olympia.amo.tests import addon_factory, collection_factory, TestCase, user_factory
from olympia.bandwagon.models import Collection
from olympia.ratings.models import Rating
@ -17,48 +15,39 @@ class TestPrimaryEmailChangeEvent(TestCase):
fxa_id = 'ABCDEF012345689'
def test_success(self):
user = user_factory(email='old-email@example.com',
fxa_id=self.fxa_id)
user = user_factory(email='old-email@example.com', fxa_id=self.fxa_id)
primary_email_change_event(
self.fxa_id,
totimestamp(datetime(2017, 10, 11)),
'new-email@example.com')
self.fxa_id, totimestamp(datetime(2017, 10, 11)), 'new-email@example.com'
)
user.reload()
assert user.email == 'new-email@example.com'
assert user.email_changed == datetime(2017, 10, 11, 0, 0)
def test_ignored_because_old_timestamp(self):
user = user_factory(email='old-email@example.com',
fxa_id=self.fxa_id)
user = user_factory(email='old-email@example.com', fxa_id=self.fxa_id)
yesterday = datetime(2017, 10, 1)
today = datetime(2017, 10, 2)
tomorrow = datetime(2017, 10, 3)
primary_email_change_event(
self.fxa_id,
totimestamp(today),
'today@example.com')
primary_email_change_event(self.fxa_id, totimestamp(today), 'today@example.com')
assert user.reload().email == 'today@example.com'
primary_email_change_event(
self.fxa_id,
totimestamp(tomorrow),
'tomorrow@example.com')
self.fxa_id, totimestamp(tomorrow), 'tomorrow@example.com'
)
assert user.reload().email == 'tomorrow@example.com'
primary_email_change_event(
self.fxa_id,
totimestamp(yesterday),
'yesterday@example.com')
self.fxa_id, totimestamp(yesterday), 'yesterday@example.com'
)
assert user.reload().email != 'yesterday@example.com'
assert user.reload().email == 'tomorrow@example.com'
def test_ignored_if_user_not_found(self):
"""Check that this doesn't throw"""
primary_email_change_event(
self.fxa_id,
totimestamp(datetime(2017, 10, 11)),
'email@example.com')
self.fxa_id, totimestamp(datetime(2017, 10, 11)), 'email@example.com'
)
class TestDeleteUserEvent(TestCase):
@ -68,9 +57,7 @@ class TestDeleteUserEvent(TestCase):
self.user = user_factory(fxa_id=self.fxa_id)
def _fire_event(self):
delete_user_event(
self.fxa_id,
totimestamp(datetime(2017, 10, 11)))
delete_user_event(self.fxa_id, totimestamp(datetime(2017, 10, 11)))
self.user.reload()
assert self.user.email is not None
assert self.user.deleted
@ -82,8 +69,9 @@ class TestDeleteUserEvent(TestCase):
collection = collection_factory(author=self.user)
another_addon = addon_factory()
Rating.objects.create(addon=another_addon, user=self.user, rating=5)
assert list(another_addon.ratings.all().values('rating', 'user')) == [{
'user': self.user.id, 'rating': 5}]
assert list(another_addon.ratings.all().values('rating', 'user')) == [
{'user': self.user.id, 'rating': 5}
]
self._fire_event()
assert not Collection.objects.filter(id=collection.id).exists()
assert not another_addon.ratings.all().exists()
@ -107,8 +95,6 @@ class TestDeleteUserEvent(TestCase):
@override_switch('fxa-account-delete', active=False)
def test_waffle_off(self):
delete_user_event(
self.fxa_id,
totimestamp(datetime(2017, 10, 11)))
delete_user_event(self.fxa_id, totimestamp(datetime(2017, 10, 11)))
self.user.reload()
assert not self.user.deleted

179
src/olympia/accounts/tests/test_utils.py
Просмотреть файл

@ -69,7 +69,8 @@ def test_default_fxa_login_url_with_state():
raw_url = utils.default_fxa_login_url(request)
url = urlparse(raw_url)
base = '{scheme}://{netloc}{path}'.format(
scheme=url.scheme, netloc=url.netloc, path=url.path)
scheme=url.scheme, netloc=url.netloc, path=url.path
)
assert base == 'https://accounts.firefox.com/oauth/authorization'
query = parse_qs(url.query)
next_path = urlsafe_b64encode(force_bytes(path)).rstrip(b'=')
@ -77,8 +78,7 @@ def test_default_fxa_login_url_with_state():
'action': ['signin'],
'client_id': ['foo'],
'scope': ['profile openid'],
'state': ['myfxastate:{next_path}'.format(
next_path=force_text(next_path))],
'state': ['myfxastate:{next_path}'.format(next_path=force_text(next_path))],
}
@ -91,7 +91,8 @@ def test_default_fxa_register_url_with_state():
raw_url = utils.default_fxa_register_url(request)
url = urlparse(raw_url)
base = '{scheme}://{netloc}{path}'.format(
scheme=url.scheme, netloc=url.netloc, path=url.path)
scheme=url.scheme, netloc=url.netloc, path=url.path
)
assert base == 'https://accounts.firefox.com/oauth/authorization'
query = parse_qs(url.query)
next_path = urlsafe_b64encode(force_bytes(path)).rstrip(b'=')
@ -99,8 +100,7 @@ def test_default_fxa_register_url_with_state():
'action': ['signup'],
'client_id': ['foo'],
'scope': ['profile openid'],
'state': ['myfxastate:{next_path}'.format(
next_path=force_text(next_path))],
'state': ['myfxastate:{next_path}'.format(next_path=force_text(next_path))],
}
@ -113,12 +113,16 @@ def test_fxa_login_url_without_requiring_two_factor_auth():
raw_url = utils.fxa_login_url(
config=FXA_CONFIG['default'],
state=request.session['fxa_state'], next_path=path, action='signin',
force_two_factor=False)
state=request.session['fxa_state'],
next_path=path,
action='signin',
force_two_factor=False,
)
url = urlparse(raw_url)
base = '{scheme}://{netloc}{path}'.format(
scheme=url.scheme, netloc=url.netloc, path=url.path)
scheme=url.scheme, netloc=url.netloc, path=url.path
)
assert base == 'https://accounts.firefox.com/oauth/authorization'
query = parse_qs(url.query)
next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=')
@ -126,8 +130,7 @@ def test_fxa_login_url_without_requiring_two_factor_auth():
'action': ['signin'],
'client_id': ['foo'],
'scope': ['profile openid'],
'state': ['myfxastate:{next_path}'.format(
next_path=force_text(next_path))],
'state': ['myfxastate:{next_path}'.format(next_path=force_text(next_path))],
}
@ -140,12 +143,16 @@ def test_fxa_login_url_requiring_two_factor_auth():
raw_url = utils.fxa_login_url(
config=FXA_CONFIG['default'],
state=request.session['fxa_state'], next_path=path, action='signin',
force_two_factor=True)
state=request.session['fxa_state'],
next_path=path,
action='signin',
force_two_factor=True,
)
url = urlparse(raw_url)
base = '{scheme}://{netloc}{path}'.format(
scheme=url.scheme, netloc=url.netloc, path=url.path)
scheme=url.scheme, netloc=url.netloc, path=url.path
)
assert base == 'https://accounts.firefox.com/oauth/authorization'
query = parse_qs(url.query)
next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=')
@ -154,8 +161,7 @@ def test_fxa_login_url_requiring_two_factor_auth():
'action': ['signin'],
'client_id': ['foo'],
'scope': ['profile openid'],
'state': ['myfxastate:{next_path}'.format(
next_path=force_text(next_path))],
'state': ['myfxastate:{next_path}'.format(next_path=force_text(next_path))],
}
@ -168,12 +174,17 @@ def test_fxa_login_url_requiring_two_factor_auth_passing_token():
raw_url = utils.fxa_login_url(
config=FXA_CONFIG['default'],
state=request.session['fxa_state'], next_path=path, action='signin',
force_two_factor=True, id_token='YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=')
state=request.session['fxa_state'],
next_path=path,
action='signin',
force_two_factor=True,
id_token='YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=',
)
url = urlparse(raw_url)
base = '{scheme}://{netloc}{path}'.format(
scheme=url.scheme, netloc=url.netloc, path=url.path)
scheme=url.scheme, netloc=url.netloc, path=url.path
)
assert base == 'https://accounts.firefox.com/oauth/authorization'
query = parse_qs(url.query)
next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=')
@ -184,8 +195,7 @@ def test_fxa_login_url_requiring_two_factor_auth_passing_token():
'id_token_hint': ['YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo='],
'prompt': ['none'],
'scope': ['profile openid'],
'state': ['myfxastate:{next_path}'.format(
next_path=force_text(next_path))],
'state': ['myfxastate:{next_path}'.format(next_path=force_text(next_path))],
}
@ -215,8 +225,11 @@ def test_fxa_login_url_when_faking_fxa_auth():
request = RequestFactory().get(path)
request.session = {'fxa_state': 'myfxastate'}
raw_url = utils.fxa_login_url(
config=FXA_CONFIG['default'], state=request.session['fxa_state'],
next_path=path, action='signin')
config=FXA_CONFIG['default'],
state=request.session['fxa_state'],
next_path=path,
action='signin',
)
url = urlparse(raw_url)
assert url.scheme == ''
assert url.netloc == ''
@ -227,46 +240,47 @@ def test_fxa_login_url_when_faking_fxa_auth():
'action': ['signin'],
'client_id': ['foo'],
'scope': ['profile openid'],
'state': ['myfxastate:{next_path}'.format(
next_path=force_text(next_path))],
'state': ['myfxastate:{next_path}'.format(next_path=force_text(next_path))],
}
class TestProcessSqsQueue(TestCase):
@mock.patch('boto3._get_default_session')
@mock.patch('olympia.accounts.utils.process_fxa_event')
@mock.patch('boto3.client')
def test_process_sqs_queue(self, client, process_fxa_event, get_session):
messages = [
{'Body': 'foo', 'ReceiptHandle': '$$$'}, {'Body': 'bar'}, None,
{'Body': 'thisonetoo'}]
{'Body': 'foo', 'ReceiptHandle': '$$$'},
{'Body': 'bar'},
None,
{'Body': 'thisonetoo'},
]
sqs = mock.MagicMock(
**{'receive_message.side_effect': [{'Messages': messages}]})
**{'receive_message.side_effect': [{'Messages': messages}]}
)
session_mock = mock.MagicMock(
**{'get_available_regions.side_effect': ['nowh-ere']})
**{'get_available_regions.side_effect': ['nowh-ere']}
)
get_session.return_value = session_mock
delete_mock = mock.MagicMock()
sqs.delete_message = delete_mock
client.return_value = sqs
with self.assertRaises(StopIteration):
utils.process_sqs_queue(
queue_url='https://sqs.nowh-ere.aws.com/123456789/')
utils.process_sqs_queue(queue_url='https://sqs.nowh-ere.aws.com/123456789/')
client.assert_called()
client.assert_called_with(
'sqs', region_name='nowh-ere'
)
client.assert_called_with('sqs', region_name='nowh-ere')
process_fxa_event.assert_called()
# The 'None' in messages would cause an exception, but it should be
# handled, and the remaining message(s) still processed.
process_fxa_event.assert_has_calls(
[mock.call('foo'), mock.call('bar'), mock.call('thisonetoo')])
[mock.call('foo'), mock.call('bar'), mock.call('thisonetoo')]
)
delete_mock.assert_called_once() # Receipt handle is present in foo.
delete_mock.assert_called_with(
QueueUrl='https://sqs.nowh-ere.aws.com/123456789/',
ReceiptHandle='$$$')
QueueUrl='https://sqs.nowh-ere.aws.com/123456789/', ReceiptHandle='$$$'
)
@mock.patch('olympia.accounts.utils.primary_email_change_event.delay')
@mock.patch('olympia.accounts.utils.delete_user_event.delay')
@ -275,15 +289,37 @@ class TestProcessSqsQueue(TestCase):
process_fxa_event(json.dumps({'Message': ''}))
process_fxa_event(json.dumps({'Message': 'ddfdfd'}))
# No timestamps
process_fxa_event(json.dumps({'Message': json.dumps(
{'email': 'foo@baa', 'event': 'primaryEmailChanged',
'uid': '999'})}))
process_fxa_event(json.dumps({'Message': json.dumps(
{'event': 'delete', 'uid': '999'})}))
process_fxa_event(
json.dumps(
{
'Message': json.dumps(
{
'email': 'foo@baa',
'event': 'primaryEmailChanged',
'uid': '999',
}
)
}
)
)
process_fxa_event(
json.dumps({'Message': json.dumps({'event': 'delete', 'uid': '999'})})
)
# Not a supported event type
process_fxa_event(json.dumps({'Message': json.dumps(
{'email': 'foo@baa', 'event': 'not-an-event', 'uid': '999',
'ts': totimestamp(datetime.now())})}))
process_fxa_event(
json.dumps(
{
'Message': json.dumps(
{
'email': 'foo@baa',
'event': 'not-an-event',
'uid': '999',
'ts': totimestamp(datetime.now()),
}
)
}
)
)
delete_mock.assert_not_called()
email_mock.assert_not_called()
@ -297,23 +333,32 @@ class TestProcessFxAEventEmail(TestCase):
def setUp(self):
self.email_changed_date = self.days_ago(42)
self.body = json.dumps({'Message': json.dumps(
{'email': 'new-email@example.com', 'event': 'primaryEmailChanged',
'uid': self.fxa_id,
'ts': totimestamp(self.email_changed_date)})})
self.body = json.dumps(
{
'Message': json.dumps(
{
'email': 'new-email@example.com',
'event': 'primaryEmailChanged',
'uid': self.fxa_id,
'ts': totimestamp(self.email_changed_date),
}
)
}
)
def test_success_integration(self):
user = user_factory(email='old-email@example.com',
fxa_id=self.fxa_id)
user = user_factory(email='old-email@example.com', fxa_id=self.fxa_id)
process_fxa_event(self.body)
user.reload()
assert user.email == 'new-email@example.com'
assert user.email_changed == self.email_changed_date
def test_success_integration_previously_changed_once(self):
user = user_factory(email='old-email@example.com',
fxa_id=self.fxa_id,
email_changed=datetime(2017, 10, 11))
user = user_factory(
email='old-email@example.com',
fxa_id=self.fxa_id,
email_changed=datetime(2017, 10, 11),
)
process_fxa_event(self.body)
user.reload()
assert user.email == 'new-email@example.com'
@ -324,9 +369,8 @@ class TestProcessFxAEventEmail(TestCase):
process_fxa_event(self.body)
primary_email_change_event.assert_called()
primary_email_change_event.assert_called_with(
self.fxa_id,
totimestamp(self.email_changed_date),
'new-email@example.com')
self.fxa_id, totimestamp(self.email_changed_date), 'new-email@example.com'
)
class TestProcessFxAEventDelete(TestCase):
@ -334,10 +378,17 @@ class TestProcessFxAEventDelete(TestCase):
def setUp(self):
self.email_changed_date = self.days_ago(42)
self.body = json.dumps({'Message': json.dumps(
{'event': 'delete',
'uid': self.fxa_id,
'ts': totimestamp(self.email_changed_date)})})
self.body = json.dumps(
{
'Message': json.dumps(
{
'event': 'delete',
'uid': self.fxa_id,
'ts': totimestamp(self.email_changed_date),
}
)
}
)
@override_switch('fxa-account-delete', active=True)
def test_success_integration(self):
@ -353,5 +404,5 @@ class TestProcessFxAEventDelete(TestCase):
process_fxa_event(self.body)
delete_user_event_mock.assert_called()
delete_user_event_mock.assert_called_with(
self.fxa_id,
totimestamp(self.email_changed_date))
self.fxa_id, totimestamp(self.email_changed_date)
)

95
src/olympia/accounts/tests/test_verify.py
Просмотреть файл

@ -8,7 +8,6 @@ from olympia.accounts import verify
class TestProfile(TestCase):
def setUp(self):
patcher = mock.patch('olympia.accounts.verify.requests.get')
self.get = patcher.start()
@ -21,9 +20,12 @@ class TestProfile(TestCase):
self.get.return_value.json.return_value = profile_data
profile = verify.get_fxa_profile('profile-plz', {})
assert profile == profile_data
self.get.assert_called_with('https://app.fxa/v1/profile', headers={
'Authorization': 'Bearer profile-plz',
})
self.get.assert_called_with(
'https://app.fxa/v1/profile',
headers={
'Authorization': 'Bearer profile-plz',
},
)
@override_settings(FXA_PROFILE_HOST='https://app.fxa/v1')
def test_success_no_email(self):
@ -32,9 +34,12 @@ class TestProfile(TestCase):
self.get.return_value.json.return_value = profile_data
with pytest.raises(verify.IdentificationError):
verify.get_fxa_profile('profile-plz', {})
self.get.assert_called_with('https://app.fxa/v1/profile', headers={
'Authorization': 'Bearer profile-plz',
})
self.get.assert_called_with(
'https://app.fxa/v1/profile',
headers={
'Authorization': 'Bearer profile-plz',
},
)
@override_settings(FXA_PROFILE_HOST='https://app.fxa/v1')
def test_failure(self):
@ -43,13 +48,15 @@ class TestProfile(TestCase):
self.get.json.return_value = profile_data
with pytest.raises(verify.IdentificationError):
verify.get_fxa_profile('profile-plz', {})
self.get.assert_called_with('https://app.fxa/v1/profile', headers={
'Authorization': 'Bearer profile-plz',
})
self.get.assert_called_with(
'https://app.fxa/v1/profile',
headers={
'Authorization': 'Bearer profile-plz',
},
)
class TestToken(TestCase):
def setUp(self):
patcher = mock.patch('olympia.accounts.verify.requests.post')
self.post = patcher.start()
@ -60,16 +67,22 @@ class TestToken(TestCase):
token_data = {'access_token': 'c0de'}
self.post.return_value.status_code = 200
self.post.return_value.json.return_value = token_data
token = verify.get_fxa_token('token-plz', {
'client_id': 'test-client-id',
'client_secret': "don't look",
})
token = verify.get_fxa_token(
'token-plz',
{
'client_id': 'test-client-id',
'client_secret': "don't look",
},
)
assert token == token_data
self.post.assert_called_with('https://app.fxa/oauth/v1/token', data={
'code': 'token-plz',
'client_id': 'test-client-id',
'client_secret': "don't look",
})
self.post.assert_called_with(
'https://app.fxa/oauth/v1/token',
data={
'code': 'token-plz',
'client_id': 'test-client-id',
'client_secret': "don't look",
},
)
@override_settings(FXA_OAUTH_HOST='https://app.fxa/oauth/v1')
def test_no_token(self):
@ -77,15 +90,21 @@ class TestToken(TestCase):
self.post.return_value.status_code = 200
self.post.return_value.json.return_value = token_data
with pytest.raises(verify.IdentificationError):
verify.get_fxa_token('token-plz', {
verify.get_fxa_token(
'token-plz',
{
'client_id': 'test-client-id',
'client_secret': "don't look",
},
)
self.post.assert_called_with(
'https://app.fxa/oauth/v1/token',
data={
'code': 'token-plz',
'client_id': 'test-client-id',
'client_secret': "don't look",
})
self.post.assert_called_with('https://app.fxa/oauth/v1/token', data={
'code': 'token-plz',
'client_id': 'test-client-id',
'client_secret': "don't look",
})
},
)
@override_settings(FXA_OAUTH_HOST='https://app.fxa/oauth/v1')
def test_failure(self):
@ -93,15 +112,21 @@ class TestToken(TestCase):
self.post.return_value.status_code = 400
self.post.json.return_value = token_data
with pytest.raises(verify.IdentificationError):
verify.get_fxa_token('token-plz', {
verify.get_fxa_token(
'token-plz',
{
'client_id': 'test-client-id',
'client_secret': "don't look",
},
)
self.post.assert_called_with(
'https://app.fxa/oauth/v1/token',
data={
'code': 'token-plz',
'client_id': 'test-client-id',
'client_secret': "don't look",
})
self.post.assert_called_with('https://app.fxa/oauth/v1/token', data={
'code': 'token-plz',
'client_id': 'test-client-id',
'client_secret': "don't look",
})
},
)
class TestIdentify(TestCase):
@ -142,7 +167,7 @@ class TestIdentify(TestCase):
def test_with_id_token(self):
self.get_token.return_value = {
'access_token': 'cafe',
'id_token': 'openidisawesome'
'id_token': 'openidisawesome',
}
self.get_profile.return_value = {'email': 'me@em.hi'}
identity = verify.fxa_identify('heya', self.CONFIG)

879
src/olympia/accounts/tests/test_views.py
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

57
src/olympia/accounts/urls.py
Просмотреть файл

@ -12,44 +12,49 @@ accounts = SimpleRouter()
accounts.register(r'account', views.AccountViewSet, basename='account')
collections = NestedSimpleRouter(accounts, r'account', lookup='user')
collections.register(r'collections', CollectionViewSet,
basename='collection')
sub_collections = NestedSimpleRouter(collections, r'collections',
lookup='collection')
sub_collections.register('addons', CollectionAddonViewSet,
basename='collection-addon')
collections.register(r'collections', CollectionViewSet, basename='collection')
sub_collections = NestedSimpleRouter(collections, r'collections', lookup='collection')
sub_collections.register('addons', CollectionAddonViewSet, basename='collection-addon')
notifications = NestedSimpleRouter(accounts, r'account', lookup='user')
notifications.register(r'notifications', views.AccountNotificationViewSet,
basename='notification')
notifications.register(
r'notifications', views.AccountNotificationViewSet, basename='notification'
)
accounts_v4 = [
re_path(r'^login/start/$',
views.LoginStartView.as_view(),
name='accounts.login_start'),
re_path(r'^session/$', views.SessionView.as_view(),
name='accounts.session'),
re_path(
r'^login/start/$', views.LoginStartView.as_view(), name='accounts.login_start'
),
re_path(r'^session/$', views.SessionView.as_view(), name='accounts.session'),
re_path(r'', include(accounts.urls)),
re_path(r'^profile/$', views.ProfileView.as_view(),
name='account-profile'),
re_path(r'^super-create/$', views.AccountSuperCreate.as_view(),
name='accounts.super-create'),
re_path(r'^unsubscribe/$',
views.AccountNotificationUnsubscribeView.as_view(),
name='account-unsubscribe'),
re_path(r'^profile/$', views.ProfileView.as_view(), name='account-profile'),
re_path(
r'^super-create/$',
views.AccountSuperCreate.as_view(),
name='accounts.super-create',
),
re_path(
r'^unsubscribe/$',
views.AccountNotificationUnsubscribeView.as_view(),
name='account-unsubscribe',
),
re_path(r'', include(collections.urls)),
re_path(r'', include(sub_collections.urls)),
re_path(r'', include(notifications.urls)),
]
accounts_v3 = accounts_v4 + [
re_path(r'^authenticate/$', views.AuthenticateView.as_view(),
name='accounts.authenticate'),
re_path(
r'^authenticate/$',
views.AuthenticateView.as_view(),
name='accounts.authenticate',
),
]
auth_callback_patterns = [
re_path(r'^authenticate-callback/$', views.AuthenticateView.as_view(),
name='accounts.authenticate'),
re_path(
r'^authenticate-callback/$',
views.AuthenticateView.as_view(),
name='accounts.authenticate',
),
]

79
src/olympia/accounts/utils.py
Просмотреть файл

@ -12,8 +12,7 @@ from django.utils.http import is_safe_url
import boto3
from olympia.accounts.tasks import (
delete_user_event, primary_email_change_event)
from olympia.accounts.tasks import delete_user_event, primary_email_change_event
from olympia.amo.urlresolvers import reverse
from olympia.amo.utils import use_fake_fxa
from olympia.core.logger import getLogger
@ -27,33 +26,44 @@ def _is_safe_url(url, request):
urlparse(settings.CODE_MANAGER_URL).netloc,
)
require_https = request.is_secure() if request else False
return is_safe_url(url, allowed_hosts=allowed_hosts,
require_https=require_https)
return is_safe_url(url, allowed_hosts=allowed_hosts, require_https=require_https)
def fxa_config(request):
config = {camel_case(key): value
for key, value in settings.FXA_CONFIG['default'].items()
if key != 'client_secret'}
config = {
camel_case(key): value
for key, value in settings.FXA_CONFIG['default'].items()
if key != 'client_secret'
}
request.session.setdefault('fxa_state', generate_fxa_state())
config.update(**{
'contentHost': settings.FXA_CONTENT_HOST,
'oauthHost': settings.FXA_OAUTH_HOST,
'profileHost': settings.FXA_PROFILE_HOST,
'scope': 'profile openid',
'state': request.session['fxa_state'],
})
config.update(
**{
'contentHost': settings.FXA_CONTENT_HOST,
'oauthHost': settings.FXA_OAUTH_HOST,
'profileHost': settings.FXA_PROFILE_HOST,
'scope': 'profile openid',
'state': request.session['fxa_state'],
}
)
if request.user.is_authenticated:
config['email'] = request.user.email
return config
def fxa_login_url(config, state, next_path=None, action=None,
force_two_factor=False, request=None, id_token=None):
def fxa_login_url(
config,
state,
next_path=None,
action=None,
force_two_factor=False,
request=None,
id_token=None,
):
if next_path and _is_safe_url(next_path, request):
state += ':' + force_text(
urlsafe_b64encode(next_path.encode('utf-8'))).rstrip('=')
state += ':' + force_text(urlsafe_b64encode(next_path.encode('utf-8'))).rstrip(
'='
)
query = {
'client_id': config['client_id'],
'scope': 'profile openid',
@ -77,8 +87,7 @@ def fxa_login_url(config, state, next_path=None, action=None,
base_url = reverse('fake-fxa-authorization')
else:
base_url = '{host}/authorization'.format(host=settings.FXA_OAUTH_HOST)
return '{base_url}?{query}'.format(
base_url=base_url, query=urlencode(query))
return '{base_url}?{query}'.format(base_url=base_url, query=urlencode(query))
def default_fxa_register_url(request):
@ -87,7 +96,8 @@ def default_fxa_register_url(request):
config=settings.FXA_CONFIG['default'],
state=request.session['fxa_state'],
next_path=path_with_query(request),
action='signup')
action='signup',
)
def default_fxa_login_url(request):
@ -96,7 +106,8 @@ def default_fxa_login_url(request):
config=settings.FXA_CONFIG['default'],
state=request.session['fxa_state'],
next_path=path_with_query(request),
action='signin')
action='signin',
)
def generate_fxa_state():
@ -133,16 +144,16 @@ def process_fxa_event(raw_body):
uid = event.get('uid')
timestamp = event.get('ts', 0)
if not (event_type and uid and timestamp):
raise ValueError(
'Properties event, uuid, and ts must all be non-empty')
raise ValueError('Properties event, uuid, and ts must all be non-empty')
except (ValueError, KeyError, TypeError) as e:
log.exception('Invalid account message: %s' % e)
else:
if event_type == 'primaryEmailChanged':
email = event.get('email')
if not email:
log.error('Email property must be non-empty for "%s" event' %
event_type)
log.error(
'Email property must be non-empty for "%s" event' % event_type
)
else:
primary_email_change_event.delay(uid, timestamp, email)
elif event_type == 'delete':
@ -156,11 +167,12 @@ def process_sqs_queue(queue_url):
log.info('Processing account events from %s', queue_url)
try:
region = queue_url.split('.')[1]
available_regions = (boto3._get_default_session()
.get_available_regions('sqs'))
available_regions = boto3._get_default_session().get_available_regions('sqs')
if region not in available_regions:
log.error('SQS misconfigured, expected region, got %s from %s' % (
region, queue_url))
log.error(
'SQS misconfigured, expected region, got %s from %s'
% (region, queue_url)
)
# Connect to the SQS queue.
# Credentials are specified in EC2 as an IAM role on prod/stage/dev.
# If you're testing locally see boto3 docs for how to specify:
@ -171,7 +183,8 @@ def process_sqs_queue(queue_url):
response = sqs.receive_message(
QueueUrl=queue_url,
WaitTimeSeconds=settings.FXA_SQS_AWS_WAIT_TIME,
MaxNumberOfMessages=10)
MaxNumberOfMessages=10,
)
msgs = response.get('Messages', []) if response else []
for message in msgs:
try:
@ -180,8 +193,8 @@ def process_sqs_queue(queue_url):
# unrecognized type. Not point leaving a backlog.
if 'ReceiptHandle' in message:
sqs.delete_message(
QueueUrl=queue_url,
ReceiptHandle=message['ReceiptHandle'])
QueueUrl=queue_url, ReceiptHandle=message['ReceiptHandle']
)
except Exception as exc:
log.exception('Error while processing message: %s' % exc)
except Exception as exc:

45
src/olympia/accounts/verify.py
Просмотреть файл

@ -38,11 +38,14 @@ def get_fxa_token(code, config):
"""
log.info('Getting token [{code}]'.format(code=code))
with statsd.timer('accounts.fxa.identify.token'):
response = requests.post(settings.FXA_OAUTH_HOST + '/token', data={
'code': code,
'client_id': config['client_id'],
'client_secret': config['client_secret'],
})
response = requests.post(
settings.FXA_OAUTH_HOST + '/token',
data={
'code': code,
'client_id': config['client_id'],
'client_secret': config['client_secret'],
},
)
if response.status_code == 200:
data = response.json()
if data.get('access_token'):
@ -51,13 +54,17 @@ def get_fxa_token(code, config):
else:
log.info('No token returned [{code}]'.format(code=code))
raise IdentificationError(
'No access token returned for {code}'.format(code=code))
'No access token returned for {code}'.format(code=code)
)
else:
log.info(
'Token returned non-200 status {status} {body} [{code}]'.format(
code=code, status=response.status_code, body=response.content))
code=code, status=response.status_code, body=response.content
)
)
raise IdentificationError(
'Could not get access token for {code}'.format(code=code))
'Could not get access token for {code}'.format(code=code)
)
def get_fxa_profile(token, config):
@ -65,9 +72,10 @@ def get_fxa_profile(token, config):
corresponding user."""
with statsd.timer('accounts.fxa.identify.profile'):
response = requests.get(
settings.FXA_PROFILE_HOST + '/profile', headers={
settings.FXA_PROFILE_HOST + '/profile',
headers={
'Authorization': 'Bearer {token}'.format(token=token),
}
},
)
if response.status_code == 200:
profile = response.json()
@ -75,10 +83,15 @@ def get_fxa_profile(token, config):
return profile
else:
log.info('Incomplete profile {profile}'.format(profile=profile))
raise IdentificationError('Profile incomplete for {token}'.format(
token=token))
raise IdentificationError(
'Profile incomplete for {token}'.format(token=token)
)
else:
log.info('Profile returned non-200 status {status} {body}'.format(
status=response.status_code, body=response.content))
raise IdentificationError('Could not find profile for {token}'.format(
token=token))
log.info(
'Profile returned non-200 status {status} {body}'.format(
status=response.status_code, body=response.content
)
)
raise IdentificationError(
'Could not find profile for {token}'.format(token=token)
)

311
src/olympia/accounts/views.py
Просмотреть файл

@ -20,9 +20,12 @@ import waffle
from corsheaders.conf import conf as corsheaders_conf
from corsheaders.middleware import (
ACCESS_CONTROL_ALLOW_ORIGIN, ACCESS_CONTROL_ALLOW_CREDENTIALS,
ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS,
ACCESS_CONTROL_MAX_AGE)
ACCESS_CONTROL_ALLOW_ORIGIN,
ACCESS_CONTROL_ALLOW_CREDENTIALS,
ACCESS_CONTROL_ALLOW_HEADERS,
ACCESS_CONTROL_ALLOW_METHODS,
ACCESS_CONTROL_MAX_AGE,
)
from django_statsd.clients import statsd
from rest_framework import serializers
from rest_framework.authentication import SessionAuthentication
@ -30,9 +33,12 @@ from rest_framework.decorators import action
from rest_framework.exceptions import PermissionDenied
from rest_framework.generics import GenericAPIView
from rest_framework.mixins import (
DestroyModelMixin, ListModelMixin, RetrieveModelMixin, UpdateModelMixin)
from rest_framework.permissions import (
AllowAny, BasePermission, IsAuthenticated)
DestroyModelMixin,
ListModelMixin,
RetrieveModelMixin,
UpdateModelMixin,
)
from rest_framework.permissions import AllowAny, BasePermission, IsAuthenticated
from rest_framework.response import Response
from rest_framework.status import HTTP_204_NO_CONTENT
from rest_framework.views import APIView
@ -48,17 +54,24 @@ from olympia.amo import messages
from olympia.amo.decorators import use_primary_db
from olympia.amo.utils import fetch_subscribed_newsletters, use_fake_fxa
from olympia.api.authentication import (
JWTKeyAuthentication, UnsubscribeTokenAuthentication,
WebTokenAuthentication)
JWTKeyAuthentication,
UnsubscribeTokenAuthentication,
WebTokenAuthentication,
)
from olympia.api.permissions import AnyOf, ByHttpMethod, GroupPermission
from olympia.users.models import UserNotification, UserProfile
from olympia.users.notifications import (
NOTIFICATIONS_COMBINED, REMOTE_NOTIFICATIONS_BY_BASKET_ID)
NOTIFICATIONS_COMBINED,
REMOTE_NOTIFICATIONS_BY_BASKET_ID,
)
from . import verify
from .serializers import (
AccountSuperCreateSerializer, PublicUserProfileSerializer,
UserNotificationSerializer, UserProfileSerializer)
AccountSuperCreateSerializer,
PublicUserProfileSerializer,
UserNotificationSerializer,
UserProfileSerializer,
)
from .utils import _is_safe_url, fxa_login_url, generate_fxa_state
@ -77,10 +90,8 @@ ERROR_STATUSES = {
}
LOGIN_ERROR_MESSAGES = {
ERROR_AUTHENTICATED: _(u'You are already logged in.'),
ERROR_NO_CODE:
_(u'Your login attempt could not be parsed. Please try again.'),
ERROR_NO_PROFILE:
_(u'Your Firefox Account could not be found. Please try again.'),
ERROR_NO_CODE: _(u'Your login attempt could not be parsed. Please try again.'),
ERROR_NO_PROFILE: _(u'Your Firefox Account could not be found. Please try again.'),
ERROR_STATE_MISMATCH: _(u'You could not be logged in. Please try again.'),
}
@ -113,7 +124,8 @@ def find_user(identity):
"""
try:
user = UserProfile.objects.get(
Q(fxa_id=identity['uid']) | Q(email=identity['email']))
Q(fxa_id=identity['uid']) | Q(email=identity['email'])
)
is_task_user = user.id == settings.TASK_USER_ID
if user.banned or is_task_user:
# If the user was banned raise a 403, it's not the prettiest
@ -128,14 +140,15 @@ def find_user(identity):
except UserProfile.MultipleObjectsReturned:
# This shouldn't happen, so let it raise.
log.error(
'Found multiple users for %s and %s',
identity['email'], identity['uid'])
'Found multiple users for %s and %s', identity['email'], identity['uid']
)
raise
def register_user(identity):
user = UserProfile.objects.create_user(
email=identity['email'], fxa_id=identity['uid'])
email=identity['email'], fxa_id=identity['uid']
)
log.info('Created user {} from FxA'.format(user))
statsd.incr('accounts.account_created_from_fxa')
return user
@ -150,13 +163,17 @@ def reregister_user(user):
def update_user(user, identity):
"""Update a user's info from FxA if needed, as well as generating the id
that is used as part of the session/api token generation."""
if (user.fxa_id != identity['uid'] or
user.email != identity['email']):
if user.fxa_id != identity['uid'] or user.email != identity['email']:
log.info(
'Updating user info from FxA for {pk}. Old {old_email} {old_uid} '
'New {new_email} {new_uid}'.format(
pk=user.pk, old_email=user.email, old_uid=user.fxa_id,
new_email=identity['email'], new_uid=identity['uid']))
pk=user.pk,
old_email=user.email,
old_uid=user.fxa_id,
new_email=identity['email'],
new_uid=identity['uid'],
)
)
user.update(fxa_id=identity['uid'], email=identity['email'])
if user.auth_id is None:
# If the user didn't have an auth id (old user account created before
@ -174,12 +191,13 @@ def login_user(sender, request, user, identity):
def fxa_error_message(message, login_help_url):
return format_html(
u'{error} <a href="{url}">{help_text}</a>',
url=login_help_url, help_text=_(u'Need help?'),
error=message)
url=login_help_url,
help_text=_(u'Need help?'),
error=message,
)
LOGIN_HELP_URL = (
'https://support.mozilla.org/kb/access-your-add-ons-firefox-accounts')
LOGIN_HELP_URL = 'https://support.mozilla.org/kb/access-your-add-ons-firefox-accounts'
def render_error(request, error, next_path=None, format=None):
@ -192,7 +210,8 @@ def render_error(request, error, next_path=None, format=None):
messages.error(
request,
fxa_error_message(LOGIN_ERROR_MESSAGES[error], LOGIN_HELP_URL),
extra_tags='fxa')
extra_tags='fxa',
)
if next_path is None:
response = HttpResponseRedirect('/')
else:
@ -207,11 +226,11 @@ def parse_next_path(state_parts, request=None):
# but it only cares if there are too few so add 4 of them.
encoded_path = state_parts[1] + '===='
try:
next_path = base64.urlsafe_b64decode(
force_bytes(encoded_path)).decode('utf-8')
next_path = base64.urlsafe_b64decode(force_bytes(encoded_path)).decode(
'utf-8'
)
except (TypeError, ValueError):
log.info('Error decoding next_path {}'.format(
encoded_path))
log.info('Error decoding next_path {}'.format(encoded_path))
pass
if not _is_safe_url(next_path, request):
next_path = None
@ -219,7 +238,6 @@ def parse_next_path(state_parts, request=None):
def with_user(format):
def outer(fn):
@functools.wraps(fn)
@use_primary_db
@ -236,29 +254,34 @@ def with_user(format):
if not data.get('code'):
log.info('No code provided.')
return render_error(
request, ERROR_NO_CODE, next_path=next_path, format=format)
elif (not request.session.get('fxa_state') or
request.session['fxa_state'] != state):
request, ERROR_NO_CODE, next_path=next_path, format=format
)
elif (
not request.session.get('fxa_state')
or request.session['fxa_state'] != state
):
log.info(
'State mismatch. URL: {url} Session: {session}'.format(
url=data.get('state'),
session=request.session.get('fxa_state'),
))
)
)
return render_error(
request, ERROR_STATE_MISMATCH, next_path=next_path,
format=format)
request, ERROR_STATE_MISMATCH, next_path=next_path, format=format
)
elif request.user.is_authenticated:
response = render_error(
request, ERROR_AUTHENTICATED, next_path=next_path,
format=format)
request, ERROR_AUTHENTICATED, next_path=next_path, format=format
)
# If the api token cookie is missing but we're still
# authenticated using the session, add it back.
if API_TOKEN_COOKIE not in request.COOKIES:
log.info('User %s was already authenticated but did not '
'have an API token cookie, adding one.',
request.user.pk)
response = add_api_token_to_response(
response, request.user)
log.info(
'User %s was already authenticated but did not '
'have an API token cookie, adding one.',
request.user.pk,
)
response = add_api_token_to_response(response, request.user)
return response
try:
if use_fake_fxa() and 'fake_fxa_email' in data:
@ -266,33 +289,36 @@ def with_user(format):
# and generate a random fxa id.
identity = {
'email': data['fake_fxa_email'],
'uid': 'fake_fxa_id-%s' % force_text(
binascii.b2a_hex(os.urandom(16))
)
'uid': 'fake_fxa_id-%s'
% force_text(binascii.b2a_hex(os.urandom(16))),
}
id_token = identity['email']
else:
identity, id_token = verify.fxa_identify(
data['code'], config=fxa_config)
data['code'], config=fxa_config
)
except verify.IdentificationError:
log.info('Profile not found. Code: {}'.format(data['code']))
return render_error(
request, ERROR_NO_PROFILE, next_path=next_path,
format=format)
request, ERROR_NO_PROFILE, next_path=next_path, format=format
)
else:
user = find_user(identity)
# We can't use waffle.flag_is_active() wrapper, because
# request.user isn't populated at this point (and we don't want
# it to be).
flag = waffle.get_waffle_flag_model().get(
'2fa-enforcement-for-developers-and-special-users')
enforce_2fa_for_developers_and_special_users = (
flag.is_active(request) or
(flag.pk and flag.is_active_for_user(user)))
if (user and
not identity.get('twoFactorAuthentication') and
enforce_2fa_for_developers_and_special_users and
(user.is_addon_developer or user.groups_list)):
'2fa-enforcement-for-developers-and-special-users'
)
enforce_2fa_for_developers_and_special_users = flag.is_active(
request
) or (flag.pk and flag.is_active_for_user(user))
if (
user
and not identity.get('twoFactorAuthentication')
and enforce_2fa_for_developers_and_special_users
and (user.is_addon_developer or user.groups_list)
):
# https://github.com/mozilla/addons/issues/732
# The user is an add-on developer (with other types of
# add-ons than just themes) or part of any group (so they
@ -316,9 +342,11 @@ def with_user(format):
)
)
return fn(
self, request, user=user, identity=identity,
next_path=next_path)
self, request, user=user, identity=identity, next_path=next_path
)
return inner
return outer
@ -347,7 +375,8 @@ def add_api_token_to_response(response, user):
max_age=settings.SESSION_COOKIE_AGE,
secure=settings.SESSION_COOKIE_SECURE,
httponly=settings.SESSION_COOKIE_HTTPONLY,
samesite=settings.SESSION_COOKIE_SAMESITE)
samesite=settings.SESSION_COOKIE_SAMESITE,
)
return response
@ -359,8 +388,7 @@ class FxAConfigMixin(object):
def get_config_name(self, request):
config_name = request.GET.get('config', self.DEFAULT_FXA_CONFIG_NAME)
if config_name not in self.ALLOWED_FXA_CONFIGS:
log.info('Using default FxA config instead of {}'.format(
config_name))
log.info('Using default FxA config instead of {}'.format(config_name))
config_name = self.DEFAULT_FXA_CONFIG_NAME
return config_name
@ -369,7 +397,6 @@ class FxAConfigMixin(object):
class LoginStartView(FxAConfigMixin, APIView):
def get(self, request):
request.session.setdefault('fxa_state', generate_fxa_state())
return HttpResponseRedirect(
@ -417,17 +444,20 @@ def logout_user(request, response):
response.delete_cookie(
API_TOKEN_COOKIE,
domain=settings.SESSION_COOKIE_DOMAIN,
samesite=settings.SESSION_COOKIE_SAMESITE)
samesite=settings.SESSION_COOKIE_SAMESITE,
)
# This view is not covered by the CORS middleware, see:
# https://github.com/mozilla/addons-server/issues/11100
class SessionView(APIView):
permission_classes = [
ByHttpMethod({
'options': AllowAny, # Needed for CORS.
'delete': IsAuthenticated,
}),
ByHttpMethod(
{
'options': AllowAny, # Needed for CORS.
'delete': IsAuthenticated,
}
),
]
def options(self, request, *args, **kwargs):
@ -446,8 +476,7 @@ class SessionView(APIView):
corsheaders_conf.CORS_ALLOW_METHODS
)
if corsheaders_conf.CORS_PREFLIGHT_MAX_AGE:
response[ACCESS_CONTROL_MAX_AGE] = (
corsheaders_conf.CORS_PREFLIGHT_MAX_AGE)
response[ACCESS_CONTROL_MAX_AGE] = corsheaders_conf.CORS_PREFLIGHT_MAX_AGE
return response
def delete(self, request, *args, **kwargs):
@ -469,19 +498,20 @@ class AllowSelf(BasePermission):
return request.user.is_authenticated and obj == request.user
class AccountViewSet(RetrieveModelMixin, UpdateModelMixin, DestroyModelMixin,
GenericViewSet):
class AccountViewSet(
RetrieveModelMixin, UpdateModelMixin, DestroyModelMixin, GenericViewSet
):
permission_classes = [
ByHttpMethod({
'get': AllowAny,
'head': AllowAny,
'options': AllowAny, # Needed for CORS.
# To edit a profile it has to yours, or be an admin.
'patch': AnyOf(AllowSelf, GroupPermission(
amo.permissions.USERS_EDIT)),
'delete': AnyOf(AllowSelf, GroupPermission(
amo.permissions.USERS_EDIT)),
}),
ByHttpMethod(
{
'get': AllowAny,
'head': AllowAny,
'options': AllowAny, # Needed for CORS.
# To edit a profile it has to yours, or be an admin.
'patch': AnyOf(AllowSelf, GroupPermission(amo.permissions.USERS_EDIT)),
'delete': AnyOf(AllowSelf, GroupPermission(amo.permissions.USERS_EDIT)),
}
),
]
# Periods are not allowed in username, but we still have some in the
# database so relax the lookup regexp to allow them to load their profile.
@ -499,10 +529,11 @@ class AccountViewSet(RetrieveModelMixin, UpdateModelMixin, DestroyModelMixin,
self.instance = super(AccountViewSet, self).get_object()
# action won't exist for other classes that are using this ViewSet.
can_view_instance = (
not getattr(self, 'action', None) or
self.self_view or
self.admin_viewing or
self.instance.is_public)
not getattr(self, 'action', None)
or self.self_view
or self.admin_viewing
or self.instance.is_public
)
if can_view_instance:
return self.instance
else:
@ -519,13 +550,13 @@ class AccountViewSet(RetrieveModelMixin, UpdateModelMixin, DestroyModelMixin,
@property
def self_view(self):
return (
self.request.user.is_authenticated and
self.get_object() == self.request.user)
self.request.user.is_authenticated
and self.get_object() == self.request.user
)
@property
def admin_viewing(self):
return acl.action_allowed_user(
self.request.user, amo.permissions.USERS_EDIT)
return acl.action_allowed_user(self.request.user, amo.permissions.USERS_EDIT)
def get_serializer_class(self):
if self.self_view or self.admin_viewing:
@ -543,8 +574,11 @@ class AccountViewSet(RetrieveModelMixin, UpdateModelMixin, DestroyModelMixin,
@action(
detail=True,
methods=['delete'], permission_classes=[
AnyOf(AllowSelf, GroupPermission(amo.permissions.USERS_EDIT))])
methods=['delete'],
permission_classes=[
AnyOf(AllowSelf, GroupPermission(amo.permissions.USERS_EDIT))
],
)
def picture(self, request, pk=None):
user = self.get_object()
user.delete_picture()
@ -560,7 +594,8 @@ class ProfileView(APIView):
account_viewset = AccountViewSet(
request=request,
permission_classes=self.permission_classes,
kwargs={'pk': str(self.request.user.pk)})
kwargs={'pk': str(self.request.user.pk)},
)
account_viewset.format_kwarg = self.format_kwarg
return account_viewset.retrieve(request)
@ -569,14 +604,14 @@ class AccountSuperCreate(APIView):
authentication_classes = [JWTKeyAuthentication]
permission_classes = [
IsAuthenticated,
GroupPermission(amo.permissions.ACCOUNTS_SUPER_CREATE)]
GroupPermission(amo.permissions.ACCOUNTS_SUPER_CREATE),
]
@waffle_switch('super-create-accounts')
def post(self, request):
serializer = AccountSuperCreateSerializer(data=request.data)
if not serializer.is_valid():
return Response({'errors': serializer.errors},
status=422)
return Response({'errors': serializer.errors}, status=422)
data = serializer.data
@ -591,7 +626,8 @@ class AccountSuperCreate(APIView):
email=email,
fxa_id=fxa_id,
display_name='Super Created {}'.format(user_token),
notes='auto-generated from API')
notes='auto-generated from API',
)
user.save()
if group:
@ -601,11 +637,12 @@ class AccountSuperCreate(APIView):
login_user(self.__class__, request, user, identity)
request.session.save()
log.info(u'API user {api_user} created and logged in a user from '
u'the super-create API: user_id: {user.pk}; '
u'user_name: {user.username}; fxa_id: {user.fxa_id}; '
u'group: {group}'
.format(user=user, api_user=request.user, group=group))
log.info(
u'API user {api_user} created and logged in a user from '
u'the super-create API: user_id: {user.pk}; '
u'user_name: {user.username}; fxa_id: {user.fxa_id}; '
u'group: {group}'.format(user=user, api_user=request.user, group=group)
)
cookie = {
'name': settings.SESSION_COOKIE_NAME,
@ -613,19 +650,21 @@ class AccountSuperCreate(APIView):
}
cookie['encoded'] = '{name}={value}'.format(**cookie)
return Response({
'user_id': user.pk,
'username': user.username,
'email': user.email,
'display_name': user.display_name,
'groups': list((g.pk, g.name, g.rules) for g in user.groups.all()),
'fxa_id': user.fxa_id,
'session_cookie': cookie,
}, status=201)
return Response(
{
'user_id': user.pk,
'username': user.username,
'email': user.email,
'display_name': user.display_name,
'groups': list((g.pk, g.name, g.rules) for g in user.groups.all()),
'fxa_id': user.fxa_id,
'session_cookie': cookie,
},
status=201,
)
class AccountNotificationMixin(object):
def get_user(self):
raise NotImplementedError
@ -633,7 +672,8 @@ class AccountNotificationMixin(object):
return UserNotification(
user=self.get_user(),
notification_id=notification.id,
enabled=notification.default_checked)
enabled=notification.default_checked,
)
def get_queryset(self, dev=False):
user = self.get_user()
@ -644,8 +684,10 @@ class AccountNotificationMixin(object):
# Put it into a dict so we can easily check for existence.
set_notifications = {
user_nfn.notification.short: user_nfn for user_nfn in queryset
if user_nfn.notification}
user_nfn.notification.short: user_nfn
for user_nfn in queryset
if user_nfn.notification
}
out = []
newsletters = None # Lazy - fetch the first time needed.
@ -665,22 +707,28 @@ class AccountNotificationMixin(object):
if notification.group == 'dev' and not include_dev:
# We only return dev notifications for developers.
continue
out.append(set_notifications.get(
notification.short, # It's been set by the user.
self._get_default_object(notification))) # Or, default.
out.append(
set_notifications.get(
notification.short, # It's been set by the user.
self._get_default_object(notification),
)
) # Or, default.
return out
class AccountNotificationViewSet(AccountNotificationMixin, ListModelMixin,
GenericViewSet):
class AccountNotificationViewSet(
AccountNotificationMixin, ListModelMixin, GenericViewSet
):
"""Returns account notifications.
If not already set by the user, defaults will be returned.
"""
permission_classes = [IsAuthenticated]
# We're pushing the primary permission checking to AccountViewSet for ease.
account_permission_classes = [
AnyOf(AllowSelf, GroupPermission(amo.permissions.USERS_EDIT))]
AnyOf(AllowSelf, GroupPermission(amo.permissions.USERS_EDIT))
]
serializer_class = UserNotificationSerializer
paginator = None
@ -692,7 +740,8 @@ class AccountNotificationViewSet(AccountNotificationMixin, ListModelMixin,
self.account_viewset = AccountViewSet(
request=self.request,
permission_classes=self.account_permission_classes,
kwargs={'pk': self.kwargs['user_pk']})
kwargs={'pk': self.kwargs['user_pk']},
)
return self.account_viewset
def create(self, request, *args, **kwargs):
@ -703,14 +752,14 @@ class AccountNotificationViewSet(AccountNotificationMixin, ListModelMixin,
enabled = request.data.get(notification.notification.short)
if enabled is not None:
serializer = self.get_serializer(
notification, partial=True, data={'enabled': enabled})
notification, partial=True, data={'enabled': enabled}
)
serializer.is_valid(raise_exception=True)
serializer.save()
return Response(self.get_serializer(queryset, many=True).data)
class AccountNotificationUnsubscribeView(AccountNotificationMixin,
GenericAPIView):
class AccountNotificationUnsubscribeView(AccountNotificationMixin, GenericAPIView):
authentication_classes = (UnsubscribeTokenAuthentication,)
permission_classes = ()
serializer_class = UserNotificationSerializer
@ -724,11 +773,13 @@ class AccountNotificationUnsubscribeView(AccountNotificationMixin,
for notification in self.get_queryset(dev=True):
if notification_name == notification.notification.short:
serializer = self.get_serializer(
notification, partial=True, data={'enabled': False})
notification, partial=True, data={'enabled': False}
)
serializer.is_valid(raise_exception=True)
serializer.save()
if not serializer:
raise serializers.ValidationError(
_('Notification [%s] does not exist') % notification_name)
_('Notification [%s] does not exist') % notification_name
)
return Response(serializer.data)

1
src/olympia/activity/__init__.py
Просмотреть файл

@ -1,4 +1,5 @@
def log_create(action, *args, **kw):
"""Use this if importing ActivityLog causes a circular import."""
from olympia.activity.models import ActivityLog
return ActivityLog.create(action, *args, **kw)

18
src/olympia/activity/admin.py
Просмотреть файл

@ -4,11 +4,23 @@ from .models import ActivityLog
class ActivityLogAdmin(admin.ModelAdmin):
list_display = ('created', 'user', '__str__',)
list_display = (
'created',
'user',
'__str__',
)
raw_id_fields = ('user',)
readonly_fields = ('created', 'user', '__str__',)
readonly_fields = (
'created',
'user',
'__str__',
)
date_hierarchy = 'created'
fields = ('user', 'created', '__str__',)
fields = (
'user',
'created',
'__str__',
)
raw_id_fields = ('user',)
def has_add_permission(self, request):

28
src/olympia/activity/management/commands/repudiate_token.py
Просмотреть файл

@ -15,25 +15,31 @@ class Command(BaseCommand):
"""Handle command arguments."""
parser.add_argument('token_uuid', nargs='*')
parser.add_argument(
'--version_id', action='store', type=int,
'--version_id',
action='store',
type=int,
dest='version_id',
help='Expire all tokens on this version.')
help='Expire all tokens on this version.',
)
def handle(self, *args, **options):
version_pk = options.get('version_id')
token_uuids = options.get('token_uuid')
if token_uuids:
done = [t.expire() for t in ActivityLogToken.objects.filter(
uuid__in=token_uuids)]
log.info(
u'%s tokens (%s) expired' % (len(done), ','.join(token_uuids)))
done = [
t.expire()
for t in ActivityLogToken.objects.filter(uuid__in=token_uuids)
]
log.info(u'%s tokens (%s) expired' % (len(done), ','.join(token_uuids)))
if version_pk:
print('Warning: --version_id ignored as tokens provided too')
elif version_pk:
done = [t.expire() for t in ActivityLogToken.objects.filter(
version__pk=version_pk)]
log.info(
u'%s tokens for version %s expired' % (len(done), version_pk))
done = [
t.expire()
for t in ActivityLogToken.objects.filter(version__pk=version_pk)
]
log.info(u'%s tokens for version %s expired' % (len(done), version_pk))
else:
raise CommandError(
u'Please provide either at least one token, or a version id.')
u'Please provide either at least one token, or a version id.'
)

220
src/olympia/activity/models.py
Просмотреть файл

@ -41,43 +41,49 @@ MAX_TOKEN_USE_COUNT = 100
class ActivityLogToken(ModelBase):
id = PositiveAutoField(primary_key=True)
version = models.ForeignKey(
Version, related_name='token', on_delete=models.CASCADE)
version = models.ForeignKey(Version, related_name='token', on_delete=models.CASCADE)
user = models.ForeignKey(
'users.UserProfile', related_name='activity_log_tokens',
on_delete=models.CASCADE)
'users.UserProfile',
related_name='activity_log_tokens',
on_delete=models.CASCADE,
)
uuid = models.UUIDField(default=uuid.uuid4, unique=True)
use_count = models.IntegerField(
default=0,
help_text='Stores the number of times the token has been used')
default=0, help_text='Stores the number of times the token has been used'
)
class Meta:
db_table = 'log_activity_tokens'
constraints = [
models.UniqueConstraint(fields=('version', 'user'),
name='version_id'),
models.UniqueConstraint(fields=('version', 'user'), name='version_id'),
]
def is_expired(self):
return self.use_count >= MAX_TOKEN_USE_COUNT
def is_valid(self):
return (not self.is_expired() and
self.version == self.version.addon.find_latest_version(
channel=self.version.channel, exclude=()))
return (
not self.is_expired()
and self.version
== self.version.addon.find_latest_version(
channel=self.version.channel, exclude=()
)
)
def expire(self):
self.update(use_count=MAX_TOKEN_USE_COUNT)
def increment_use(self):
self.__class__.objects.filter(pk=self.pk).update(
use_count=models.expressions.F('use_count') + 1)
use_count=models.expressions.F('use_count') + 1
)
self.use_count = self.use_count + 1
class ActivityLogEmails(ModelBase):
"""A log of message ids of incoming emails so we don't duplicate process
them."""
messageid = models.CharField(max_length=255, unique=True)
class Meta:
@ -88,6 +94,7 @@ class AddonLog(ModelBase):
"""
This table is for indexing the activity log by addon.
"""
addon = models.ForeignKey(Addon, on_delete=models.CASCADE)
activity_log = models.ForeignKey('ActivityLog', on_delete=models.CASCADE)
@ -101,8 +108,9 @@ class AddonLog(ModelBase):
# ``arguments = [{'addons.addon':12}, {'addons.addon':1}, ... ]``
arguments = json.loads(self.activity_log._arguments)
except Exception:
log.info('unserializing data from addon_log failed: %s' %
self.activity_log.id)
log.info(
'unserializing data from addon_log failed: %s' % self.activity_log.id
)
return None
new_arguments = []
@ -120,6 +128,7 @@ class CommentLog(ModelBase):
"""
This table is for indexing the activity log by comment.
"""
activity_log = models.ForeignKey('ActivityLog', on_delete=models.CASCADE)
comments = models.TextField()
@ -132,6 +141,7 @@ class VersionLog(ModelBase):
"""
This table is for indexing the activity log by version.
"""
activity_log = models.ForeignKey('ActivityLog', on_delete=models.CASCADE)
version = models.ForeignKey(Version, on_delete=models.CASCADE)
@ -145,6 +155,7 @@ class UserLog(ModelBase):
This table is for indexing the activity log by user.
Note: This includes activity performed unto the user.
"""
activity_log = models.ForeignKey('ActivityLog', on_delete=models.CASCADE)
user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
@ -157,6 +168,7 @@ class GroupLog(ModelBase):
"""
This table is for indexing the activity log by access group.
"""
id = PositiveAutoField(primary_key=True)
activity_log = models.ForeignKey('ActivityLog', on_delete=models.CASCADE)
group = models.ForeignKey(Group, on_delete=models.CASCADE)
@ -170,6 +182,7 @@ class BlockLog(ModelBase):
"""
This table is for indexing the activity log by Blocklist Block.
"""
id = PositiveAutoField(primary_key=True)
activity_log = models.ForeignKey('ActivityLog', on_delete=models.CASCADE)
block = models.ForeignKey(Block, on_delete=models.SET_NULL, null=True)
@ -186,14 +199,15 @@ class DraftComment(ModelBase):
This is being used by the commenting API by the code-manager.
"""
id = PositiveAutoField(primary_key=True)
version = models.ForeignKey(Version, on_delete=models.CASCADE)
user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
filename = models.CharField(max_length=255, null=True, blank=True)
lineno = models.PositiveIntegerField(null=True)
canned_response = models.ForeignKey(
CannedResponse, null=True, default=None,
on_delete=models.SET_DEFAULT)
CannedResponse, null=True, default=None, on_delete=models.SET_DEFAULT
)
comment = models.TextField(blank=True)
class Meta:
@ -241,8 +255,10 @@ class ActivityLogManager(ManagerBase):
return self.filter(blocklog__guid=guid)
def for_developer(self):
return self.exclude(action__in=constants.activity.LOG_ADMINS +
constants.activity.LOG_HIDE_DEVELOPER)
return self.exclude(
action__in=constants.activity.LOG_ADMINS
+ constants.activity.LOG_HIDE_DEVELOPER
)
def admin_events(self):
return self.filter(action__in=constants.activity.LOG_ADMINS)
@ -252,45 +268,55 @@ class ActivityLogManager(ManagerBase):
def review_queue(self):
qs = self._by_type()
return (qs.filter(action__in=constants.activity.LOG_REVIEW_QUEUE)
.exclude(user__id=settings.TASK_USER_ID))
return qs.filter(action__in=constants.activity.LOG_REVIEW_QUEUE).exclude(
user__id=settings.TASK_USER_ID
)
def review_log(self):
qs = self._by_type()
return (
qs.filter(action__in=constants.activity.LOG_REVIEWER_REVIEW_ACTION)
.exclude(user__id=settings.TASK_USER_ID))
return qs.filter(
action__in=constants.activity.LOG_REVIEWER_REVIEW_ACTION
).exclude(user__id=settings.TASK_USER_ID)
def total_ratings(self, theme=False):
"""Return the top users, and their # of reviews."""
qs = self._by_type()
action_ids = ([amo.LOG.THEME_REVIEW.id] if theme
else constants.activity.LOG_REVIEWER_REVIEW_ACTION)
return (qs.values('user', 'user__display_name', 'user__username')
.filter(action__in=action_ids)
.exclude(user__id=settings.TASK_USER_ID)
.annotate(approval_count=models.Count('id'))
.order_by('-approval_count'))
action_ids = (
[amo.LOG.THEME_REVIEW.id]
if theme
else constants.activity.LOG_REVIEWER_REVIEW_ACTION
)
return (
qs.values('user', 'user__display_name', 'user__username')
.filter(action__in=action_ids)
.exclude(user__id=settings.TASK_USER_ID)
.annotate(approval_count=models.Count('id'))
.order_by('-approval_count')
)
def monthly_reviews(self, theme=False):
"""Return the top users for the month, and their # of reviews."""
qs = self._by_type()
now = datetime.now()
created_date = datetime(now.year, now.month, 1)
actions = ([constants.activity.LOG.THEME_REVIEW.id] if theme
else constants.activity.LOG_REVIEWER_REVIEW_ACTION)
return (qs.values('user', 'user__display_name', 'user__username')
.filter(created__gte=created_date,
action__in=actions)
.exclude(user__id=settings.TASK_USER_ID)
.annotate(approval_count=models.Count('id'))
.order_by('-approval_count'))
actions = (
[constants.activity.LOG.THEME_REVIEW.id]
if theme
else constants.activity.LOG_REVIEWER_REVIEW_ACTION
)
return (
qs.values('user', 'user__display_name', 'user__username')
.filter(created__gte=created_date, action__in=actions)
.exclude(user__id=settings.TASK_USER_ID)
.annotate(approval_count=models.Count('id'))
.order_by('-approval_count')
)
def user_approve_reviews(self, user):
qs = self._by_type()
return qs.filter(
action__in=constants.activity.LOG_REVIEWER_REVIEW_ACTION,
user__id=user.id)
action__in=constants.activity.LOG_REVIEWER_REVIEW_ACTION, user__id=user.id
)
def current_month_user_approve_reviews(self, user):
now = datetime.now()
@ -299,8 +325,14 @@ class ActivityLogManager(ManagerBase):
def user_position(self, values_qs, user):
try:
return next(i for (i, d) in enumerate(list(values_qs))
if d.get('user') == user.id) + 1
return (
next(
i
for (i, d) in enumerate(list(values_qs))
if d.get('user') == user.id
)
+ 1
)
except StopIteration:
return None
@ -314,9 +346,8 @@ class ActivityLogManager(ManagerBase):
qs = self.get_queryset()
table = 'log_activity_addon'
return qs.extra(
tables=[table],
where=['%s.activity_log_id=%s.id'
% (table, 'log_activity')])
tables=[table], where=['%s.activity_log_id=%s.id' % (table, 'log_activity')]
)
class SafeFormatter(string.Formatter):
@ -330,10 +361,9 @@ class SafeFormatter(string.Formatter):
class ActivityLog(ModelBase):
TYPES = sorted(
[(value.id, key)
for key, value in constants.activity.LOG_BY_ID.items()])
user = models.ForeignKey(
'users.UserProfile', null=True, on_delete=models.SET_NULL)
[(value.id, key) for key, value in constants.activity.LOG_BY_ID.items()]
)
user = models.ForeignKey('users.UserProfile', null=True, on_delete=models.SET_NULL)
action = models.SmallIntegerField(choices=TYPES)
_arguments = models.TextField(blank=True, db_column='arguments')
_details = models.TextField(blank=True, db_column='details')
@ -375,9 +405,7 @@ class ActivityLog(ModelBase):
# `[{'addons.addon':12}, {'addons.addon':1}, ... ]`
activity.arguments_data = json.loads(activity._arguments)
except Exception as e:
log.info(
'unserializing data from activity_log failed: %s',
activity.id)
log.info('unserializing data from activity_log failed: %s', activity.id)
log.info(e)
activity.arguments_data = []
@ -499,31 +527,36 @@ class ActivityLog(ModelBase):
for arg in self.arguments:
if isinstance(arg, Addon) and not addon:
if arg.has_listed_versions():
addon = self.f(u'<a href="{0}">{1}</a>',
arg.get_url_path(), arg.name)
addon = self.f(
u'<a href="{0}">{1}</a>', arg.get_url_path(), arg.name
)
else:
addon = self.f(u'{0}', arg.name)
arguments.remove(arg)
if isinstance(arg, Rating) and not rating:
rating = self.f(u'<a href="{0}">{1}</a>',
arg.get_url_path(), ugettext('Review'))
rating = self.f(
u'<a href="{0}">{1}</a>', arg.get_url_path(), ugettext('Review')
)
arguments.remove(arg)
if isinstance(arg, Version) and not version:
text = ugettext('Version {0}')
if arg.channel == amo.RELEASE_CHANNEL_LISTED:
version = self.f(u'<a href="{1}">%s</a>' % text,
arg.version, arg.get_url_path())
version = self.f(
u'<a href="{1}">%s</a>' % text, arg.version, arg.get_url_path()
)
else:
version = self.f(text, arg.version)
arguments.remove(arg)
if isinstance(arg, Collection) and not collection:
collection = self.f(u'<a href="{0}">{1}</a>',
arg.get_url_path(), arg.name)
collection = self.f(
u'<a href="{0}">{1}</a>', arg.get_url_path(), arg.name
)
arguments.remove(arg)
if isinstance(arg, Tag) and not tag:
if arg.can_reverse():
tag = self.f(u'<a href="{0}">{1}</a>',
arg.get_url_path(), arg.tag_text)
tag = self.f(
u'<a href="{0}">{1}</a>', arg.get_url_path(), arg.tag_text
)
else:
tag = self.f('{0}', arg.tag_text)
if isinstance(arg, Group) and not group:
@ -532,17 +565,19 @@ class ActivityLog(ModelBase):
if isinstance(arg, File) and not file_:
validation = 'passed'
if self.action in (
amo.LOG.UNLISTED_SIGNED.id,
amo.LOG.UNLISTED_SIGNED_VALIDATION_FAILED.id):
amo.LOG.UNLISTED_SIGNED.id,
amo.LOG.UNLISTED_SIGNED_VALIDATION_FAILED.id,
):
validation = 'ignored'
file_ = self.f(u'<a href="{0}">{1}</a> (validation {2})',
arg.get_url_path(),
arg.filename,
validation)
file_ = self.f(
u'<a href="{0}">{1}</a> (validation {2})',
arg.get_url_path(),
arg.filename,
validation,
)
arguments.remove(arg)
if (self.action == amo.LOG.CHANGE_STATUS.id and
not isinstance(arg, Addon)):
if self.action == amo.LOG.CHANGE_STATUS.id and not isinstance(arg, Addon):
# Unfortunately, this action has been abused in the past and
# the non-addon argument could be a string or an int. If it's
# an int, we want to retrieve the string and translate it.
@ -609,8 +644,8 @@ class ActivityLog(ModelBase):
# creating a new one, especially useful for log entries created
# in a loop.
al = ActivityLog(
user=user, action=action.id,
created=kw.get('created', timezone.now()))
user=user, action=action.id, created=kw.get('created', timezone.now())
)
al.set_arguments(args)
if 'details' in kw:
al.details = kw['details']
@ -618,8 +653,10 @@ class ActivityLog(ModelBase):
if 'details' in kw and 'comments' in al.details:
CommentLog.objects.create(
comments=al.details['comments'], activity_log=al,
created=kw.get('created', timezone.now()))
comments=al.details['comments'],
activity_log=al,
created=kw.get('created', timezone.now()),
)
for arg in args:
if isinstance(arg, tuple):
@ -631,27 +668,38 @@ class ActivityLog(ModelBase):
if class_ == Addon:
AddonLog.objects.create(
addon_id=id_, activity_log=al,
created=kw.get('created', timezone.now()))
addon_id=id_,
activity_log=al,
created=kw.get('created', timezone.now()),
)
elif class_ == Version:
VersionLog.objects.create(
version_id=id_, activity_log=al,
created=kw.get('created', timezone.now()))
version_id=id_,
activity_log=al,
created=kw.get('created', timezone.now()),
)
elif class_ == UserProfile:
UserLog.objects.create(
user_id=id_, activity_log=al,
created=kw.get('created', timezone.now()))
user_id=id_,
activity_log=al,
created=kw.get('created', timezone.now()),
)
elif class_ == Group:
GroupLog.objects.create(
group_id=id_, activity_log=al,
created=kw.get('created', timezone.now()))
group_id=id_,
activity_log=al,
created=kw.get('created', timezone.now()),
)
elif class_ == Block:
BlockLog.objects.create(
block_id=id_, activity_log=al, guid=arg.guid,
created=kw.get('created', timezone.now()))
block_id=id_,
activity_log=al,
guid=arg.guid,
created=kw.get('created', timezone.now()),
)
# Index by every user
UserLog.objects.create(
activity_log=al, user=user,
created=kw.get('created', timezone.now()))
activity_log=al, user=user, created=kw.get('created', timezone.now())
)
return al

17
src/olympia/activity/serializers.py
Просмотреть файл

@ -16,8 +16,15 @@ class ActivityLogSerializer(serializers.ModelSerializer):
class Meta:
model = ActivityLog
fields = ('id', 'action', 'action_label', 'comments', 'user', 'date',
'highlight')
fields = (
'id',
'action',
'action_label',
'comments',
'user',
'date',
'highlight',
)
def __init__(self, *args, **kwargs):
super(ActivityLogSerializer, self).__init__(*args, **kwargs)
@ -50,9 +57,5 @@ class ActivityLogSerializer(serializers.ModelSerializer):
}
request = self.context.get('request')
if request and is_gate_active(request, 'activity-user-shim'):
data.update({
'id': None,
'username': None,
'url': None
})
data.update({'id': None, 'username': None, 'url': None})
return data

18
src/olympia/activity/tasks.py
Просмотреть файл

@ -21,19 +21,19 @@ def process_email(message, spam_rating, **kwargs):
if header.get('Name', '').lower() == 'message-id':
msg_id = header.get('Value')
if not msg_id:
log.error('No MessageId in message, aborting.', extra={
'message_obj': message
})
log.error('No MessageId in message, aborting.', extra={'message_obj': message})
return
_, created = ActivityLogEmails.objects.get_or_create(messageid=msg_id)
if not created:
log.warning('Already processed email [%s], skipping', msg_id, extra={
'message_obj': message
})
log.warning(
'Already processed email [%s], skipping',
msg_id,
extra={'message_obj': message},
)
return
res = add_email_to_activity_log_wrapper(message, spam_rating)
if not res:
log.error('Failed to process email [%s].', msg_id, extra={
'message_obj': message
})
log.error(
'Failed to process email [%s].', msg_id, extra={'message_obj': message}
)

38
src/olympia/activity/tests/test_commands.py
Просмотреть файл

@ -11,28 +11,36 @@ from olympia.amo.tests import TestCase, addon_factory, user_factory
class TestRepudiateActivityLogToken(TestCase):
def setUp(self):
addon = addon_factory()
self.version = addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
self.version = addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
self.token1 = ActivityLogToken.objects.create(
uuid='5a0b8a83d501412589cc5d562334b46b',
version=self.version, user=user_factory())
version=self.version,
user=user_factory(),
)
self.token2 = ActivityLogToken.objects.create(
uuid='8a0b8a834e71412589cc5d562334b46b',
version=self.version, user=user_factory())
version=self.version,
user=user_factory(),
)
self.token3 = ActivityLogToken.objects.create(
uuid='336ae924bc23804cef345d562334b46b',
version=self.version, user=user_factory())
version=self.version,
user=user_factory(),
)
addon2 = addon_factory()
addon2_version = addon2.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
addon2_version = addon2.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
self.token_diff_version = ActivityLogToken.objects.create(
uuid='470023efdac5730773340eaf3080b589',
version=addon2_version, user=user_factory())
version=addon2_version,
user=user_factory(),
)
def test_with_tokens(self):
call_command('repudiate_token',
'5a0b8a83d501412589cc5d562334b46b',
'8a0b8a834e71412589cc5d562334b46b')
call_command(
'repudiate_token',
'5a0b8a83d501412589cc5d562334b46b',
'8a0b8a834e71412589cc5d562334b46b',
)
assert self.token1.reload().is_expired()
assert self.token2.reload().is_expired()
assert not self.token3.reload().is_expired()
@ -46,9 +54,11 @@ class TestRepudiateActivityLogToken(TestCase):
assert not self.token_diff_version.reload().is_expired()
def test_with_token_and_version_ignores_version(self):
call_command('repudiate_token',
'5a0b8a83d501412589cc5d562334b46b',
version_id=self.version.id)
call_command(
'repudiate_token',
'5a0b8a83d501412589cc5d562334b46b',
version_id=self.version.id,
)
assert self.token1.reload().is_expired() # token supplied is expired.
assert not self.token2.reload().is_expired() # version supplied isn't.
assert not self.token3.reload().is_expired() # check the others too.

157
src/olympia/activity/tests/test_models.py
Просмотреть файл

@ -6,11 +6,14 @@ from pyquery import PyQuery as pq
from olympia import amo, core
from olympia.activity.models import (
MAX_TOKEN_USE_COUNT, ActivityLog, ActivityLogToken, AddonLog,
DraftComment)
MAX_TOKEN_USE_COUNT,
ActivityLog,
ActivityLogToken,
AddonLog,
DraftComment,
)
from olympia.addons.models import Addon, AddonUser
from olympia.amo.tests import (
TestCase, addon_factory, user_factory, version_factory)
from olympia.amo.tests import TestCase, addon_factory, user_factory, version_factory
from olympia.bandwagon.models import Collection
from olympia.ratings.models import Rating
from olympia.reviewers.models import CannedResponse
@ -24,11 +27,13 @@ class TestActivityLogToken(TestCase):
super(TestActivityLogToken, self).setUp()
self.addon = addon_factory()
self.version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
channel=amo.RELEASE_CHANNEL_LISTED
)
self.version.update(created=self.days_ago(1))
self.user = user_factory()
self.token = ActivityLogToken.objects.create(
version=self.version, user=self.user)
version=self.version, user=self.user
)
def test_uuid_is_automatically_created(self):
assert self.token.uuid
@ -45,7 +50,8 @@ class TestActivityLogToken(TestCase):
assert self.token.is_expired()
# But the version is still the latest version.
assert self.version == self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
channel=amo.RELEASE_CHANNEL_LISTED
)
assert not self.token.is_valid()
def test_increment_use(self):
@ -53,7 +59,8 @@ class TestActivityLogToken(TestCase):
self.token.increment_use()
assert self.token.use_count == 1
token_from_db = ActivityLogToken.objects.get(
version=self.version, user=self.user)
version=self.version, user=self.user
)
assert token_from_db.use_count == 1
def test_validity_version_out_of_date(self):
@ -66,7 +73,8 @@ class TestActivityLogToken(TestCase):
def test_validity_still_valid_if_new_version_in_different_channel(self):
version_factory(addon=self.addon, channel=amo.RELEASE_CHANNEL_UNLISTED)
assert self.version == self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
channel=amo.RELEASE_CHANNEL_LISTED
)
# The token isn't expired.
assert not self.token.is_expired()
@ -86,8 +94,7 @@ class TestActivityLog(TestCase):
def setUp(self):
super(TestActivityLog, self).setUp()
self.user = UserProfile.objects.create(
username='yolo', display_name='Yolo')
self.user = UserProfile.objects.create(username='yolo', display_name='Yolo')
self.request = Mock()
self.request.user = self.user
core.set_user(self.user)
@ -176,13 +183,15 @@ class TestActivityLog(TestCase):
def test_arguments_old_reviews_app(self):
addon = Addon.objects.get()
rating = Rating.objects.create(
addon=addon, user=self.user, user_responsible=self.user, rating=5)
addon=addon, user=self.user, user_responsible=self.user, rating=5
)
activity = ActivityLog.objects.latest('pk')
# Override _arguments to use reviews.review instead of ratings.rating,
# as old data already present in the db would use.
activity._arguments = (
u'[{"addons.addon": %d}, {"reviews.review": %d}]' % (
addon.pk, rating.pk))
activity._arguments = u'[{"addons.addon": %d}, {"reviews.review": %d}]' % (
addon.pk,
rating.pk,
)
assert activity.arguments == [addon, rating]
def test_no_arguments(self):
@ -208,8 +217,9 @@ class TestActivityLog(TestCase):
"""
user = UserProfile(username='Marlboro Manatee')
user.save()
ActivityLog.create(amo.LOG.ADD_USER_WITH_ROLE,
user, 'developer', Addon.objects.get())
ActivityLog.create(
amo.LOG.ADD_USER_WITH_ROLE, user, 'developer', Addon.objects.get()
)
entries = ActivityLog.objects.for_user(self.request.user)
assert len(entries) == 1
entries = ActivityLog.objects.for_user(user)
@ -217,8 +227,9 @@ class TestActivityLog(TestCase):
def test_version_log(self):
version = Version.objects.all()[0]
ActivityLog.create(amo.LOG.REJECT_VERSION, version.addon, version,
user=self.request.user)
ActivityLog.create(
amo.LOG.REJECT_VERSION, version.addon, version, user=self.request.user
)
entries = ActivityLog.objects.for_versions(version)
assert len(entries) == 1
assert version.get_url_path() in str(entries[0])
@ -229,8 +240,8 @@ class TestActivityLog(TestCase):
addon_factory() # To create an extra unrelated version
for version in Version.objects.all():
ActivityLog.create(
amo.LOG.REJECT_VERSION, version.addon, version,
user=self.request.user)
amo.LOG.REJECT_VERSION, version.addon, version, user=self.request.user
)
entries = ActivityLog.objects.for_versions(addon.versions.all())
assert len(entries) == 2
@ -239,8 +250,9 @@ class TestActivityLog(TestCase):
# Get the url before the addon is changed to unlisted.
url_path = version.get_url_path()
self.make_addon_unlisted(version.addon)
ActivityLog.create(amo.LOG.REJECT_VERSION, version.addon, version,
user=self.request.user)
ActivityLog.create(
amo.LOG.REJECT_VERSION, version.addon, version, user=self.request.user
)
entries = ActivityLog.objects.for_versions(version)
assert len(entries) == 1
assert url_path not in str(entries[0])
@ -248,18 +260,22 @@ class TestActivityLog(TestCase):
def test_version_log_transformer(self):
addon = Addon.objects.get()
version = addon.current_version
ActivityLog.create(amo.LOG.REJECT_VERSION, addon, version,
user=self.request.user)
ActivityLog.create(
amo.LOG.REJECT_VERSION, addon, version, user=self.request.user
)
version_two = Version(addon=addon, license=version.license,
version='1.2.3')
version_two = Version(addon=addon, license=version.license, version='1.2.3')
version_two.save()
ActivityLog.create(amo.LOG.REJECT_VERSION, addon, version_two,
user=self.request.user)
ActivityLog.create(
amo.LOG.REJECT_VERSION, addon, version_two, user=self.request.user
)
versions = (Version.objects.filter(addon=addon).order_by('-created')
.transform(Version.transformer_activity))
versions = (
Version.objects.filter(addon=addon)
.order_by('-created')
.transform(Version.transformer_activity)
)
assert len(versions[0].all_activity) == 1
assert len(versions[1].all_activity) == 1
@ -271,17 +287,18 @@ class TestActivityLog(TestCase):
addon = addon.reload()
au = AddonUser(addon=addon, user=self.user)
ActivityLog.create(
amo.LOG.CHANGE_USER_WITH_ROLE, au.user,
str(au.get_role_display()), addon)
amo.LOG.CHANGE_USER_WITH_ROLE, au.user, str(au.get_role_display()), addon
)
log = ActivityLog.objects.get()
log_expected = ('Yolo role changed to Owner for <a href="/en-US/'
'firefox/addon/a3615/">Delicious &lt;script src='
'&#34;x.js&#34;&gt;Bookmarks</a>.')
log_expected = (
'Yolo role changed to Owner for <a href="/en-US/'
'firefox/addon/a3615/">Delicious &lt;script src='
'&#34;x.js&#34;&gt;Bookmarks</a>.'
)
assert log.to_string() == log_expected
rendered = amo.utils.from_string('<p>{{ log }}</p>').render(
{'log': log})
rendered = amo.utils.from_string('<p>{{ log }}</p>').render({'log': log})
assert rendered == '<p>%s</p>' % log_expected
def test_tag_no_match(self):
@ -295,42 +312,53 @@ class TestActivityLog(TestCase):
def test_change_status(self):
addon = Addon.objects.get()
log = ActivityLog.create(
amo.LOG.CHANGE_STATUS, addon, amo.STATUS_APPROVED)
expected = ('<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to Approved.')
log = ActivityLog.create(amo.LOG.CHANGE_STATUS, addon, amo.STATUS_APPROVED)
expected = (
'<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to Approved.'
)
assert str(log) == expected
log.arguments = [amo.STATUS_DISABLED, addon]
expected = ('<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to '
'Disabled by Mozilla.')
expected = (
'<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to '
'Disabled by Mozilla.'
)
assert str(log) == expected
log.arguments = [addon, amo.STATUS_NULL]
expected = ('<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to Incomplete.')
expected = (
'<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to Incomplete.'
)
assert str(log) == expected
log.arguments = [addon, 666]
expected = ('<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to 666.')
expected = (
'<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to 666.'
)
assert str(log) == expected
log.arguments = [addon, 'Some String']
expected = ('<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to Some String.')
expected = (
'<a href="/en-US/firefox/addon/a3615/">'
'Delicious Bookmarks</a> status changed to Some String.'
)
assert str(log) == expected
def test_str_activity_file(self):
addon = Addon.objects.get()
log = ActivityLog.create(
amo.LOG.UNLISTED_SIGNED, addon.current_version.current_file)
amo.LOG.UNLISTED_SIGNED, addon.current_version.current_file
)
assert str(log) == (
'<a href="/firefox/downloads/file/67442/'
'delicious_bookmarks-2.1.072-fx.xpi">'
'delicious_bookmarks-2.1.072-fx.xpi</a>'
' (validation ignored) was signed.')
' (validation ignored) was signed.'
)
class TestActivityLogCount(TestCase):
@ -366,8 +394,7 @@ class TestActivityLogCount(TestCase):
assert result[0]['approval_count'] == 5
def test_review_last_month(self):
log = ActivityLog.create(amo.LOG.APPROVE_VERSION,
Addon.objects.get())
log = ActivityLog.create(amo.LOG.APPROVE_VERSION, Addon.objects.get())
log.update(created=self.lm)
assert len(ActivityLog.objects.monthly_reviews()) == 0
@ -382,8 +409,7 @@ class TestActivityLogCount(TestCase):
assert result[0]['approval_count'] == 5
def test_total_last_month(self):
log = ActivityLog.create(amo.LOG.APPROVE_VERSION,
Addon.objects.get())
log = ActivityLog.create(amo.LOG.APPROVE_VERSION, Addon.objects.get())
log.update(created=self.lm)
result = ActivityLog.objects.total_ratings()
assert len(result) == 1
@ -415,8 +441,7 @@ class TestActivityLogCount(TestCase):
assert result == 3
result = ActivityLog.objects.user_approve_reviews(other).count()
assert result == 2
another = UserProfile.objects.create(
email="no@mtrala.la", username="a")
another = UserProfile.objects.create(email="no@mtrala.la", username="a")
result = ActivityLog.objects.user_approve_reviews(another).count()
assert result == 0
@ -425,7 +450,8 @@ class TestActivityLogCount(TestCase):
ActivityLog.objects.update(created=self.days_ago(40))
self.add_approve_logs(2)
result = ActivityLog.objects.current_month_user_approve_reviews(
self.user).count()
self.user
).count()
assert result == 2
def test_log_admin(self):
@ -440,14 +466,12 @@ class TestActivityLogCount(TestCase):
class TestDraftComment(TestCase):
def test_default_requirements(self):
addon = addon_factory()
user = user_factory()
# user and version are the absolute minimum required to
# create a DraftComment
comment = DraftComment.objects.create(
user=user, version=addon.current_version)
comment = DraftComment.objects.create(user=user, version=addon.current_version)
assert comment.user == user
assert comment.version == addon.current_version
@ -464,11 +488,12 @@ class TestDraftComment(TestCase):
name=u'Terms of services',
response=u'test',
category=amo.CANNED_RESPONSE_CATEGORY_OTHER,
type=amo.CANNED_RESPONSE_TYPE_ADDON)
type=amo.CANNED_RESPONSE_TYPE_ADDON,
)
DraftComment.objects.create(
user=user, version=addon.current_version,
canned_response=canned_response)
user=user, version=addon.current_version, canned_response=canned_response
)
canned_response.delete()

18
src/olympia/activity/tests/test_serializers.py
Просмотреть файл

@ -9,10 +9,8 @@ from olympia.amo.tests import TestCase, addon_factory, user_factory
class LogMixin(object):
def log(self, comments, action, created=None):
version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
details = {'comments': comments,
'version': version.version}
version = self.addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
details = {'comments': comments, 'version': version.version}
kwargs = {'user': self.user, 'details': details}
al = ActivityLog.create(action, self.addon, version, **kwargs)
if created:
@ -96,19 +94,21 @@ class TestReviewNotesSerializerOutput(TestCase, LogMixin):
self.entry = self.log(u'ßäď ŞŤųƒƒ', amo.LOG.REQUEST_ADMIN_REVIEW_CODE)
result = self.serialize()
assert result['action_label'] == (
amo.LOG.REQUEST_ADMIN_REVIEW_CODE.short)
assert result['action_label'] == (amo.LOG.REQUEST_ADMIN_REVIEW_CODE.short)
# Comments should be the santized text rather than the actual content.
assert result['comments'] == amo.LOG.REQUEST_ADMIN_REVIEW_CODE.sanitize
assert result['comments'].startswith(
'The addon has been flagged for Admin Review.')
'The addon has been flagged for Admin Review.'
)
def test_log_entry_without_details(self):
# Create a log but without a details property.
self.entry = ActivityLog.create(
amo.LOG.APPROVAL_NOTES_CHANGED, self.addon,
amo.LOG.APPROVAL_NOTES_CHANGED,
self.addon,
self.addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED),
user=self.user)
user=self.user,
)
result = self.serialize()
# Should output an empty string.
assert result['comments'] == ''

15
src/olympia/activity/tests/test_tasks.py
Просмотреть файл

@ -15,8 +15,7 @@ def test_process_email(_mock):
message = sample_message_content.get('Message')
process_email(message, 0)
assert _mock.call_count == 1
assert ActivityLogEmails.objects.filter(
messageid='This is a MessageID').exists()
assert ActivityLogEmails.objects.filter(messageid='This is a MessageID').exists()
# don't try to process the same message twice
process_email(message, 0)
assert _mock.call_count == 1
@ -27,11 +26,9 @@ def test_process_email(_mock):
@mock.patch('olympia.activity.tasks.add_email_to_activity_log_wrapper')
def test_process_email_different_messageid(_mock):
# Test 'Message-ID' works too.
message = {'CustomHeaders': [
{'Name': 'Message-ID', 'Value': '<gmail_tastic>'}]}
message = {'CustomHeaders': [{'Name': 'Message-ID', 'Value': '<gmail_tastic>'}]}
process_email(message, 0)
assert ActivityLogEmails.objects.filter(
messageid='<gmail_tastic>').exists()
assert ActivityLogEmails.objects.filter(messageid='<gmail_tastic>').exists()
assert _mock.call_count == 1
# don't try to process the same message twice
process_email(message, 0)
@ -43,11 +40,9 @@ def test_process_email_different_messageid(_mock):
@mock.patch('olympia.activity.tasks.add_email_to_activity_log_wrapper')
def test_process_email_different_messageid_case(_mock):
# Test 'Message-Id' (different case)
message = {'CustomHeaders': [
{'Name': 'Message-Id', 'Value': '<its_ios>'}]}
message = {'CustomHeaders': [{'Name': 'Message-Id', 'Value': '<its_ios>'}]}
process_email(message, 0)
assert ActivityLogEmails.objects.filter(
messageid='<its_ios>').exists()
assert ActivityLogEmails.objects.filter(messageid='<its_ios>').exists()
assert _mock.call_count == 1
# don't try to process the same message twice
process_email(message, 0)

264
src/olympia/activity/tests/test_utils.py
Просмотреть файл

@ -16,14 +16,21 @@ from waffle.testutils import override_switch
from olympia import amo
from olympia.access.models import Group, GroupUser
from olympia.activity.models import (
MAX_TOKEN_USE_COUNT, ActivityLog, ActivityLogToken)
from olympia.activity.models import MAX_TOKEN_USE_COUNT, ActivityLog, ActivityLogToken
from olympia.activity.utils import (
ACTIVITY_MAIL_GROUP, ActivityEmailEncodingError, ActivityEmailError,
ActivityEmailParser, ActivityEmailTokenError, ActivityEmailUUIDError,
ACTIVITY_MAIL_GROUP,
ActivityEmailEncodingError,
ActivityEmailError,
ActivityEmailParser,
ActivityEmailTokenError,
ActivityEmailUUIDError,
add_email_to_activity_log,
add_email_to_activity_log_wrapper, log_and_notify,
notify_about_activity_log, NOTIFICATIONS_FROM_EMAIL, send_activity_mail)
add_email_to_activity_log_wrapper,
log_and_notify,
notify_about_activity_log,
NOTIFICATIONS_FROM_EMAIL,
send_activity_mail,
)
from olympia.amo.templatetags.jinja_helpers import absolutify
from olympia.amo.tests import TestCase, addon_factory, user_factory
from olympia.amo.urlresolvers import reverse
@ -39,8 +46,7 @@ class TestEmailParser(TestCase):
def test_basic_email(self):
parser = ActivityEmailParser(sample_message_content['Message'])
assert parser.get_uuid() == '5a0b8a83d501412589cc5d562334b46b'
assert parser.reply == (
'This is a developer reply to an AMO. It\'s nice.')
assert parser.reply == ('This is a developer reply to an AMO. It\'s nice.')
def test_with_invalid_msg(self):
with self.assertRaises(ActivityEmailEncodingError):
@ -79,11 +85,15 @@ class TestEmailBouncing(TestCase):
BOUNCE_REPLY = (
'Hello,\n\nAn email was received, apparently from you. Unfortunately '
'we couldn\'t process it because of:\n%s\n\nPlease visit %s to leave '
'a reply instead.\n--\nMozilla Add-ons\n%s\n')
'a reply instead.\n--\nMozilla Add-ons\n%s\n'
)
def setUp(self):
self.bounce_reply = (
self.BOUNCE_REPLY % ('%s', settings.SITE_URL, settings.SITE_URL))
self.bounce_reply = self.BOUNCE_REPLY % (
'%s',
settings.SITE_URL,
settings.SITE_URL,
)
self.email_text = sample_message_content['Message']
@mock.patch('olympia.activity.utils.ActivityLog.create')
@ -94,8 +104,8 @@ class TestEmailBouncing(TestCase):
user = user_factory()
self.grant_permission(user, '*:*')
ActivityLogToken.objects.create(
user=user, version=version,
uuid='5a0b8a83d501412589cc5d562334b46b')
user=user, version=version, uuid='5a0b8a83d501412589cc5d562334b46b'
)
# Make log_mock return false for some reason.
log_mock.return_value = False
@ -103,8 +113,7 @@ class TestEmailBouncing(TestCase):
assert not add_email_to_activity_log_wrapper(self.email_text, 0)
assert len(mail.outbox) == 1
out = mail.outbox[0]
assert out.body == (
self.bounce_reply % 'Undefined Error.')
assert out.body == (self.bounce_reply % 'Undefined Error.')
assert out.subject == 'Re: This is the subject of a test message.'
assert out.to == ['sender@example.com']
@ -114,25 +123,29 @@ class TestEmailBouncing(TestCase):
assert len(mail.outbox) == 1
out = mail.outbox[0]
assert out.body == (
self.bounce_reply %
'UUID found in email address TO: header but is not a valid token '
'(5a0b8a83d501412589cc5d562334b46b).')
self.bounce_reply
% 'UUID found in email address TO: header but is not a valid token '
'(5a0b8a83d501412589cc5d562334b46b).'
)
assert out.subject == 'Re: This is the subject of a test message.'
assert out.to == ['sender@example.com']
def test_exception_because_invalid_email(self):
# Fails because the token doesn't exist in ActivityToken.objects
email_text = copy.deepcopy(self.email_text)
email_text['To'] = [{
'EmailAddress': 'foobar@addons.mozilla.org',
'FriendlyName': 'not a valid activity mail reply'}]
email_text['To'] = [
{
'EmailAddress': 'foobar@addons.mozilla.org',
'FriendlyName': 'not a valid activity mail reply',
}
]
assert not add_email_to_activity_log_wrapper(email_text, 0)
assert len(mail.outbox) == 1
out = mail.outbox[0]
assert out.body == (
self.bounce_reply %
'TO: address does not contain activity email uuid ('
'foobar@addons.mozilla.org).')
self.bounce_reply % 'TO: address does not contain activity email uuid ('
'foobar@addons.mozilla.org).'
)
assert out.subject == 'Re: This is the subject of a test message.'
assert out.to == ['sender@example.com']
@ -145,31 +158,38 @@ class TestEmailBouncing(TestCase):
assert not add_email_to_activity_log_wrapper(message, 0)
assert len(mail.outbox) == 1
assert mail.outbox[0].body == (
self.bounce_reply % 'Invalid or malformed json message object.')
self.bounce_reply % 'Invalid or malformed json message object.'
)
assert mail.outbox[0].subject == 'Re: your email to us'
assert mail.outbox[0].to == ['bob@dole.org']
def test_exception_in_parser_but_from_defined(self):
"""Unlikely scenario of an email missing a body but having a From."""
self._test_exception_in_parser_but_can_send_email(
{'From': {'EmailAddress': 'bob@dole.org'}})
{'From': {'EmailAddress': 'bob@dole.org'}}
)
def test_exception_in_parser_but_reply_to_defined(self):
"""Even more unlikely scenario of an email missing a body but having a
ReplyTo."""
self._test_exception_in_parser_but_can_send_email(
{'ReplyTo': {'EmailAddress': 'bob@dole.org'}})
{'ReplyTo': {'EmailAddress': 'bob@dole.org'}}
)
def test_exception_to_notifications_alias(self):
email_text = copy.deepcopy(self.email_text)
email_text['To'] = [{
'EmailAddress': 'notifications@%s' % settings.INBOUND_EMAIL_DOMAIN,
'FriendlyName': 'not a valid activity mail reply'}]
email_text['To'] = [
{
'EmailAddress': 'notifications@%s' % settings.INBOUND_EMAIL_DOMAIN,
'FriendlyName': 'not a valid activity mail reply',
}
]
assert not add_email_to_activity_log_wrapper(email_text, 0)
assert len(mail.outbox) == 1
out = mail.outbox[0]
assert ('This email address is not meant to receive emails '
'directly.') in out.body
assert (
'This email address is not meant to receive emails ' 'directly.'
) in out.body
assert out.subject == 'Re: This is the subject of a test message.'
assert out.to == ['sender@example.com']
@ -195,11 +215,9 @@ class TestEmailBouncing(TestCase):
class TestAddEmailToActivityLog(TestCase):
def setUp(self):
self.addon = addon_factory(name='Badger', status=amo.STATUS_NOMINATED)
version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
version = self.addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
self.profile = user_factory()
self.token = ActivityLogToken.objects.create(
version=version, user=self.profile)
self.token = ActivityLogToken.objects.create(version=version, user=self.profile)
self.token.update(uuid='5a0b8a83d501412589cc5d562334b46b')
self.parser = ActivityEmailParser(sample_message_content['Message'])
@ -238,8 +256,7 @@ class TestAddEmailToActivityLog(TestCase):
assert not add_email_to_activity_log(self.parser)
def test_broken_token(self):
parser = ActivityEmailParser(
copy.deepcopy(sample_message_content['Message']))
parser = ActivityEmailParser(copy.deepcopy(sample_message_content['Message']))
parser.email['To'][0]['EmailAddress'] = 'reviewreply+1234@foo.bar'
with self.assertRaises(ActivityEmailUUIDError):
assert not add_email_to_activity_log(parser)
@ -252,17 +269,16 @@ class TestAddEmailToActivityLog(TestCase):
class TestLogAndNotify(TestCase):
def setUp(self):
self.developer = user_factory()
self.developer2 = user_factory()
self.reviewer = user_factory(reviewer_name='Revîewer')
self.grant_permission(self.reviewer, 'Addons:Review',
'Addon Reviewers')
self.grant_permission(self.reviewer, 'Addons:Review', 'Addon Reviewers')
self.addon = addon_factory()
self.version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
channel=amo.RELEASE_CHANNEL_LISTED
)
self.addon.addonuser_set.create(user=self.developer)
self.addon.addonuser_set.create(user=self.developer2)
self.task_user = user_factory(id=settings.TASK_USER_ID)
@ -271,9 +287,11 @@ class TestLogAndNotify(TestCase):
author = author or self.reviewer
details = {
'comments': u'I spy, with my líttle €ye...',
'version': self.version.version}
'version': self.version.version,
}
activity = ActivityLog.create(
action, self.addon, self.version, user=author, details=details)
action, self.addon, self.version, user=author, details=details
)
activity.update(created=self.days_ago(1))
return activity
@ -290,7 +308,9 @@ class TestLogAndNotify(TestCase):
subject = call[0][0]
body = call[0][1]
assert subject == u'Mozilla Add-ons: %s %s' % (
self.addon.name, self.version.version)
self.addon.name,
self.version.version,
)
assert ('visit %s' % url) in body
assert ('receiving this email because %s' % reason_text) in body
assert 'If we do not hear from you within' not in body
@ -311,8 +331,7 @@ class TestLogAndNotify(TestCase):
assert logs[0].details['comments'] == u'Thïs is á reply'
assert send_mail_mock.call_count == 2 # One author, one reviewer.
sender = formataddr(
(self.developer.name, NOTIFICATIONS_FROM_EMAIL))
sender = formataddr((self.developer.name, NOTIFICATIONS_FROM_EMAIL))
assert sender == send_mail_mock.call_args_list[0][1]['from_email']
recipients = self._recipients(send_mail_mock)
assert len(recipients) == 2
@ -324,15 +343,18 @@ class TestLogAndNotify(TestCase):
self._check_email(
send_mail_mock.call_args_list[0],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.')
'you are listed as an author of this add-on.',
)
review_url = absolutify(
reverse('reviewers.review',
kwargs={'addon_id': self.version.addon.pk,
'channel': 'listed'},
add_prefix=False))
reverse(
'reviewers.review',
kwargs={'addon_id': self.version.addon.pk, 'channel': 'listed'},
add_prefix=False,
)
)
self._check_email(
send_mail_mock.call_args_list[1],
review_url, 'you reviewed this add-on.')
send_mail_mock.call_args_list[1], review_url, 'you reviewed this add-on.'
)
@mock.patch('olympia.activity.utils.send_mail')
def test_reviewer_reply(self, send_mail_mock):
@ -349,8 +371,7 @@ class TestLogAndNotify(TestCase):
assert logs[0].details['comments'] == u'Thîs ïs a revïewer replyîng'
assert send_mail_mock.call_count == 2 # Both authors.
sender = formataddr(
(self.reviewer.reviewer_name, NOTIFICATIONS_FROM_EMAIL))
sender = formataddr((self.reviewer.reviewer_name, NOTIFICATIONS_FROM_EMAIL))
assert sender == send_mail_mock.call_args_list[0][1]['from_email']
recipients = self._recipients(send_mail_mock)
assert len(recipients) == 2
@ -362,11 +383,13 @@ class TestLogAndNotify(TestCase):
self._check_email(
send_mail_mock.call_args_list[0],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.')
'you are listed as an author of this add-on.',
)
self._check_email(
send_mail_mock.call_args_list[1],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.')
'you are listed as an author of this add-on.',
)
@mock.patch('olympia.activity.utils.send_mail')
def test_log_with_no_comment(self, send_mail_mock):
@ -374,31 +397,30 @@ class TestLogAndNotify(TestCase):
self._create(amo.LOG.REJECT_VERSION, self.reviewer)
action = amo.LOG.APPROVAL_NOTES_CHANGED
log_and_notify(
action=action, comments=None, note_creator=self.developer,
version=self.version)
action=action,
comments=None,
note_creator=self.developer,
version=self.version,
)
logs = ActivityLog.objects.filter(action=action.id)
assert len(logs) == 1
assert not logs[0].details # No details json because no comment.
assert send_mail_mock.call_count == 2 # One author, one reviewer.
sender = formataddr(
(self.developer.name, NOTIFICATIONS_FROM_EMAIL))
sender = formataddr((self.developer.name, NOTIFICATIONS_FROM_EMAIL))
assert sender == send_mail_mock.call_args_list[0][1]['from_email']
recipients = self._recipients(send_mail_mock)
assert len(recipients) == 2
assert self.reviewer.email in recipients
assert self.developer2.email in recipients
assert u'Approval notes changed' in (
send_mail_mock.call_args_list[0][0][1])
assert u'Approval notes changed' in (
send_mail_mock.call_args_list[1][0][1])
assert u'Approval notes changed' in (send_mail_mock.call_args_list[0][0][1])
assert u'Approval notes changed' in (send_mail_mock.call_args_list[1][0][1])
def test_staff_cc_group_is_empty_no_failure(self):
Group.objects.create(name=ACTIVITY_MAIL_GROUP, rules='None:None')
log_and_notify(amo.LOG.REJECT_VERSION, u'á', self.reviewer,
self.version)
log_and_notify(amo.LOG.REJECT_VERSION, u'á', self.reviewer, self.version)
@mock.patch('olympia.activity.utils.send_mail')
def test_staff_cc_group_get_mail(self, send_mail_mock):
@ -411,21 +433,24 @@ class TestLogAndNotify(TestCase):
assert len(logs) == 1
recipients = self._recipients(send_mail_mock)
sender = formataddr(
(self.developer.name, NOTIFICATIONS_FROM_EMAIL))
sender = formataddr((self.developer.name, NOTIFICATIONS_FROM_EMAIL))
assert sender == send_mail_mock.call_args_list[0][1]['from_email']
assert len(recipients) == 2
# self.reviewers wasn't on the thread, but gets an email anyway.
assert self.reviewer.email in recipients
assert self.developer2.email in recipients
review_url = absolutify(
reverse('reviewers.review',
kwargs={'addon_id': self.version.addon.pk,
'channel': 'listed'},
add_prefix=False))
self._check_email(send_mail_mock.call_args_list[1],
review_url,
'you are member of the activity email cc group.')
reverse(
'reviewers.review',
kwargs={'addon_id': self.version.addon.pk, 'channel': 'listed'},
add_prefix=False,
)
)
self._check_email(
send_mail_mock.call_args_list[1],
review_url,
'you are member of the activity email cc group.',
)
@mock.patch('olympia.activity.utils.send_mail')
def test_task_user_doesnt_get_mail(self, send_mail_mock):
@ -450,8 +475,9 @@ class TestLogAndNotify(TestCase):
"""If a reviewer has now left the team don't email them."""
self._create(amo.LOG.REJECT_VERSION, self.reviewer)
# Take his joob!
GroupUser.objects.get(group=Group.objects.get(name='Addon Reviewers'),
user=self.reviewer).delete()
GroupUser.objects.get(
group=Group.objects.get(name='Addon Reviewers'), user=self.reviewer
).delete()
action = amo.LOG.DEVELOPER_REPLY_VERSION
comments = u'Thïs is á reply'
@ -487,20 +513,26 @@ class TestLogAndNotify(TestCase):
# The developer who sent it doesn't get their email back.
assert self.developer.email not in recipients
self._check_email(send_mail_mock.call_args_list[0],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.')
self._check_email(
send_mail_mock.call_args_list[0],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.',
)
review_url = absolutify(
reverse('reviewers.review', add_prefix=False,
kwargs={'channel': 'listed', 'addon_id': self.addon.pk}))
self._check_email(send_mail_mock.call_args_list[1],
review_url, 'you reviewed this add-on.')
reverse(
'reviewers.review',
add_prefix=False,
kwargs={'channel': 'listed', 'addon_id': self.addon.pk},
)
)
self._check_email(
send_mail_mock.call_args_list[1], review_url, 'you reviewed this add-on.'
)
@mock.patch('olympia.activity.utils.send_mail')
def test_review_url_unlisted(self, send_mail_mock):
self.version.update(channel=amo.RELEASE_CHANNEL_UNLISTED)
self.grant_permission(self.reviewer, 'Addons:ReviewUnlisted',
'Addon Reviewers')
self.grant_permission(self.reviewer, 'Addons:ReviewUnlisted', 'Addon Reviewers')
# One from the reviewer.
self._create(amo.LOG.COMMENT_VERSION, self.reviewer)
@ -523,14 +555,21 @@ class TestLogAndNotify(TestCase):
# The developer who sent it doesn't get their email back.
assert self.developer.email not in recipients
self._check_email(send_mail_mock.call_args_list[0],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.')
self._check_email(
send_mail_mock.call_args_list[0],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.',
)
review_url = absolutify(
reverse('reviewers.review', add_prefix=False,
kwargs={'channel': 'unlisted', 'addon_id': self.addon.pk}))
self._check_email(send_mail_mock.call_args_list[1],
review_url, 'you reviewed this add-on.')
reverse(
'reviewers.review',
add_prefix=False,
kwargs={'channel': 'unlisted', 'addon_id': self.addon.pk},
)
)
self._check_email(
send_mail_mock.call_args_list[1], review_url, 'you reviewed this add-on.'
)
@mock.patch('olympia.activity.utils.send_mail')
def test_from_name_escape(self, send_mail_mock):
@ -543,7 +582,8 @@ class TestLogAndNotify(TestCase):
log_and_notify(action, comments, self.reviewer, self.version)
sender = r'"mr \"quote\" escape" <notifications@%s>' % (
settings.INBOUND_EMAIL_DOMAIN)
settings.INBOUND_EMAIL_DOMAIN
)
assert sender == send_mail_mock.call_args_list[0][1]['from_email']
@mock.patch('olympia.activity.utils.send_mail')
@ -566,8 +606,7 @@ class TestLogAndNotify(TestCase):
assert ActivityLog.objects.count() == 1 # No new activity created.
assert send_mail_mock.call_count == 2 # Both authors.
sender = formataddr((
self.reviewer.reviewer_name, NOTIFICATIONS_FROM_EMAIL))
sender = formataddr((self.reviewer.reviewer_name, NOTIFICATIONS_FROM_EMAIL))
assert sender == send_mail_mock.call_args_list[0][1]['from_email']
recipients = self._recipients(send_mail_mock)
assert len(recipients) == 2
@ -579,11 +618,13 @@ class TestLogAndNotify(TestCase):
self._check_email(
send_mail_mock.call_args_list[0],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.')
'you are listed as an author of this add-on.',
)
self._check_email(
send_mail_mock.call_args_list[1],
absolutify(self.addon.get_dev_url('versions')),
'you are listed as an author of this add-on.')
'you are listed as an author of this add-on.',
)
@pytest.mark.django_db
@ -591,25 +632,32 @@ def test_send_activity_mail():
subject = u'This ïs ã subject'
message = u'And... this ïs a messãge!'
addon = addon_factory()
latest_version = addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
latest_version = addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
user = user_factory()
recipients = [user, ]
recipients = [
user,
]
from_email = 'bob@bob.bob'
action = ActivityLog.create(amo.LOG.DEVELOPER_REPLY_VERSION, user=user)
send_activity_mail(
subject, message, latest_version, recipients, from_email, action.id)
subject, message, latest_version, recipients, from_email, action.id
)
assert len(mail.outbox) == 1
assert mail.outbox[0].body == message
assert mail.outbox[0].subject == subject
uuid = latest_version.token.get(user=user).uuid.hex
reference_header = '<{addon}/{version}@{site}>'.format(
addon=latest_version.addon.id, version=latest_version.id,
site=settings.INBOUND_EMAIL_DOMAIN)
addon=latest_version.addon.id,
version=latest_version.id,
site=settings.INBOUND_EMAIL_DOMAIN,
)
message_id = '<{addon}/{version}/{action}@{site}>'.format(
addon=latest_version.addon.id, version=latest_version.id,
action=action.id, site=settings.INBOUND_EMAIL_DOMAIN)
addon=latest_version.addon.id,
version=latest_version.id,
action=action.id,
site=settings.INBOUND_EMAIL_DOMAIN,
)
assert mail.outbox[0].extra_headers['In-Reply-To'] == reference_header
assert mail.outbox[0].extra_headers['References'] == reference_header

152
src/olympia/activity/tests/test_views.py
Просмотреть файл

@ -13,14 +13,21 @@ from olympia.activity.views import EmailCreationPermission, inbound_email
from olympia.addons.models import AddonUser, AddonRegionalRestrictions
from olympia.addons.utils import generate_addon_guid
from olympia.amo.tests import (
APITestClient, TestCase, addon_factory, req_factory_factory, reverse_ns,
user_factory, version_factory)
APITestClient,
TestCase,
addon_factory,
req_factory_factory,
reverse_ns,
user_factory,
version_factory,
)
from olympia.users.models import UserProfile
class ReviewNotesViewSetDetailMixin(LogMixin):
"""Tests that play with addon state and permissions. Shared between review
note viewset detail tests since both need to react the same way."""
def _test_url(self):
raise NotImplementedError
@ -139,25 +146,29 @@ class ReviewNotesViewSetDetailMixin(LogMixin):
def test_developer_geo_restricted(self):
AddonRegionalRestrictions.objects.create(
addon=self.addon, excluded_regions=['AB', 'CD'])
addon=self.addon, excluded_regions=['AB', 'CD']
)
self._login_developer()
response = self.client.get(self.url, HTTP_X_COUNTRY_CODE='fr')
assert response.status_code == 200
AddonRegionalRestrictions.objects.filter(
addon=self.addon).update(excluded_regions=['AB', 'CD', 'FR'])
AddonRegionalRestrictions.objects.filter(addon=self.addon).update(
excluded_regions=['AB', 'CD', 'FR']
)
response = self.client.get(self.url, HTTP_X_COUNTRY_CODE='fr')
assert response.status_code == 200
def test_reviewer_geo_restricted(self):
AddonRegionalRestrictions.objects.create(
addon=self.addon, excluded_regions=['AB', 'CD'])
addon=self.addon, excluded_regions=['AB', 'CD']
)
self._login_reviewer()
response = self.client.get(self.url, HTTP_X_COUNTRY_CODE='fr')
assert response.status_code == 200
AddonRegionalRestrictions.objects.filter(
addon=self.addon).update(excluded_regions=['AB', 'CD', 'FR'])
AddonRegionalRestrictions.objects.filter(addon=self.addon).update(
excluded_regions=['AB', 'CD', 'FR']
)
response = self.client.get(self.url, HTTP_X_COUNTRY_CODE='fr')
assert response.status_code == 200
@ -168,12 +179,13 @@ class TestReviewNotesViewSetDetail(ReviewNotesViewSetDetailMixin, TestCase):
def setUp(self):
super(TestReviewNotesViewSetDetail, self).setUp()
self.addon = addon_factory(
guid=generate_addon_guid(), name=u'My Addôn', slug='my-addon')
guid=generate_addon_guid(), name=u'My Addôn', slug='my-addon'
)
self.user = user_factory()
self.version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
self.note = self.log(u'noôo!', amo.LOG.REVIEWER_REPLY_VERSION,
self.days_ago(0))
channel=amo.RELEASE_CHANNEL_LISTED
)
self.note = self.log(u'noôo!', amo.LOG.REVIEWER_REPLY_VERSION, self.days_ago(0))
self._set_tested_url()
def _test_url(self):
@ -186,10 +198,14 @@ class TestReviewNotesViewSetDetail(ReviewNotesViewSetDetailMixin, TestCase):
assert result['highlight'] # Its the first reply so highlight
def _set_tested_url(self, pk=None, version_pk=None, addon_pk=None):
self.url = reverse_ns('version-reviewnotes-detail', kwargs={
'addon_pk': addon_pk or self.addon.pk,
'version_pk': version_pk or self.version.pk,
'pk': pk or self.note.pk})
self.url = reverse_ns(
'version-reviewnotes-detail',
kwargs={
'addon_pk': addon_pk or self.addon.pk,
'version_pk': version_pk or self.version.pk,
'pk': pk or self.note.pk,
},
)
def test_get_note_not_found(self):
self._login_reviewer(permission='*:*')
@ -204,22 +220,26 @@ class TestReviewNotesViewSetList(ReviewNotesViewSetDetailMixin, TestCase):
def setUp(self):
super(TestReviewNotesViewSetList, self).setUp()
self.addon = addon_factory(
guid=generate_addon_guid(), name=u'My Addôn', slug='my-addon')
guid=generate_addon_guid(), name=u'My Addôn', slug='my-addon'
)
self.user = user_factory()
self.note = self.log(u'noôo!', amo.LOG.APPROVE_VERSION,
self.days_ago(3))
self.note2 = self.log(u'réply!', amo.LOG.DEVELOPER_REPLY_VERSION,
self.days_ago(2))
self.note3 = self.log(u'yéss!', amo.LOG.REVIEWER_REPLY_VERSION,
self.days_ago(1))
self.note = self.log(u'noôo!', amo.LOG.APPROVE_VERSION, self.days_ago(3))
self.note2 = self.log(
u'réply!', amo.LOG.DEVELOPER_REPLY_VERSION, self.days_ago(2)
)
self.note3 = self.log(
u'yéss!', amo.LOG.REVIEWER_REPLY_VERSION, self.days_ago(1)
)
self.version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
channel=amo.RELEASE_CHANNEL_LISTED
)
self._set_tested_url()
def test_queries(self):
self.note4 = self.log(u'fiiiine', amo.LOG.REVIEWER_REPLY_VERSION,
self.days_ago(0))
self.note4 = self.log(
u'fiiiine', amo.LOG.REVIEWER_REPLY_VERSION, self.days_ago(0)
)
self._login_developer()
with self.assertNumQueries(17):
# - 2 savepoints because of tests
@ -259,9 +279,13 @@ class TestReviewNotesViewSetList(ReviewNotesViewSetDetailMixin, TestCase):
assert not result_version['highlight'] # The dev replied so read it.
def _set_tested_url(self, pk=None, version_pk=None, addon_pk=None):
self.url = reverse_ns('version-reviewnotes-list', kwargs={
'addon_pk': addon_pk or self.addon.pk,
'version_pk': version_pk or self.version.pk})
self.url = reverse_ns(
'version-reviewnotes-list',
kwargs={
'addon_pk': addon_pk or self.addon.pk,
'version_pk': version_pk or self.version.pk,
},
)
def test_admin_activity_hidden_from_developer(self):
# Add an extra activity note but a type we don't show the developer.
@ -277,19 +301,21 @@ class TestReviewNotesViewSetCreate(TestCase):
def setUp(self):
super(TestReviewNotesViewSetCreate, self).setUp()
self.addon = addon_factory(
guid=generate_addon_guid(), name=u'My Addôn', slug='my-addon')
guid=generate_addon_guid(), name=u'My Addôn', slug='my-addon'
)
self.version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
self.url = reverse_ns('version-reviewnotes-list', kwargs={
'addon_pk': self.addon.pk,
'version_pk': self.version.pk})
channel=amo.RELEASE_CHANNEL_LISTED
)
self.url = reverse_ns(
'version-reviewnotes-list',
kwargs={'addon_pk': self.addon.pk, 'version_pk': self.version.pk},
)
def _post_reply(self):
return self.client.post(self.url, {'comments': u'comménty McCómm€nt'})
def get_review_activity_queryset(self):
return ActivityLog.objects.filter(
action__in=amo.LOG_REVIEW_QUEUE_DEVELOPER)
return ActivityLog.objects.filter(action__in=amo.LOG_REVIEW_QUEUE_DEVELOPER)
def test_anonymous_is_401(self):
assert self._post_reply().status_code == 401
@ -317,8 +343,8 @@ class TestReviewNotesViewSetCreate(TestCase):
rdata = response.data
assert reply.pk == rdata['id']
assert (
str(reply.details['comments']) == rdata['comments'] ==
u'comménty McCómm€nt')
str(reply.details['comments']) == rdata['comments'] == u'comménty McCómm€nt'
)
assert reply.user == self.user
assert reply.user.name == rdata['user']['name'] == self.user.name
assert reply.action == amo.LOG.DEVELOPER_REPLY_VERSION.id
@ -343,8 +369,8 @@ class TestReviewNotesViewSetCreate(TestCase):
rdata = response.data
assert reply.pk == rdata['id']
assert (
str(reply.details['comments']) == rdata['comments'] ==
u'comménty McCómm€nt')
str(reply.details['comments']) == rdata['comments'] == u'comménty McCómm€nt'
)
assert reply.user == self.user
assert reply.user.name == rdata['user']['name'] == self.user.name
assert reply.action == amo.LOG.REVIEWER_REPLY_VERSION.id
@ -367,14 +393,14 @@ class TestReviewNotesViewSetCreate(TestCase):
assert not self.get_review_activity_queryset().exists()
def test_reply_to_deleted_version_is_400(self):
old_version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
old_version = self.addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
new_version = version_factory(addon=self.addon)
old_version.delete()
# Just in case, make sure the add-on is still public.
self.addon.reload()
assert new_version == self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
channel=amo.RELEASE_CHANNEL_LISTED
)
assert self.addon.status
self.user = user_factory()
@ -385,22 +411,22 @@ class TestReviewNotesViewSetCreate(TestCase):
assert not self.get_review_activity_queryset().exists()
def test_cant_reply_to_old_version(self):
old_version = self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
old_version = self.addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
old_version.update(created=self.days_ago(1))
new_version = version_factory(addon=self.addon)
assert new_version == self.addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
channel=amo.RELEASE_CHANNEL_LISTED
)
self.user = user_factory()
self.grant_permission(self.user, 'Addons:Review')
self.client.login_api(self.user)
# First check we can reply to new version
new_url = reverse_ns('version-reviewnotes-list', kwargs={
'addon_pk': self.addon.pk,
'version_pk': new_version.pk})
response = self.client.post(
new_url, {'comments': u'comménty McCómm€nt'})
new_url = reverse_ns(
'version-reviewnotes-list',
kwargs={'addon_pk': self.addon.pk, 'version_pk': new_version.pk},
)
response = self.client.post(new_url, {'comments': u'comménty McCómm€nt'})
assert response.status_code == 201
assert self.get_review_activity_queryset().count() == 1
@ -427,7 +453,6 @@ class TestReviewNotesViewSetCreate(TestCase):
@override_settings(INBOUND_EMAIL_SECRET_KEY='SOME SECRET KEY')
@override_settings(INBOUND_EMAIL_VALIDATION_KEY='validation key')
class TestEmailApi(TestCase):
def get_request(self, data):
# Request body should be a bytes string, so it needs to be encoded
# after having built the json representation of it, then fed into
@ -442,7 +467,8 @@ class TestEmailApi(TestCase):
def get_validation_request(self, data):
req = req_factory_factory(
url=reverse_ns('inbound-email-api'), post=True, data=data)
url=reverse_ns('inbound-email-api'), post=True, data=data
)
req.META['REMOTE_ADDR'] = '10.10.10.10'
return req
@ -450,13 +476,12 @@ class TestEmailApi(TestCase):
user = user_factory()
self.grant_permission(user, '*:*')
addon = addon_factory()
version = addon.find_latest_version(
channel=amo.RELEASE_CHANNEL_LISTED)
version = addon.find_latest_version(channel=amo.RELEASE_CHANNEL_LISTED)
req = self.get_request(sample_message_content)
ActivityLogToken.objects.create(
user=user, version=version,
uuid='5a0b8a83d501412589cc5d562334b46b')
user=user, version=version, uuid='5a0b8a83d501412589cc5d562334b46b'
)
res = inbound_email(req)
assert res.status_code == 201
@ -468,7 +493,8 @@ class TestEmailApi(TestCase):
def test_allowed(self):
assert EmailCreationPermission().has_permission(
self.get_request({'SecretKey': 'SOME SECRET KEY'}), None)
self.get_request({'SecretKey': 'SOME SECRET KEY'}), None
)
def test_ip_denied(self):
req = self.get_request({'SecretKey': 'SOME SECRET KEY'})
@ -486,8 +512,8 @@ class TestEmailApi(TestCase):
@mock.patch('olympia.activity.tasks.process_email.apply_async')
def test_successful(self, _mock):
req = self.get_request(
{'SecretKey': 'SOME SECRET KEY', 'Message': 'something',
'SpamScore': 4.56})
{'SecretKey': 'SOME SECRET KEY', 'Message': 'something', 'SpamScore': 4.56}
)
res = inbound_email(req)
_mock.assert_called_with(('something', 4.56))
assert res.status_code == 201
@ -502,7 +528,8 @@ class TestEmailApi(TestCase):
@mock.patch('olympia.activity.tasks.process_email.apply_async')
def test_validation_response(self, _mock):
req = self.get_validation_request(
{'SecretKey': 'SOME SECRET KEY', 'Type': 'Validation'})
{'SecretKey': 'SOME SECRET KEY', 'Type': 'Validation'}
)
res = inbound_email(req)
assert not _mock.called
assert res.status_code == 200
@ -512,7 +539,8 @@ class TestEmailApi(TestCase):
@mock.patch('olympia.activity.tasks.process_email.apply_async')
def test_validation_response_wrong_secret(self, _mock):
req = self.get_validation_request(
{'SecretKey': 'WRONG SECRET', 'Type': 'Validation'})
{'SecretKey': 'WRONG SECRET', 'Type': 'Validation'}
)
res = inbound_email(req)
assert not _mock.called
assert res.status_code == 403

232
src/olympia/activity/utils.py
Просмотреть файл

@ -57,17 +57,19 @@ class ActivityEmailToNotificationsError(ActivityEmailError):
class ActivityEmailParser(object):
"""Utility to parse email replies."""
address_prefix = REPLY_TO_PREFIX
def __init__(self, message):
invalid_email = (
not isinstance(message, dict) or
not message.get('TextBody', None))
invalid_email = not isinstance(message, dict) or not message.get(
'TextBody', None
)
if invalid_email:
log.exception('ActivityEmailParser didn\'t get a valid message.')
raise ActivityEmailEncodingError(
'Invalid or malformed json message object.')
'Invalid or malformed json message object.'
)
self.email = message
reply = self._extra_email_reply_parse(self.email['TextBody'])
@ -95,25 +97,27 @@ class ActivityEmailParser(object):
for address in addresses:
if address.startswith(self.address_prefix):
# Strip everything between "reviewreply+" and the "@" sign.
return address[len(self.address_prefix):].split('@')[0]
return address[len(self.address_prefix) :].split('@')[0]
elif address == NOTIFICATIONS_FROM_EMAIL:
# Someone sent an email to notifications@
to_notifications_alias = True
if to_notifications_alias:
log.exception('TO: notifications email used (%s)'
% ', '.join(addresses))
log.exception('TO: notifications email used (%s)' % ', '.join(addresses))
raise ActivityEmailToNotificationsError(
'This email address is not meant to receive emails directly. '
'If you want to get in contact with add-on reviewers, please '
'reply to the original email or join us in Matrix on '
'https://chat.mozilla.org/#/room/#addon-reviewers:mozilla.org '
'. Thank you.')
'. Thank you.'
)
log.debug(
'TO: address missing or not related to activity emails. (%s)',
', '.join(addresses))
', '.join(addresses),
)
raise ActivityEmailUUIDError(
'TO: address does not contain activity email uuid (%s).'
% ', '.join(addresses))
% ', '.join(addresses)
)
def add_email_to_activity_log_wrapper(message, spam_rating):
@ -137,8 +141,7 @@ def add_email_to_activity_log_wrapper(message, spam_rating):
except Exception:
log.exception('Bouncing invalid email failed.')
else:
log.info(
f'Skipping email bounce because probable spam ({spam_rating})')
log.info(f'Skipping email bounce because probable spam ({spam_rating})')
return note
@ -151,7 +154,8 @@ def add_email_to_activity_log(parser):
log.error('An email was skipped with non-existing uuid %s.' % uuid)
raise ActivityEmailUUIDError(
'UUID found in email address TO: header but is not a valid token '
'(%s).' % uuid)
'(%s).' % uuid
)
version = token.version
user = token.user
@ -161,31 +165,43 @@ def add_email_to_activity_log(parser):
if log_type:
note = log_and_notify(log_type, parser.reply, user, version)
log.info('A new note has been created (from %s using '
'tokenid %s).' % (user.id, uuid))
log.info(
'A new note has been created (from %s using '
'tokenid %s).' % (user.id, uuid)
)
token.increment_use()
return note
else:
log.error('%s did not have perms to reply to email thread %s.'
% (user.email, version.id))
log.error(
'%s did not have perms to reply to email thread %s.'
% (user.email, version.id)
)
raise ActivityEmailTokenError(
'You don\'t have permission to reply to this add-on. You '
'have to be a listed developer currently, or an AMO '
'reviewer.')
'reviewer.'
)
else:
log.warning('%s tried to use an invalid activity email token for '
'version %s.', user.email, version.id)
reason = ('it\'s for an old version of the addon'
if not token.is_expired() else
'there have been too many replies')
log.warning(
'%s tried to use an invalid activity email token for ' 'version %s.',
user.email,
version.id,
)
reason = (
'it\'s for an old version of the addon'
if not token.is_expired()
else 'there have been too many replies'
)
raise ActivityEmailTokenError(
'You can\'t reply to this email as the reply token is no '
'longer valid because %s.' % reason)
'longer valid because %s.' % reason
)
else:
log.info('Ignored email reply from banned user %s for version %s.'
% (user.id, version.id))
raise ActivityEmailError('Your account is not allowed to send '
'replies.')
log.info(
'Ignored email reply from banned user %s for version %s.'
% (user.id, version.id)
)
raise ActivityEmailError('Your account is not allowed to send ' 'replies.')
def action_from_user(user, version):
@ -197,16 +213,17 @@ def action_from_user(user, version):
def template_from_user(user, version):
template = 'activity/emails/developer.txt'
if (not version.addon.authors.filter(pk=user.pk).exists() and
acl.is_user_any_kind_of_reviewer(user)):
if not version.addon.authors.filter(
pk=user.pk
).exists() and acl.is_user_any_kind_of_reviewer(user):
template = 'activity/emails/from_reviewer.txt'
return loader.get_template(template)
def log_and_notify(action, comments, note_creator, version, perm_setting=None,
detail_kwargs=None):
"""Record an action through ActivityLog and notify relevant users about it.
"""
def log_and_notify(
action, comments, note_creator, version, perm_setting=None, detail_kwargs=None
):
"""Record an action through ActivityLog and notify relevant users about it."""
log_kwargs = {
'user': note_creator,
'created': datetime.now(),
@ -223,13 +240,13 @@ def log_and_notify(action, comments, note_creator, version, perm_setting=None,
if not note:
return
notify_about_activity_log(
version.addon, version, note, perm_setting=perm_setting)
notify_about_activity_log(version.addon, version, note, perm_setting=perm_setting)
return note
def notify_about_activity_log(addon, version, note, perm_setting=None,
send_to_reviewers=True, send_to_staff=True):
def notify_about_activity_log(
addon, version, note, perm_setting=None, send_to_reviewers=True, send_to_staff=True
):
"""Notify relevant users about an ActivityLog note."""
comments = (note.details or {}).get('comments')
if not comments:
@ -258,13 +275,21 @@ def notify_about_activity_log(addon, version, note, perm_setting=None,
# Not being localised because we don't know the recipients locale.
with translation.override('en-US'):
subject = reviewer_subject = u'Mozilla Add-ons: %s %s' % (
addon.name, version.version)
addon.name,
version.version,
)
# Build and send the mail for authors.
template = template_from_user(note.user, version)
from_email = formataddr((note.author_name, NOTIFICATIONS_FROM_EMAIL))
send_activity_mail(
subject, template.render(author_context_dict),
version, addon_authors, from_email, note.id, perm_setting)
subject,
template.render(author_context_dict),
version,
addon_authors,
from_email,
note.id,
perm_setting,
)
if send_to_reviewers or send_to_staff:
# If task_user doesn't exist that's no big issue (i.e. in tests)
@ -278,45 +303,63 @@ def notify_about_activity_log(addon, version, note, perm_setting=None,
# for automated messages), build the context for them and send them
# their copy.
log_users = {
alog.user for alog in ActivityLog.objects.for_versions(version) if
acl.is_user_any_kind_of_reviewer(alog.user)}
alog.user
for alog in ActivityLog.objects.for_versions(version)
if acl.is_user_any_kind_of_reviewer(alog.user)
}
reviewers = log_users - addon_authors - task_user - {note.user}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('reviewers.review',
kwargs={
'addon_id': version.addon.pk,
'channel': amo.CHANNEL_CHOICES_API[version.channel]
}, add_prefix=False))
reverse(
'reviewers.review',
kwargs={
'addon_id': version.addon.pk,
'channel': amo.CHANNEL_CHOICES_API[version.channel],
},
add_prefix=False,
)
)
reviewer_context_dict['email_reason'] = 'you reviewed this add-on'
send_activity_mail(
reviewer_subject, template.render(reviewer_context_dict),
version, reviewers, from_email, note.id, perm_setting)
reviewer_subject,
template.render(reviewer_context_dict),
version,
reviewers,
from_email,
note.id,
perm_setting,
)
if send_to_staff:
# Collect staff that want a copy of the email, build the context for
# them and send them their copy.
staff = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
staff_cc = (
staff - reviewers - addon_authors - task_user - {note.user})
staff = set(UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
staff_cc = staff - reviewers - addon_authors - task_user - {note.user}
staff_cc_context_dict = reviewer_context_dict.copy()
staff_cc_context_dict['email_reason'] = (
'you are member of the activity email cc group')
staff_cc_context_dict[
'email_reason'
] = 'you are member of the activity email cc group'
send_activity_mail(
reviewer_subject, template.render(staff_cc_context_dict),
version, staff_cc, from_email, note.id, perm_setting)
reviewer_subject,
template.render(staff_cc_context_dict),
version,
staff_cc,
from_email,
note.id,
perm_setting,
)
def send_activity_mail(subject, message, version, recipients, from_email,
unique_id, perm_setting=None):
thread_id = '{addon}/{version}'.format(
addon=version.addon.id, version=version.id)
def send_activity_mail(
subject, message, version, recipients, from_email, unique_id, perm_setting=None
):
thread_id = '{addon}/{version}'.format(addon=version.addon.id, version=version.id)
reference_header = '<{thread}@{site}>'.format(
thread=thread_id, site=settings.INBOUND_EMAIL_DOMAIN)
thread=thread_id, site=settings.INBOUND_EMAIL_DOMAIN
)
message_id = '<{thread}/{message}@{site}>'.format(
thread=thread_id, message=unique_id,
site=settings.INBOUND_EMAIL_DOMAIN)
thread=thread_id, message=unique_id, site=settings.INBOUND_EMAIL_DOMAIN
)
headers = {
'In-Reply-To': reference_header,
'References': reference_header,
@ -325,20 +368,33 @@ def send_activity_mail(subject, message, version, recipients, from_email,
for recipient in recipients:
token, created = ActivityLogToken.objects.get_or_create(
version=version, user=recipient)
version=version, user=recipient
)
if not created:
token.update(use_count=0)
else:
log.info('Created token with UUID %s for user: %s.' % (
token.uuid, recipient.id))
log.info(
'Created token with UUID %s for user: %s.' % (token.uuid, recipient.id)
)
reply_to = "%s%s@%s" % (
REPLY_TO_PREFIX, token.uuid.hex, settings.INBOUND_EMAIL_DOMAIN)
log.info('Sending activity email to %s for %s version %s' % (
recipient, version.addon.pk, version.pk))
REPLY_TO_PREFIX,
token.uuid.hex,
settings.INBOUND_EMAIL_DOMAIN,
)
log.info(
'Sending activity email to %s for %s version %s'
% (recipient, version.addon.pk, version.pk)
)
send_mail(
subject, message, recipient_list=[recipient.email],
from_email=from_email, use_deny_list=False, headers=headers,
perm_setting=perm_setting, reply_to=[reply_to])
subject,
message,
recipient_list=[recipient.email],
from_email=from_email,
use_deny_list=False,
headers=headers,
perm_setting=perm_setting,
reply_to=[reply_to],
)
NOT_PENDING_IDS = (
@ -353,26 +409,36 @@ NOT_PENDING_IDS = (
def filter_queryset_to_pending_replies(queryset, log_type_ids=NOT_PENDING_IDS):
latest_reply_date = queryset.filter(
action__in=log_type_ids).values_list('created', flat=True).first()
latest_reply_date = (
queryset.filter(action__in=log_type_ids)
.values_list('created', flat=True)
.first()
)
if not latest_reply_date:
return queryset
return queryset.filter(created__gt=latest_reply_date)
def bounce_mail(message, reason):
recipient = (None if not isinstance(message, dict)
else message.get('From', message.get('ReplyTo')))
recipient = (
None
if not isinstance(message, dict)
else message.get('From', message.get('ReplyTo'))
)
if not recipient:
log.error('Tried to bounce incoming activity mail but no From or '
'ReplyTo header present.')
log.error(
'Tried to bounce incoming activity mail but no From or '
'ReplyTo header present.'
)
return
body = (loader.get_template('activity/emails/bounce.txt').
render({'reason': reason, 'SITE_URL': settings.SITE_URL}))
body = loader.get_template('activity/emails/bounce.txt').render(
{'reason': reason, 'SITE_URL': settings.SITE_URL}
)
send_mail(
'Re: %s' % message.get('Subject', 'your email to us'),
body,
recipient_list=[recipient['EmailAddress']],
from_email=settings.ADDONS_EMAIL,
use_deny_list=False)
use_deny_list=False,
)

67
src/olympia/activity/views.py
Просмотреть файл

@ -7,7 +7,10 @@ from django.utils.translation import ugettext
from rest_framework import status
from rest_framework.decorators import (
api_view, authentication_classes, permission_classes)
api_view,
authentication_classes,
permission_classes,
)
from rest_framework.exceptions import ParseError
from rest_framework.mixins import ListModelMixin, RetrieveModelMixin
from rest_framework.response import Response
@ -20,14 +23,22 @@ from olympia.activity.models import ActivityLog
from olympia.activity.serializers import ActivityLogSerializer
from olympia.activity.tasks import process_email
from olympia.activity.utils import (
action_from_user, filter_queryset_to_pending_replies, log_and_notify)
action_from_user,
filter_queryset_to_pending_replies,
log_and_notify,
)
from olympia.addons.views import AddonChildMixin
from olympia.api.permissions import (
AllowAddonAuthor, AllowReviewer, AllowReviewerUnlisted, AnyOf)
AllowAddonAuthor,
AllowReviewer,
AllowReviewerUnlisted,
AnyOf,
)
class VersionReviewNotesViewSet(AddonChildMixin, ListModelMixin,
RetrieveModelMixin, GenericViewSet):
class VersionReviewNotesViewSet(
AddonChildMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet
):
permission_classes = [
AnyOf(AllowAddonAuthor, AllowReviewer, AllowReviewerUnlisted),
]
@ -39,8 +50,8 @@ class VersionReviewNotesViewSet(AddonChildMixin, ListModelMixin,
def get_addon_object(self):
return super(VersionReviewNotesViewSet, self).get_addon_object(
permission_classes=self.permission_classes,
georestriction_classes=[])
permission_classes=self.permission_classes, georestriction_classes=[]
)
def get_version_object(self):
if not hasattr(self, 'version_object'):
@ -48,9 +59,11 @@ class VersionReviewNotesViewSet(AddonChildMixin, ListModelMixin,
self.version_object = get_object_or_404(
# Fetch the version without transforms, using the addon related
# manager to avoid reloading it from the database.
addon.versions(
manager='unfiltered_for_relations').all().no_transforms(),
pk=self.kwargs['version_pk'])
addon.versions(manager='unfiltered_for_relations')
.all()
.no_transforms(),
pk=self.kwargs['version_pk'],
)
return self.version_object
def check_object_permissions(self, request, obj):
@ -61,20 +74,28 @@ class VersionReviewNotesViewSet(AddonChildMixin, ListModelMixin,
def get_serializer_context(self):
ctx = super(VersionReviewNotesViewSet, self).get_serializer_context()
ctx['to_highlight'] = list(filter_queryset_to_pending_replies(
self.get_queryset()).values_list('pk', flat=True))
ctx['to_highlight'] = list(
filter_queryset_to_pending_replies(self.get_queryset()).values_list(
'pk', flat=True
)
)
return ctx
def create(self, request, *args, **kwargs):
version = self.get_version_object()
latest_version = version.addon.find_latest_version(
channel=version.channel, exclude=())
channel=version.channel, exclude=()
)
if version != latest_version:
raise ParseError(ugettext(
'Only latest versions of addons can have notes added.'))
raise ParseError(
ugettext('Only latest versions of addons can have notes added.')
)
activity_object = log_and_notify(
action_from_user(request.user, version), request.data['comments'],
request.user, version)
action_from_user(request.user, version),
request.data['comments'],
request.user,
version,
)
serializer = self.get_serializer(activity_object)
return Response(serializer.data, status=status.HTTP_201_CREATED)
@ -95,8 +116,7 @@ class EmailCreationPermission(object):
secret_key = data.get('SecretKey', '')
if not secret_key == settings.INBOUND_EMAIL_SECRET_KEY:
log.info('Invalid secret key [%s] provided; data [%s]' % (
secret_key, data))
log.info('Invalid secret key [%s] provided; data [%s]' % (secret_key, data))
return False
remote_ip = request.META.get('REMOTE_ADDR', '')
@ -115,15 +135,12 @@ def inbound_email(request):
validation_response = settings.INBOUND_EMAIL_VALIDATION_KEY
if request.data.get('Type', '') == 'Validation':
# Its just a verification check that the end-point is working.
return Response(data=validation_response,
status=status.HTTP_200_OK)
return Response(data=validation_response, status=status.HTTP_200_OK)
message = request.data.get('Message', None)
if not message:
raise ParseError(
detail='Message not present in the POST data.')
raise ParseError(detail='Message not present in the POST data.')
spam_rating = request.data.get('SpamScore', 0.0)
process_email.apply_async((message, spam_rating))
return Response(data=validation_response,
status=status.HTTP_201_CREATED)
return Response(data=validation_response, status=status.HTTP_201_CREATED)

257
src/olympia/addons/admin.py
Просмотреть файл

@ -7,9 +7,11 @@ from django.conf.urls import url
from django.contrib import admin
from django.core import validators
from django.forms.models import modelformset_factory
from django.http.response import (HttpResponseForbidden,
HttpResponseNotAllowed,
HttpResponseRedirect)
from django.http.response import (
HttpResponseForbidden,
HttpResponseNotAllowed,
HttpResponseRedirect,
)
from django.shortcuts import get_object_or_404
from django.urls import resolve
from django.utils.encoding import force_text
@ -47,9 +49,11 @@ class AddonUserInline(admin.TabularInline):
return format_html(
'<a href="{}">Admin User Profile</a> ({})',
reverse('admin:users_userprofile_change', args=(obj.user.pk,)),
obj.user.email)
obj.user.email,
)
else:
return ''
user_profile_link.short_description = 'User Profile'
@ -66,8 +70,14 @@ class FileInline(admin.TabularInline):
extra = 0
max_num = 0
fields = (
'created', 'version__version', 'version__channel', 'platform',
'status', 'version__is_blocked', 'hash_link')
'created',
'version__version',
'version__channel',
'platform',
'status',
'version__is_blocked',
'hash_link',
)
editable_fields = ('status',)
readonly_fields = tuple(set(fields) - set(editable_fields))
can_delete = False
@ -76,12 +86,13 @@ class FileInline(admin.TabularInline):
checks_class = FileInlineChecks
def version__version(self, obj):
return obj.version.version + (
' - Deleted' if obj.version.deleted else '')
return obj.version.version + (' - Deleted' if obj.version.deleted else '')
version__version.short_description = 'Version'
def version__channel(self, obj):
return obj.version.get_channel_display()
version__channel.short_description = 'Channel'
def version__is_blocked(self, obj):
@ -91,12 +102,14 @@ class FileInline(admin.TabularInline):
url = block.get_admin_url_path()
template = '<a href="{}">Blocked ({} - {})</a>'
return format_html(template, url, block.min_version, block.max_version)
version__is_blocked.short_description = 'Block status'
def hash_link(self, obj):
url = reverse('zadmin.recalc_hash', args=(obj.id,))
template = '<a href="{}" class="recalc" title="{}">Recalc Hash</a>'
return format_html(template, url, obj.hash)
hash_link.short_description = 'Hash'
def get_formset(self, request, obj=None, **kwargs):
@ -117,64 +130,117 @@ class FileInline(admin.TabularInline):
def get_queryset(self, request):
self.pager = amo.utils.paginate(
request,
Version.unfiltered.filter(addon=self.instance).values_list(
'pk', flat=True),
30)
Version.unfiltered.filter(addon=self.instance).values_list('pk', flat=True),
30,
)
# A list coercion so this doesn't result in a subquery with a LIMIT
# which MySQL doesn't support (at this time).
versions = list(self.pager.object_list)
qs = super().get_queryset(request).filter(
version__in=versions).order_by('-version__id')
qs = (
super()
.get_queryset(request)
.filter(version__in=versions)
.order_by('-version__id')
)
return qs.select_related('version')
class AddonAdmin(admin.ModelAdmin):
class Media:
css = {
'all': ('css/admin/l10n.css', 'css/admin/pagination.css')
}
js = (
'admin/js/jquery.init.js', 'js/admin/l10n.js',
'js/admin/recalc_hash.js'
)
css = {'all': ('css/admin/l10n.css', 'css/admin/pagination.css')}
js = ('admin/js/jquery.init.js', 'js/admin/l10n.js', 'js/admin/recalc_hash.js')
exclude = ('authors',)
list_display = ('__str__', 'type', 'guid', 'status', 'average_rating',
'reviewer_links')
list_display = (
'__str__',
'type',
'guid',
'status',
'average_rating',
'reviewer_links',
)
list_filter = ('type', 'status')
search_fields = ('id', '^guid', '^slug')
inlines = (AddonUserInline, FileInline)
readonly_fields = ('id', 'created',
'average_rating', 'bayesian_rating', 'guid',
'total_ratings_link', 'text_ratings_count',
'weekly_downloads', 'average_daily_users')
readonly_fields = (
'id',
'created',
'average_rating',
'bayesian_rating',
'guid',
'total_ratings_link',
'text_ratings_count',
'weekly_downloads',
'average_daily_users',
)
fieldsets = (
(None, {
'fields': ('id', 'created', 'name', 'slug', 'guid',
'default_locale', 'type',
'status'),
}),
('Details', {
'fields': ('summary', 'description', 'homepage', 'eula',
'privacy_policy', 'developer_comments', 'icon_type',
),
}),
('Support', {
'fields': ('support_url', 'support_email'),
}),
('Stats', {
'fields': ('total_ratings_link', 'average_rating',
'bayesian_rating', 'text_ratings_count',
'weekly_downloads', 'average_daily_users'),
}),
('Flags', {
'fields': ('disabled_by_user', 'requires_payment',
'is_experimental', 'reputation'),
}),
('Dictionaries and Language Packs', {
'fields': ('target_locale',),
}))
(
None,
{
'fields': (
'id',
'created',
'name',
'slug',
'guid',
'default_locale',
'type',
'status',
),
},
),
(
'Details',
{
'fields': (
'summary',
'description',
'homepage',
'eula',
'privacy_policy',
'developer_comments',
'icon_type',
),
},
),
(
'Support',
{
'fields': ('support_url', 'support_email'),
},
),
(
'Stats',
{
'fields': (
'total_ratings_link',
'average_rating',
'bayesian_rating',
'text_ratings_count',
'weekly_downloads',
'average_daily_users',
),
},
),
(
'Flags',
{
'fields': (
'disabled_by_user',
'requires_payment',
'is_experimental',
'reputation',
),
},
),
(
'Dictionaries and Language Packs',
{
'fields': ('target_locale',),
},
),
)
actions = ['git_extract_action']
def queryset(self, request):
@ -184,20 +250,28 @@ class AddonAdmin(admin.ModelAdmin):
def wrap(view):
def wrapper(*args, **kwargs):
return self.admin_site.admin_view(view)(*args, **kwargs)
return functools.update_wrapper(wrapper, view)
urlpatterns = super(AddonAdmin, self).get_urls()
custom_urlpatterns = [
url(r'^(?P<object_id>.+)/git_extract/$',
url(
r'^(?P<object_id>.+)/git_extract/$',
wrap(self.git_extract_view),
name='addons_git_extract'),
name='addons_git_extract',
),
]
return custom_urlpatterns + urlpatterns
def total_ratings_link(self, obj):
return related_content_link(
obj, Rating, 'addon', related_manager='without_replies',
count=obj.total_ratings)
obj,
Rating,
'addon',
related_manager='without_replies',
count=obj.total_ratings,
)
total_ratings_link.short_description = _(u'Ratings')
def reviewer_links(self, obj):
@ -244,34 +318,42 @@ class AddonAdmin(admin.ModelAdmin):
url += '?' + request.GET.urlencode()
return http.HttpResponsePermanentRedirect(url)
return super().change_view(request, object_id, form_url,
extra_context=extra_context)
return super().change_view(
request, object_id, form_url, extra_context=extra_context
)
def render_change_form(self, request, context, add=False, change=False,
form_url='', obj=None):
def render_change_form(
self, request, context, add=False, change=False, form_url='', obj=None
):
context.update(
{
'external_site_url': settings.EXTERNAL_SITE_URL,
'has_listed_versions': obj.has_listed_versions(
include_deleted=True
) if obj else False,
'has_unlisted_versions': obj.has_unlisted_versions(
include_deleted=True
) if obj else False
'has_listed_versions': obj.has_listed_versions(include_deleted=True)
if obj
else False,
'has_unlisted_versions': obj.has_unlisted_versions(include_deleted=True)
if obj
else False,
}
)
return super().render_change_form(request=request, context=context,
add=add, change=change,
form_url=form_url, obj=obj)
return super().render_change_form(
request=request,
context=context,
add=add,
change=change,
form_url=form_url,
obj=obj,
)
def save_model(self, request, obj, form, change):
super().save_model(request, obj, form, change)
if 'status' in form.changed_data:
ActivityLog.create(
amo.LOG.CHANGE_STATUS, obj, form.cleaned_data['status'])
log.info('Addon "%s" status changed to: %s' % (
obj.slug, form.cleaned_data['status']))
ActivityLog.create(amo.LOG.CHANGE_STATUS, obj, form.cleaned_data['status'])
log.info(
'Addon "%s" status changed to: %s'
% (obj.slug, form.cleaned_data['status'])
)
def git_extract_action(self, request, qs):
addon_ids = []
@ -280,8 +362,9 @@ class AddonAdmin(admin.ModelAdmin):
addon_ids.append(force_text(addon))
kw = {'addons': ', '.join(addon_ids)}
self.message_user(
request, ugettext('Git extraction triggered for '
'"%(addons)s".' % kw))
request, ugettext('Git extraction triggered for ' '"%(addons)s".' % kw)
)
git_extract_action.short_description = "Git-Extract"
def git_extract_view(self, request, object_id, extra_context=None):
@ -296,7 +379,8 @@ class AddonAdmin(admin.ModelAdmin):
self.git_extract_action(request, (obj,))
return HttpResponseRedirect(
reverse('admin:addons_addon_change', args=(obj.pk, )))
reverse('admin:addons_addon_change', args=(obj.pk,))
)
class FrozenAddonAdmin(admin.ModelAdmin):
@ -313,7 +397,8 @@ class ReplacementAddonForm(forms.ModelForm):
if path.startswith(site):
raise forms.ValidationError(
'Paths for [%s] should be relative, not full URLs '
'including the domain name' % site)
'including the domain name' % site
)
validators.URLValidator()(path)
else:
path = ('/' if not path.startswith('/') else '') + path
@ -334,7 +419,8 @@ class ReplacementAddonAdmin(admin.ModelAdmin):
guid_param = urlencode({'guid': obj.guid})
return format_html(
'<a href="{}">Test</a>',
reverse('addons.find_replacement') + '?%s' % guid_param)
reverse('addons.find_replacement') + '?%s' % guid_param,
)
def guid_slug(self, obj):
try:
@ -354,12 +440,12 @@ class ReplacementAddonAdmin(admin.ModelAdmin):
# won't be able to make any changes but they can see the list.
if obj is not None:
return super(ReplacementAddonAdmin, self).has_change_permission(
request, obj=obj)
request, obj=obj
)
else:
return (
acl.action_allowed(request, amo.permissions.ADDONS_EDIT) or
super(ReplacementAddonAdmin, self).has_change_permission(
request, obj=obj))
return acl.action_allowed(request, amo.permissions.ADDONS_EDIT) or super(
ReplacementAddonAdmin, self
).has_change_permission(request, obj=obj)
@admin.register(models.AddonRegionalRestrictions)
@ -374,16 +460,19 @@ class AddonRegionalRestrictionsAdmin(admin.ModelAdmin):
def addon__name(self, obj):
return str(obj.addon)
addon__name.short_description = 'Addon'
def _send_mail(self, obj, action):
message = (
f'Regional restriction for addon "{obj.addon.name}" '
f'[{obj.addon.id}] {action}: {obj.excluded_regions}')
f'[{obj.addon.id}] {action}: {obj.excluded_regions}'
)
send_mail(
f'Regional Restriction {action} for Add-on',
message,
recipient_list=('amo-admins@mozilla.com',))
recipient_list=('amo-admins@mozilla.com',),
)
def delete_model(self, request, obj):
self._send_mail(obj, 'deleted')

59
src/olympia/addons/api_urls.py
Просмотреть файл

@ -6,9 +6,17 @@ from rest_framework_nested.routers import NestedSimpleRouter
from olympia.activity.views import VersionReviewNotesViewSet
from .views import (
AddonAutoCompleteSearchView, AddonFeaturedView, AddonRecommendationView,
AddonSearchView, AddonVersionViewSet, AddonViewSet, CompatOverrideView,
LanguageToolsView, ReplacementAddonView, StaticCategoryView)
AddonAutoCompleteSearchView,
AddonFeaturedView,
AddonRecommendationView,
AddonSearchView,
AddonVersionViewSet,
AddonViewSet,
CompatOverrideView,
LanguageToolsView,
ReplacementAddonView,
StaticCategoryView,
)
addons = SimpleRouter()
@ -18,32 +26,43 @@ addons.register(r'addon', AddonViewSet, basename='addon')
sub_addons = NestedSimpleRouter(addons, r'addon', lookup='addon')
sub_addons.register('versions', AddonVersionViewSet, basename='addon-version')
sub_versions = NestedSimpleRouter(sub_addons, r'versions', lookup='version')
sub_versions.register(r'reviewnotes', VersionReviewNotesViewSet,
basename='version-reviewnotes')
sub_versions.register(
r'reviewnotes', VersionReviewNotesViewSet, basename='version-reviewnotes'
)
urls = [
re_path(r'', include(addons.urls)),
re_path(r'', include(sub_addons.urls)),
re_path(r'', include(sub_versions.urls)),
re_path(r'^autocomplete/$', AddonAutoCompleteSearchView.as_view(),
name='addon-autocomplete'),
re_path(
r'^autocomplete/$',
AddonAutoCompleteSearchView.as_view(),
name='addon-autocomplete',
),
re_path(r'^search/$', AddonSearchView.as_view(), name='addon-search'),
re_path(r'^categories/$', StaticCategoryView.as_view(),
name='category-list'),
re_path(r'^language-tools/$', LanguageToolsView.as_view(),
name='addon-language-tools'),
re_path(r'^replacement-addon/$', ReplacementAddonView.as_view(),
name='addon-replacement-addon'),
re_path(r'^recommendations/$', AddonRecommendationView.as_view(),
name='addon-recommendations'),
re_path(r'^categories/$', StaticCategoryView.as_view(), name='category-list'),
re_path(
r'^language-tools/$', LanguageToolsView.as_view(), name='addon-language-tools'
),
re_path(
r'^replacement-addon/$',
ReplacementAddonView.as_view(),
name='addon-replacement-addon',
),
re_path(
r'^recommendations/$',
AddonRecommendationView.as_view(),
name='addon-recommendations',
),
]
addons_v3 = urls + [
re_path(r'^compat-override/$', CompatOverrideView.as_view(),
name='addon-compat-override'),
re_path(r'^featured/$', AddonFeaturedView.as_view(),
name='addon-featured'),
re_path(
r'^compat-override/$',
CompatOverrideView.as_view(),
name='addon-compat-override',
),
re_path(r'^featured/$', AddonFeaturedView.as_view(), name='addon-featured'),
]
addons_v4 = urls

65
src/olympia/addons/cron.py
Просмотреть файл

@ -11,7 +11,8 @@ from olympia.addons.tasks import (
update_addon_average_daily_users as _update_addon_average_daily_users,
update_addon_hotness as _update_addon_hotness,
update_addon_weekly_downloads as _update_addon_weekly_downloads,
update_appsupport)
update_appsupport,
)
from olympia.amo.celery import create_chunked_tasks_signatures
from olympia.amo.decorators import use_primary_db
from olympia.amo.utils import chunked
@ -19,7 +20,8 @@ from olympia.files.models import File
from olympia.stats.utils import (
get_addons_and_average_daily_users_from_bigquery,
get_addons_and_weekly_downloads_from_bigquery,
get_averages_by_addon_from_bigquery)
get_averages_by_addon_from_bigquery,
)
log = olympia.core.logger.getLogger('z.cron')
@ -32,8 +34,7 @@ def update_addon_average_daily_users(chunk_size=250):
# In order to reset the `average_daily_users` values of add-ons that
# don't exist in BigQuery, we prepare a set of `(guid, 0)` for most
# add-ons.
Addon.unfiltered
.filter(type__in=amo.ADDON_TYPES_WITH_STATS)
Addon.unfiltered.filter(type__in=amo.ADDON_TYPES_WITH_STATS)
.exclude(guid__isnull=True)
.exclude(guid__exact='')
.exclude(average_daily_users=0)
@ -47,8 +48,7 @@ def update_addon_average_daily_users(chunk_size=250):
counts.update(get_addons_and_average_daily_users_from_bigquery())
counts = list(counts.items())
log.info('Preparing update of `average_daily_users` for %s add-ons.',
len(counts))
log.info('Preparing update of `average_daily_users` for %s add-ons.', len(counts))
create_chunked_tasks_signatures(
_update_addon_average_daily_users, counts, chunk_size
@ -88,24 +88,28 @@ def addon_last_updated():
_change_last_updated(next)
# Get anything that didn't match above.
other = (Addon.objects.filter(last_updated__isnull=True)
.values_list('id', 'created'))
other = Addon.objects.filter(last_updated__isnull=True).values_list('id', 'created')
_change_last_updated(dict(other))
def update_addon_appsupport():
# Find all the add-ons that need their app support details updated.
newish = (Q(last_updated__gte=F('appsupport__created')) |
Q(appsupport__created__isnull=True))
newish = Q(last_updated__gte=F('appsupport__created')) | Q(
appsupport__created__isnull=True
)
has_app_and_file = Q(
versions__apps__isnull=False,
versions__files__status__in=amo.VALID_FILE_STATUSES)
ids = (Addon.objects.valid().distinct()
.filter(newish, has_app_and_file).values_list('id', flat=True))
versions__files__status__in=amo.VALID_FILE_STATUSES,
)
ids = (
Addon.objects.valid()
.distinct()
.filter(newish, has_app_and_file)
.values_list('id', flat=True)
)
task_log.info('Updating appsupport for %d new-ish addons.' % len(ids))
ts = [update_appsupport.subtask(args=[chunk])
for chunk in chunked(ids, 20)]
ts = [update_appsupport.subtask(args=[chunk]) for chunk in chunked(ids, 20)]
group(ts).apply_async()
@ -117,10 +121,11 @@ def hide_disabled_files():
See also unhide_disabled_files().
"""
ids = (File.objects.filter(
Q(version__addon__status=amo.STATUS_DISABLED) |
Q(version__addon__disabled_by_user=True) |
Q(status=amo.STATUS_DISABLED)).values_list('id', flat=True))
ids = File.objects.filter(
Q(version__addon__status=amo.STATUS_DISABLED)
| Q(version__addon__disabled_by_user=True)
| Q(status=amo.STATUS_DISABLED)
).values_list('id', flat=True)
for chunk in chunked(ids, 300):
qs = File.objects.select_related('version').filter(id__in=chunk)
for file_ in qs:
@ -137,10 +142,11 @@ def unhide_disabled_files():
See also hide_disabled_files().
"""
ids = (File.objects.exclude(
Q(version__addon__status=amo.STATUS_DISABLED) |
Q(version__addon__disabled_by_user=True) |
Q(status=amo.STATUS_DISABLED)).values_list('id', flat=True))
ids = File.objects.exclude(
Q(version__addon__status=amo.STATUS_DISABLED)
| Q(version__addon__disabled_by_user=True)
| Q(status=amo.STATUS_DISABLED)
).values_list('id', flat=True)
for chunk in chunked(ids, 300):
qs = File.objects.select_related('version').filter(id__in=chunk)
for file_ in qs:
@ -172,17 +178,14 @@ def update_addon_hotness(chunk_size=300):
.values_list('guid', flat=True)
)
averages = {
guid: {'avg_this_week': 1, 'avg_three_weeks_before': 1}
for guid in amo_guids
guid: {'avg_this_week': 1, 'avg_three_weeks_before': 1} for guid in amo_guids
}
log.info('Found %s add-on GUIDs in AMO DB.', len(averages))
bq_averages = get_averages_by_addon_from_bigquery(
today=date.today(), exclude=frozen_guids
)
log.info(
'Found %s add-on GUIDs with averages in BigQuery.', len(bq_averages)
)
log.info('Found %s add-on GUIDs with averages in BigQuery.', len(bq_averages))
averages.update(bq_averages)
log.info('Preparing update of `hotness` for %s add-ons.', len(averages))
@ -200,8 +203,7 @@ def update_addon_weekly_downloads(chunk_size=250):
# In order to reset the `weekly_downloads` values of add-ons that
# don't exist in BigQuery, we prepare a set of `(hashed_guid, 0)`
# for most add-ons.
Addon.objects
.filter(type__in=amo.ADDON_TYPES_WITH_STATS)
Addon.objects.filter(type__in=amo.ADDON_TYPES_WITH_STATS)
.exclude(guid__isnull=True)
.exclude(guid__exact='')
.exclude(weekly_downloads=0)
@ -212,8 +214,7 @@ def update_addon_weekly_downloads(chunk_size=250):
counts.update(get_addons_and_weekly_downloads_from_bigquery())
counts = list(counts.items())
log.info('Preparing update of `weekly_downloads` for %s add-ons.',
len(counts))
log.info('Preparing update of `weekly_downloads` for %s add-ons.', len(counts))
create_chunked_tasks_signatures(
_update_addon_weekly_downloads, counts, chunk_size

22
src/olympia/addons/decorators.py
Просмотреть файл

@ -8,15 +8,16 @@ from olympia.addons.models import Addon
def owner_or_unlisted_reviewer(request, addon):
return (acl.check_unlisted_addons_reviewer(request) or
# We don't want "admins" here, because it includes anyone with the
# "Addons:Edit" perm, we only want those with
# "Addons:ReviewUnlisted" perm (which is checked above).
acl.check_addon_ownership(request, addon, admin=False, dev=True))
return (
acl.check_unlisted_addons_reviewer(request)
# We don't want "admins" here, because it includes anyone with the
# "Addons:Edit" perm, we only want those with
# "Addons:ReviewUnlisted" perm (which is checked above).
or acl.check_addon_ownership(request, addon, admin=False, dev=True)
)
def addon_view(
f, qs=Addon.objects.all, include_deleted_when_checking_versions=False):
def addon_view(f, qs=Addon.objects.all, include_deleted_when_checking_versions=False):
@functools.wraps(f)
def wrapper(request, addon_id=None, *args, **kw):
"""Provides an addon instance to the view given addon_id, which can be
@ -44,11 +45,12 @@ def addon_view(
# If the addon has no listed versions it needs either an author
# (owner/viewer/dev/support) or an unlisted addon reviewer.
has_listed_versions = addon.has_listed_versions(
include_deleted=include_deleted_when_checking_versions)
if not (has_listed_versions or
owner_or_unlisted_reviewer(request, addon)):
include_deleted=include_deleted_when_checking_versions
)
if not (has_listed_versions or owner_or_unlisted_reviewer(request, addon)):
raise http.Http404
return f(request, addon, *args, **kw)
return wrapper

10
src/olympia/addons/forms.py
Просмотреть файл

@ -40,9 +40,13 @@ class AdminBaseFileFormSet(BaseModelFormSet):
for form in self.initial_forms:
if 'status' in form.changed_data:
admin_log.info(
'Addon "%s" file (ID:%d) status changed to: %s' % (
self.instance.slug, form.instance.id,
form.cleaned_data['status']))
'Addon "%s" file (ID:%d) status changed to: %s'
% (
self.instance.slug,
form.instance.id,
form.cleaned_data['status'],
)
)
return objs
def save_new_objects(self, commit=True):

374
src/olympia/addons/indexers.py
Просмотреть файл

@ -18,6 +18,7 @@ log = olympia.core.logger.getLogger('z.es')
class AddonIndexer(BaseSearchIndexer):
"""Fields we don't need to expose in the results, only used for filtering
or sorting."""
hidden_fields = (
'*.raw',
'boost',
@ -52,16 +53,20 @@ class AddonIndexer(BaseSearchIndexer):
# for instance.
'tokenizer': 'standard',
'filter': [
'standard', 'custom_word_delimiter', 'lowercase',
'stop', 'custom_dictionary_decompounder', 'unique',
]
'standard',
'custom_word_delimiter',
'lowercase',
'stop',
'custom_dictionary_decompounder',
'unique',
],
},
'trigram': {
# Analyzer that splits the text into trigrams.
'tokenizer': 'ngram_tokenizer',
'filter': [
'lowercase',
]
],
},
},
'tokenizer': {
@ -69,7 +74,7 @@ class AddonIndexer(BaseSearchIndexer):
'type': 'ngram',
'min_gram': 3,
'max_gram': 3,
'token_chars': ['letter', 'digit']
'token_chars': ['letter', 'digit'],
}
},
'normalizer': {
@ -89,7 +94,7 @@ class AddonIndexer(BaseSearchIndexer):
# and Foo Bar. (preserve_original: True makes us index both
# the original and the split version.)
'type': 'word_delimiter',
'preserve_original': True
'preserve_original': True,
},
'custom_dictionary_decompounder': {
# This filter is also useful for add-on names that have
@ -100,32 +105,110 @@ class AddonIndexer(BaseSearchIndexer):
# helping users looking for 'tab password' find that addon.
'type': 'dictionary_decompounder',
'word_list': [
'all', 'auto', 'ball', 'bar', 'block', 'blog',
'bookmark', 'browser', 'bug', 'button', 'cat', 'chat',
'click', 'clip', 'close', 'color', 'context', 'cookie',
'cool', 'css', 'delete', 'dictionary', 'down',
'download', 'easy', 'edit', 'fill', 'fire', 'firefox',
'fix', 'flag', 'flash', 'fly', 'forecast', 'fox',
'foxy', 'google', 'grab', 'grease', 'html', 'http',
'image', 'input', 'inspect', 'inspector', 'iris', 'js',
'key', 'keys', 'lang', 'link', 'mail', 'manager',
'map', 'mega', 'menu', 'menus', 'monkey', 'name',
'net', 'new', 'open', 'password', 'persona', 'privacy',
'query', 'screen', 'scroll', 'search', 'secure',
'select', 'smart', 'spring', 'status', 'style',
'super', 'sync', 'tab', 'text', 'think', 'this',
'time', 'title', 'translate', 'tree', 'undo', 'upload',
'url', 'user', 'video', 'window', 'with', 'word',
'all',
'auto',
'ball',
'bar',
'block',
'blog',
'bookmark',
'browser',
'bug',
'button',
'cat',
'chat',
'click',
'clip',
'close',
'color',
'context',
'cookie',
'cool',
'css',
'delete',
'dictionary',
'down',
'download',
'easy',
'edit',
'fill',
'fire',
'firefox',
'fix',
'flag',
'flash',
'fly',
'forecast',
'fox',
'foxy',
'google',
'grab',
'grease',
'html',
'http',
'image',
'input',
'inspect',
'inspector',
'iris',
'js',
'key',
'keys',
'lang',
'link',
'mail',
'manager',
'map',
'mega',
'menu',
'menus',
'monkey',
'name',
'net',
'new',
'open',
'password',
'persona',
'privacy',
'query',
'screen',
'scroll',
'search',
'secure',
'select',
'smart',
'spring',
'status',
'style',
'super',
'sync',
'tab',
'text',
'think',
'this',
'time',
'title',
'translate',
'tree',
'undo',
'upload',
'url',
'user',
'video',
'window',
'with',
'word',
'zilla',
]
],
},
}
},
}
}
@classmethod
def get_model(cls):
from olympia.addons.models import Addon
return Addon
@classmethod
@ -147,8 +230,9 @@ class AddonIndexer(BaseSearchIndexer):
version_mapping = {
'type': 'object',
'properties': {
'compatible_apps': {'properties': {app.id: appver_mapping
for app in amo.APP_USAGE}},
'compatible_apps': {
'properties': {app.id: appver_mapping for app in amo.APP_USAGE}
},
# Keep '<version>.id' indexed to be able to run exists queries
# on it.
'id': {'type': 'long'},
@ -159,23 +243,17 @@ class AddonIndexer(BaseSearchIndexer):
'id': {'type': 'long', 'index': False},
'created': {'type': 'date', 'index': False},
'hash': {'type': 'keyword', 'index': False},
'filename': {
'type': 'keyword', 'index': False},
'filename': {'type': 'keyword', 'index': False},
'is_webextension': {'type': 'boolean'},
'is_mozilla_signed_extension': {'type': 'boolean'},
'is_restart_required': {
'type': 'boolean', 'index': False},
'platform': {
'type': 'byte', 'index': False},
'is_restart_required': {'type': 'boolean', 'index': False},
'platform': {'type': 'byte', 'index': False},
'size': {'type': 'long', 'index': False},
'strict_compatibility': {
'type': 'boolean', 'index': False},
'strict_compatibility': {'type': 'boolean', 'index': False},
'status': {'type': 'byte'},
'permissions': {
'type': 'keyword', 'index': False},
'optional_permissions': {
'type': 'keyword', 'index': False},
}
'permissions': {'type': 'keyword', 'index': False},
'optional_permissions': {'type': 'keyword', 'index': False},
},
},
'license': {
'type': 'object',
@ -183,19 +261,17 @@ class AddonIndexer(BaseSearchIndexer):
'id': {'type': 'long', 'index': False},
'builtin': {'type': 'boolean', 'index': False},
'name_translations': cls.get_translations_definition(),
'url': {'type': 'text', 'index': False}
'url': {'type': 'text', 'index': False},
},
},
'release_notes_translations':
cls.get_translations_definition(),
'release_notes_translations': cls.get_translations_definition(),
'version': {'type': 'keyword', 'index': False},
}
},
}
mapping = {
doc_name: {
'properties': {
'id': {'type': 'long'},
'app': {'type': 'byte'},
'average_daily_users': {'type': 'long'},
'bayesian_rating': {'type': 'double'},
@ -248,22 +324,20 @@ class AddonIndexer(BaseSearchIndexer):
'trigrams': {
'type': 'text',
'analyzer': 'trigram',
}
}
},
},
},
'platforms': {'type': 'byte'},
'previews': {
'type': 'object',
'properties': {
'id': {'type': 'long', 'index': False},
'caption_translations':
cls.get_translations_definition(),
'caption_translations': cls.get_translations_definition(),
'modified': {'type': 'date', 'index': False},
'sizes': {
'type': 'object',
'properties': {
'thumbnail': {'type': 'short',
'index': False},
'thumbnail': {'type': 'short', 'index': False},
'image': {'type': 'short', 'index': False},
},
},
@ -274,14 +348,14 @@ class AddonIndexer(BaseSearchIndexer):
'properties': {
'group_id': {'type': 'byte'},
'approved_for_apps': {'type': 'byte'},
}
},
},
'ratings': {
'type': 'object',
'properties': {
'count': {'type': 'short', 'index': False},
'average': {'type': 'float', 'index': False}
}
'average': {'type': 'float', 'index': False},
},
},
'slug': {'type': 'keyword'},
'requires_payment': {'type': 'boolean', 'index': False},
@ -297,16 +371,23 @@ class AddonIndexer(BaseSearchIndexer):
# Add fields that we expect to return all translations without being
# analyzed/indexed.
cls.attach_translation_mappings(
mapping, ('description', 'developer_comments', 'homepage', 'name',
'summary', 'support_email', 'support_url'))
mapping,
(
'description',
'developer_comments',
'homepage',
'name',
'summary',
'support_email',
'support_url',
),
)
# Add language-specific analyzers for localized fields that are
# analyzed/indexed.
cls.attach_language_specific_analyzers(
mapping, ('description', 'summary'))
cls.attach_language_specific_analyzers(mapping, ('description', 'summary'))
cls.attach_language_specific_analyzers_with_raw_variant(
mapping, ('name',))
cls.attach_language_specific_analyzers_with_raw_variant(mapping, ('name',))
return mapping
@ -314,33 +395,43 @@ class AddonIndexer(BaseSearchIndexer):
def extract_version(cls, obj, version_obj):
from olympia.versions.models import License, Version
data = {
'id': version_obj.pk,
'compatible_apps': cls.extract_compatibility_info(
obj, version_obj),
'files': [{
'id': file_.id,
'created': file_.created,
'filename': file_.filename,
'hash': file_.hash,
'is_webextension': file_.is_webextension,
'is_mozilla_signed_extension': (
file_.is_mozilla_signed_extension),
'is_restart_required': file_.is_restart_required,
'platform': file_.platform,
'size': file_.size,
'status': file_.status,
'strict_compatibility': file_.strict_compatibility,
'permissions': file_.permissions,
'optional_permissions': file_.optional_permissions,
} for file_ in version_obj.all_files],
'reviewed': version_obj.reviewed,
'version': version_obj.version,
} if version_obj else None
data = (
{
'id': version_obj.pk,
'compatible_apps': cls.extract_compatibility_info(obj, version_obj),
'files': [
{
'id': file_.id,
'created': file_.created,
'filename': file_.filename,
'hash': file_.hash,
'is_webextension': file_.is_webextension,
'is_mozilla_signed_extension': (
file_.is_mozilla_signed_extension
),
'is_restart_required': file_.is_restart_required,
'platform': file_.platform,
'size': file_.size,
'status': file_.status,
'strict_compatibility': file_.strict_compatibility,
'permissions': file_.permissions,
'optional_permissions': file_.optional_permissions,
}
for file_ in version_obj.all_files
],
'reviewed': version_obj.reviewed,
'version': version_obj.version,
}
if version_obj
else None
)
if data and version_obj:
attach_trans_dict(Version, [version_obj])
data.update(cls.extract_field_api_translations(
version_obj, 'release_notes', db_field='release_notes_id'))
data.update(
cls.extract_field_api_translations(
version_obj, 'release_notes', db_field='release_notes_id'
)
)
if version_obj.license:
data['license'] = {
'id': version_obj.license.id,
@ -348,8 +439,9 @@ class AddonIndexer(BaseSearchIndexer):
'url': version_obj.license.url,
}
attach_trans_dict(License, [version_obj.license])
data['license'].update(cls.extract_field_api_translations(
version_obj.license, 'name'))
data['license'].update(
cls.extract_field_api_translations(version_obj.license, 'name')
)
return data
@classmethod
@ -361,8 +453,7 @@ class AddonIndexer(BaseSearchIndexer):
if appver:
min_, max_ = appver.min.version_int, appver.max.version_int
min_human, max_human = appver.min.version, appver.max.version
if not version_obj.files.filter(
strict_compatibility=True).exists():
if not version_obj.files.filter(strict_compatibility=True).exists():
# The files attached to this version are not using strict
# compatibility, so the max version essentially needs to be
# ignored - let's fake a super high one. We leave max_human
@ -373,12 +464,16 @@ class AddonIndexer(BaseSearchIndexer):
# want to reindex every time a new version of the app is
# released, so we directly index a super high version as the
# max.
min_human, max_human = amo.D2C_MIN_VERSIONS.get(
app.id, '1.0'), amo.FAKE_MAX_VERSION,
min_human, max_human = (
amo.D2C_MIN_VERSIONS.get(app.id, '1.0'),
amo.FAKE_MAX_VERSION,
)
min_, max_ = version_int(min_human), version_int(max_human)
compatible_apps[app.id] = {
'min': min_, 'min_human': min_human,
'max': max_, 'max_human': max_human,
'min': min_,
'min_human': min_human,
'max': max_,
'max_human': max_human,
}
return compatible_apps
@ -387,19 +482,32 @@ class AddonIndexer(BaseSearchIndexer):
"""Extract indexable attributes from an add-on."""
from olympia.addons.models import Preview
attrs = ('id', 'average_daily_users', 'bayesian_rating',
'contributions', 'created',
'default_locale', 'guid', 'hotness', 'icon_hash', 'icon_type',
'is_disabled', 'is_experimental',
'last_updated',
'modified', 'requires_payment', 'slug',
'status', 'type', 'weekly_downloads')
attrs = (
'id',
'average_daily_users',
'bayesian_rating',
'contributions',
'created',
'default_locale',
'guid',
'hotness',
'icon_hash',
'icon_type',
'is_disabled',
'is_experimental',
'last_updated',
'modified',
'requires_payment',
'slug',
'status',
'type',
'weekly_downloads',
)
data = {attr: getattr(obj, attr) for attr in attrs}
data['colors'] = None
if obj.current_version:
data['platforms'] = [p.id for p in
obj.current_version.supported_platforms]
data['platforms'] = [p.id for p in obj.current_version.supported_platforms]
# Extract dominant colors from static themes.
if obj.type == amo.ADDON_STATICTHEME:
@ -409,19 +517,25 @@ class AddonIndexer(BaseSearchIndexer):
data['app'] = [app.id for app in obj.compatible_apps.keys()]
# Boost by the number of users on a logarithmic scale.
data['boost'] = float(data['average_daily_users'] ** .2)
data['boost'] = float(data['average_daily_users'] ** 0.2)
# Quadruple the boost if the add-on is public.
if (obj.status == amo.STATUS_APPROVED and not obj.is_experimental and
'boost' in data):
if (
obj.status == amo.STATUS_APPROVED
and not obj.is_experimental
and 'boost' in data
):
data['boost'] = float(max(data['boost'], 1) * 4)
# We can use all_categories because the indexing code goes through the
# transformer that sets it.
data['category'] = [cat.id for cat in obj.all_categories]
data['current_version'] = cls.extract_version(
obj, obj.current_version)
data['current_version'] = cls.extract_version(obj, obj.current_version)
data['listed_authors'] = [
{'name': a.name, 'id': a.id, 'username': a.username,
'is_public': a.is_public}
{
'name': a.name,
'id': a.id,
'username': a.username,
'is_public': a.is_public,
}
for a in obj.listed_authors
]
@ -429,18 +543,25 @@ class AddonIndexer(BaseSearchIndexer):
data['has_privacy_policy'] = bool(obj.privacy_policy)
data['is_recommended'] = bool(
obj.promoted and obj.promoted.group == RECOMMENDED)
obj.promoted and obj.promoted.group == RECOMMENDED
)
data['previews'] = [{'id': preview.id, 'modified': preview.modified,
'sizes': preview.sizes}
for preview in obj.current_previews]
data['previews'] = [
{'id': preview.id, 'modified': preview.modified, 'sizes': preview.sizes}
for preview in obj.current_previews
]
data['promoted'] = {
'group_id': obj.promoted.group_id,
# store the app approvals because .approved_applications needs it.
'approved_for_apps': [
app.id for app in obj.promoted.approved_applications],
} if obj.promoted else None
data['promoted'] = (
{
'group_id': obj.promoted.group_id,
# store the app approvals because .approved_applications needs it.
'approved_for_apps': [
app.id for app in obj.promoted.approved_applications
],
}
if obj.promoted
else None
)
data['ratings'] = {
'average': obj.average_rating,
@ -455,14 +576,14 @@ class AddonIndexer(BaseSearchIndexer):
# First, deal with the 3 fields that need everything:
for field in ('description', 'name', 'summary'):
data.update(cls.extract_field_api_translations(obj, field))
data.update(cls.extract_field_search_translation(
obj, field, obj.default_locale))
data.update(
cls.extract_field_search_translation(obj, field, obj.default_locale)
)
data.update(cls.extract_field_analyzed_translations(obj, field))
# Then add fields that only need to be returned to the API without
# contributing to search relevancy.
for field in ('developer_comments', 'homepage', 'support_email',
'support_url'):
for field in ('developer_comments', 'homepage', 'support_email', 'support_url'):
data.update(cls.extract_field_api_translations(obj, field))
if obj.type != amo.ADDON_STATICTHEME:
# Also do that for preview captions, which are set on each preview
@ -470,7 +591,8 @@ class AddonIndexer(BaseSearchIndexer):
attach_trans_dict(Preview, obj.current_previews)
for i, preview in enumerate(obj.current_previews):
data['previews'][i].update(
cls.extract_field_api_translations(preview, 'caption'))
cls.extract_field_api_translations(preview, 'caption')
)
return data
@ -505,8 +627,6 @@ class AddonIndexer(BaseSearchIndexer):
"""
from olympia.addons.tasks import index_addons
ids = cls.get_model().unfiltered.values_list(
'id', flat=True).order_by('id')
ids = cls.get_model().unfiltered.values_list('id', flat=True).order_by('id')
chunk_size = 150
return create_chunked_tasks_signatures(
index_addons, list(ids), chunk_size)
return create_chunked_tasks_signatures(index_addons, list(ids), chunk_size)

31
src/olympia/addons/management/commands/fix_langpacks_with_max_version_star.py
Просмотреть файл

@ -13,7 +13,8 @@ class Command(BaseCommand):
def find_affected_langpacks(self):
qs = Version.unfiltered.filter(
addon__type=amo.ADDON_LPAPP, apps__max__version='*').distinct()
addon__type=amo.ADDON_LPAPP, apps__max__version='*'
).distinct()
return qs
def fix_max_appversion_for_version(self, version):
@ -22,32 +23,42 @@ class Command(BaseCommand):
log.info(
'Version %s for addon %s min version is not compatible '
'with %s, skipping this version for that app.',
version, version.addon, app.pretty)
version,
version.addon,
app.pretty,
)
continue
if version.compatible_apps[app].max.version != '*':
log.info(
'Version %s for addon %s max version is not "*" for %s '
'app, skipping this version for that app.',
version, version.addon, app.pretty)
version,
version.addon,
app.pretty,
)
continue
min_appversion_str = version.compatible_apps[app].min.version
max_appversion_str = '%d.*' % version_dict(
min_appversion_str)['major']
max_appversion_str = '%d.*' % version_dict(min_appversion_str)['major']
log.warning(
'Version %s for addon %s min version is %s for %s app, '
'max will be changed to %s instead of *',
version, version.addon, min_appversion_str, app.pretty,
max_appversion_str)
version,
version.addon,
min_appversion_str,
app.pretty,
max_appversion_str,
)
max_appversion = AppVersion.objects.get(
application=app.id, version=max_appversion_str)
application=app.id, version=max_appversion_str
)
version.compatible_apps[app].max = max_appversion
version.compatible_apps[app].save()
def handle(self, *args, **options):
versions = self.find_affected_langpacks()
log.info(
'Found %d langpack versions with an incorrect max version',
versions.count())
'Found %d langpack versions with an incorrect max version', versions.count()
)
for version in versions:
log.info('Fixing version %s for addon %s', version, version.addon)
self.fix_max_appversion_for_version(version)

128
src/olympia/addons/management/commands/process_addons.py
Просмотреть файл

@ -31,96 +31,108 @@ def get_recalc_needed_filters():
summary_modified = F('_current_version__autoapprovalsummary__modified')
# We don't take deleted reports into account
valid_abuse_report_states = (
AbuseReport.STATES.UNTRIAGED, AbuseReport.STATES.VALID,
AbuseReport.STATES.SUSPICIOUS)
AbuseReport.STATES.UNTRIAGED,
AbuseReport.STATES.VALID,
AbuseReport.STATES.SUSPICIOUS,
)
return [
# Only recalculate add-ons that received recent abuse reports
# possibly through their authors.
Q(
abuse_reports__state__in=valid_abuse_report_states,
abuse_reports__created__gte=summary_modified
) |
Q(
abuse_reports__created__gte=summary_modified,
)
| Q(
authors__abuse_reports__state__in=valid_abuse_report_states,
authors__abuse_reports__created__gte=summary_modified
) |
authors__abuse_reports__created__gte=summary_modified,
)
# And check ratings that have a rating of 3 or less
Q(
| Q(
_current_version__ratings__deleted=False,
_current_version__ratings__created__gte=summary_modified,
_current_version__ratings__rating__lte=3)
_current_version__ratings__rating__lte=3,
)
]
tasks = {
'find_inconsistencies_between_es_and_db': {
'method': find_inconsistencies_between_es_and_db, 'qs': []},
'method': find_inconsistencies_between_es_and_db,
'qs': [],
},
'get_preview_sizes': {'method': get_preview_sizes, 'qs': []},
'recalculate_post_review_weight': {
'method': recalculate_post_review_weight,
'qs': [
Q(**{current_autoapprovalsummary + 'verdict': amo.AUTO_APPROVED}) &
~Q(**{current_autoapprovalsummary + 'confirmed': True})
]},
Q(**{current_autoapprovalsummary + 'verdict': amo.AUTO_APPROVED})
& ~Q(**{current_autoapprovalsummary + 'confirmed': True})
],
},
'constantly_recalculate_post_review_weight': {
# This re-calculates the whole post-review weight which can be costly
# so take that into account. We may want to optimize that later
# in case we notice things are slower than needed - cgrebs 20190730
'method': recalculate_post_review_weight,
'kwargs': {'only_current_version': True},
'qs': get_recalc_needed_filters()},
'qs': get_recalc_needed_filters(),
},
'resign_addons_for_cose': {
'method': sign_addons,
'qs': [
# Only resign public add-ons where the latest version has been
# created before the 5th of April
Q(status=amo.STATUS_APPROVED,
_current_version__created__lt=datetime(2019, 4, 5))
]
Q(
status=amo.STATUS_APPROVED,
_current_version__created__lt=datetime(2019, 4, 5),
)
],
},
'recreate_previews': {
'method': recreate_previews,
'qs': [
~Q(type=amo.ADDON_STATICTHEME)
]
'qs': [~Q(type=amo.ADDON_STATICTHEME)],
},
'recreate_theme_previews': {
'method': recreate_theme_previews,
'qs': [
Q(type=amo.ADDON_STATICTHEME, status__in=[
amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW])
Q(
type=amo.ADDON_STATICTHEME,
status__in=[amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW],
)
],
'kwargs': {'only_missing': False},
},
'create_missing_theme_previews': {
'method': recreate_theme_previews,
'qs': [
Q(type=amo.ADDON_STATICTHEME, status__in=[
amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW])
Q(
type=amo.ADDON_STATICTHEME,
status__in=[amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW],
)
],
'kwargs': {'only_missing': True},
},
'add_dynamic_theme_tag_for_theme_api': {
'method': add_dynamic_theme_tag,
'qs': [
Q(status=amo.STATUS_APPROVED,
_current_version__files__is_webextension=True)
]
Q(status=amo.STATUS_APPROVED, _current_version__files__is_webextension=True)
],
},
'extract_colors_from_static_themes': {
'method': extract_colors_from_static_themes,
'qs': [Q(type=amo.ADDON_STATICTHEME)]
'qs': [Q(type=amo.ADDON_STATICTHEME)],
},
'delete_obsolete_addons': {
'method': delete_addons,
'qs': [
Q(type__in=(_ADDON_THEME,
amo.ADDON_LPADDON,
amo.ADDON_PLUGIN,
_ADDON_PERSONA,
_ADDON_WEBAPP,
))
Q(
type__in=(
_ADDON_THEME,
amo.ADDON_LPADDON,
amo.ADDON_PLUGIN,
_ADDON_PERSONA,
_ADDON_WEBAPP,
)
)
],
'allowed_kwargs': ('with_deleted',),
},
@ -140,6 +152,7 @@ class Command(BaseCommand):
allowed_kwargs: any extra boolean kwargs that can be applied via
additional arguments. Make sure to add it to `add_arguments` too.
"""
def add_arguments(self, parser):
"""Handle command arguments."""
parser.add_argument(
@ -147,27 +160,31 @@ class Command(BaseCommand):
action='store',
dest='task',
type=str,
help='Run task on the addons.')
help='Run task on the addons.',
)
parser.add_argument(
'--with-deleted',
action='store_true',
dest='with_deleted',
help='Include deleted add-ons when determining which '
'add-ons to process.')
'add-ons to process.',
)
parser.add_argument(
'--ids',
action='store',
dest='ids',
help='Only apply task to specific addon ids (comma-separated).')
help='Only apply task to specific addon ids (comma-separated).',
)
parser.add_argument(
'--limit',
action='store',
dest='limit',
type=int,
help='Only apply task to the first X addon ids.')
help='Only apply task to the first X addon ids.',
)
parser.add_argument(
'--batch-size',
@ -175,7 +192,8 @@ class Command(BaseCommand):
dest='batch_size',
type=int,
default=100,
help='Split the add-ons into X size chunks. Default 100.')
help='Split the add-ons into X size chunks. Default 100.',
)
parser.add_argument(
'--channel',
@ -185,12 +203,12 @@ class Command(BaseCommand):
choices=('listed', 'unlisted'),
help=(
'Only select add-ons who have either listed or unlisted '
'versions. Add-ons that have both will be returned too.'))
'versions. Add-ons that have both will be returned too.'
),
)
def get_pks(self, manager, q_objects, distinct=False):
pks = (manager.filter(q_objects)
.values_list('pk', flat=True)
.order_by('id'))
pks = manager.filter(q_objects).values_list('pk', flat=True).order_by('id')
if distinct:
pks = pks.distinct()
return pks
@ -198,8 +216,9 @@ class Command(BaseCommand):
def handle(self, *args, **options):
task = tasks.get(options.get('task'))
if not task:
raise CommandError('Unknown task provided. Options are: %s'
% ', '.join(tasks.keys()))
raise CommandError(
'Unknown task provided. Options are: %s' % ', '.join(tasks.keys())
)
if options.get('with_deleted'):
addon_manager = Addon.unfiltered
else:
@ -211,30 +230,27 @@ class Command(BaseCommand):
ids_list = options.get('ids').split(',')
addon_manager = addon_manager.filter(id__in=ids_list)
pks = self.get_pks(
addon_manager, *task['qs'], distinct=task.get('distinct'))
pks = self.get_pks(addon_manager, *task['qs'], distinct=task.get('distinct'))
if options.get('limit'):
pks = pks[:options.get('limit')]
pks = pks[: options.get('limit')]
if 'pre' in task:
# This is run in process to ensure its run before the tasks.
pks = task['pre'](pks)
if pks:
kwargs = task.get('kwargs', {})
if task.get('allowed_kwargs'):
kwargs.update({
arg: options.get(arg, None)
for arg in task['allowed_kwargs']})
kwargs.update(
{arg: options.get(arg, None) for arg in task['allowed_kwargs']}
)
# All the remaining tasks go in one group.
grouping = []
for chunk in chunked(pks, options.get('batch_size')):
grouping.append(
task['method'].subtask(args=[chunk], kwargs=kwargs))
grouping.append(task['method'].subtask(args=[chunk], kwargs=kwargs))
# Add the post task on to the end.
post = None
if 'post' in task:
post = task['post'].subtask(
args=[], kwargs=kwargs, immutable=True)
post = task['post'].subtask(args=[], kwargs=kwargs, immutable=True)
ts = chord(grouping, post)
else:
ts = group(grouping)

892
src/olympia/addons/models.py
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

300
src/olympia/addons/serializers.py
Просмотреть файл

@ -8,12 +8,16 @@ from rest_framework import exceptions, serializers
from olympia import amo
from olympia.accounts.serializers import (
BaseUserSerializer, UserProfileBasketSyncSerializer)
BaseUserSerializer,
UserProfileBasketSyncSerializer,
)
from olympia.amo.templatetags.jinja_helpers import absolutify
from olympia.amo.urlresolvers import get_outgoing_url, reverse
from olympia.api.fields import (
ESTranslationSerializerField, ReverseChoiceField,
TranslationSerializerField)
ESTranslationSerializerField,
ReverseChoiceField,
TranslationSerializerField,
)
from olympia.api.serializers import BaseESSerializer
from olympia.api.utils import is_gate_active
from olympia.applications.models import AppVersion
@ -27,28 +31,39 @@ from olympia.promoted.models import PromotedAddon
from olympia.search.filters import AddonAppVersionQueryParam
from olympia.users.models import UserProfile
from olympia.versions.models import (
ApplicationsVersions, License, Version, VersionPreview)
ApplicationsVersions,
License,
Version,
VersionPreview,
)
from .models import Addon, Preview, ReplacementAddon, attach_tags
class FileSerializer(serializers.ModelSerializer):
url = serializers.SerializerMethodField()
platform = ReverseChoiceField(
choices=list(amo.PLATFORM_CHOICES_API.items()))
platform = ReverseChoiceField(choices=list(amo.PLATFORM_CHOICES_API.items()))
status = ReverseChoiceField(choices=list(amo.STATUS_CHOICES_API.items()))
permissions = serializers.ListField(
child=serializers.CharField())
optional_permissions = serializers.ListField(
child=serializers.CharField())
permissions = serializers.ListField(child=serializers.CharField())
optional_permissions = serializers.ListField(child=serializers.CharField())
is_restart_required = serializers.BooleanField()
class Meta:
model = File
fields = ('id', 'created', 'hash', 'is_restart_required',
'is_webextension', 'is_mozilla_signed_extension',
'platform', 'size', 'status', 'url', 'permissions',
'optional_permissions')
fields = (
'id',
'created',
'hash',
'is_restart_required',
'is_webextension',
'is_mozilla_signed_extension',
'platform',
'size',
'status',
'url',
'permissions',
'optional_permissions',
)
def get_url(self, obj):
return obj.get_absolute_url()
@ -62,8 +77,14 @@ class PreviewSerializer(serializers.ModelSerializer):
class Meta:
# Note: this serializer can also be used for VersionPreview.
model = Preview
fields = ('id', 'caption', 'image_size', 'image_url', 'thumbnail_size',
'thumbnail_url')
fields = (
'id',
'caption',
'image_size',
'image_url',
'thumbnail_size',
'thumbnail_url',
)
def get_image_url(self, obj):
return absolutify(obj.image_url)
@ -138,8 +159,7 @@ class LicenseSerializer(serializers.ModelSerializer):
def to_representation(self, instance):
data = super(LicenseSerializer, self).to_representation(instance)
request = self.context.get('request', None)
if request and is_gate_active(
request, 'del-version-license-is-custom'):
if request and is_gate_active(request, 'del-version-license-is-custom'):
data.pop('is_custom', None)
return data
@ -167,9 +187,17 @@ class SimpleVersionSerializer(MinimalVersionSerializer):
class Meta:
model = Version
fields = ('id', 'compatibility', 'edit_url', 'files',
'is_strict_compatibility_enabled', 'license',
'release_notes', 'reviewed', 'version')
fields = (
'id',
'compatibility',
'edit_url',
'files',
'is_strict_compatibility_enabled',
'license',
'release_notes',
'reviewed',
'version',
)
def to_representation(self, instance):
# Help the LicenseSerializer find the version we're currently
@ -181,30 +209,41 @@ class SimpleVersionSerializer(MinimalVersionSerializer):
def get_compatibility(self, obj):
return {
app.short: {
'min': compat.min.version if compat else (
amo.D2C_MIN_VERSIONS.get(app.id, '1.0')),
'max': compat.max.version if compat else amo.FAKE_MAX_VERSION
} for app, compat in obj.compatible_apps.items()
'min': compat.min.version
if compat
else (amo.D2C_MIN_VERSIONS.get(app.id, '1.0')),
'max': compat.max.version if compat else amo.FAKE_MAX_VERSION,
}
for app, compat in obj.compatible_apps.items()
}
def get_edit_url(self, obj):
return absolutify(obj.addon.get_dev_url(
'versions.edit', args=[obj.pk], prefix_only=True))
return absolutify(
obj.addon.get_dev_url('versions.edit', args=[obj.pk], prefix_only=True)
)
def get_is_strict_compatibility_enabled(self, obj):
return any(file_.strict_compatibility for file_ in obj.all_files)
class VersionSerializer(SimpleVersionSerializer):
channel = ReverseChoiceField(
choices=list(amo.CHANNEL_CHOICES_API.items()))
channel = ReverseChoiceField(choices=list(amo.CHANNEL_CHOICES_API.items()))
license = LicenseSerializer()
class Meta:
model = Version
fields = ('id', 'channel', 'compatibility', 'edit_url', 'files',
'is_strict_compatibility_enabled', 'license',
'release_notes', 'reviewed', 'version')
fields = (
'id',
'channel',
'compatibility',
'edit_url',
'files',
'is_strict_compatibility_enabled',
'license',
'release_notes',
'reviewed',
'version',
)
class CurrentVersionSerializer(SimpleVersionSerializer):
@ -216,9 +255,12 @@ class CurrentVersionSerializer(SimpleVersionSerializer):
request = self.context.get('request')
view = self.context.get('view')
addon = obj.addon
if (request and request.GET.get('appversion') and
getattr(view, 'action', None) == 'retrieve' and
addon.type == amo.ADDON_LPAPP):
if (
request
and request.GET.get('appversion')
and getattr(view, 'action', None) == 'retrieve'
and addon.type == amo.ADDON_LPAPP
):
obj = self.get_current_compatible_version(addon)
return super(CurrentVersionSerializer, self).to_representation(obj)
@ -241,12 +283,13 @@ class CurrentVersionSerializer(SimpleVersionSerializer):
raise exceptions.ParseError(str(exc))
version_qs = Version.objects.latest_public_compatible_with(
application, appversions).filter(addon=addon)
application, appversions
).filter(addon=addon)
return version_qs.first() or addon.current_version
class ESCompactLicenseSerializer(BaseESSerializer, CompactLicenseSerializer):
translated_fields = ('name', )
translated_fields = ('name',)
def __init__(self, *args, **kwargs):
super(ESCompactLicenseSerializer, self).__init__(*args, **kwargs)
@ -287,16 +330,14 @@ class AddonDeveloperSerializer(BaseUserSerializer):
picture_url = serializers.SerializerMethodField()
class Meta(BaseUserSerializer.Meta):
fields = BaseUserSerializer.Meta.fields + (
'picture_url',)
fields = BaseUserSerializer.Meta.fields + ('picture_url',)
read_only_fields = fields
class PromotedAddonSerializer(serializers.ModelSerializer):
GROUP_CHOICES = [(group.id, group.api_name) for group in PROMOTED_GROUPS]
apps = serializers.SerializerMethodField()
category = ReverseChoiceField(
choices=GROUP_CHOICES, source='group_id')
category = ReverseChoiceField(choices=GROUP_CHOICES, source='group_id')
class Meta:
model = PromotedAddon
@ -378,15 +419,17 @@ class AddonSerializer(serializers.ModelSerializer):
'tags',
'type',
'url',
'weekly_downloads'
'weekly_downloads',
)
def to_representation(self, obj):
data = super(AddonSerializer, self).to_representation(obj)
request = self.context.get('request', None)
if ('request' in self.context and
'wrap_outgoing_links' in self.context['request'].GET):
if (
'request' in self.context
and 'wrap_outgoing_links' in self.context['request'].GET
):
for key in ('homepage', 'support_url', 'contributions_url'):
if key in data:
data[key] = self.outgoingify(data[key])
@ -403,8 +446,10 @@ class AddonSerializer(serializers.ModelSerializer):
if isinstance(data, str):
return get_outgoing_url(data)
elif isinstance(data, dict):
return {key: get_outgoing_url(value) if value else None
for key, value in data.items()}
return {
key: get_outgoing_url(value) if value else None
for key, value in data.items()
}
# None or empty string... don't bother.
return data
@ -446,8 +491,8 @@ class AddonSerializer(serializers.ModelSerializer):
query = QueryDict(parts.query, mutable=True)
query.update(amo.CONTRIBUTE_UTM_PARAMS)
return urlunsplit(
(parts.scheme, parts.netloc, parts.path, query.urlencode(),
parts.fragment))
(parts.scheme, parts.netloc, parts.path, query.urlencode(), parts.fragment)
)
def get_edit_url(self, obj):
return absolutify(obj.get_dev_url())
@ -464,8 +509,7 @@ class AddonSerializer(serializers.ModelSerializer):
def get_icons(self, obj):
get_icon = obj.get_icon_url
return {str(size): absolutify(get_icon(size))
for size in amo.ADDON_ICON_SIZES}
return {str(size): absolutify(get_icon(size)) for size in amo.ADDON_ICON_SIZES}
def get_ratings(self, obj):
return {
@ -503,12 +547,19 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
_score = serializers.SerializerMethodField()
datetime_fields = ('created', 'last_updated', 'modified')
translated_fields = ('name', 'description', 'developer_comments',
'homepage', 'summary', 'support_email', 'support_url')
translated_fields = (
'name',
'description',
'developer_comments',
'homepage',
'summary',
'support_email',
'support_url',
)
class Meta:
model = Addon
fields = AddonSerializer.Meta.fields + ('_score', )
fields = AddonSerializer.Meta.fields + ('_score',)
def fake_preview_object(self, obj, data, model_class=Preview):
# This is what ESPreviewSerializer.fake_object() would do, but we do
@ -524,33 +575,40 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
preview_serializer._attach_fields(preview, data, ('modified',))
# Attach translations.
preview_serializer._attach_translations(
preview, data, preview_serializer.translated_fields)
preview, data, preview_serializer.translated_fields
)
return preview
def fake_file_object(self, obj, data):
file_ = File(
id=data['id'], created=self.handle_date(data['created']),
hash=data['hash'], filename=data['filename'],
id=data['id'],
created=self.handle_date(data['created']),
hash=data['hash'],
filename=data['filename'],
is_webextension=data.get('is_webextension'),
is_mozilla_signed_extension=data.get(
'is_mozilla_signed_extension'),
is_mozilla_signed_extension=data.get('is_mozilla_signed_extension'),
is_restart_required=data.get('is_restart_required', False),
platform=data['platform'], size=data['size'],
platform=data['platform'],
size=data['size'],
status=data['status'],
strict_compatibility=data.get('strict_compatibility', False),
version=obj)
version=obj,
)
file_.permissions = data.get(
'permissions', data.get('webext_permissions_list', []))
file_.optional_permissions = data.get(
'optional_permissions', [])
'permissions', data.get('webext_permissions_list', [])
)
file_.optional_permissions = data.get('optional_permissions', [])
return file_
def fake_version_object(self, obj, data, channel):
if data:
version = Version(
addon=obj, id=data['id'],
addon=obj,
id=data['id'],
reviewed=self.handle_date(data['reviewed']),
version=data['version'], channel=channel)
version=data['version'],
channel=channel,
)
version.all_files = [
self.fake_file_object(version, file_data)
for file_data in data.get('files', [])
@ -563,22 +621,26 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
app_name = APPS_ALL[int(app_id)]
compatible_apps[app_name] = ApplicationsVersions(
min=AppVersion(version=compat_dict.get('min_human', '')),
max=AppVersion(version=compat_dict.get('max_human', '')))
max=AppVersion(version=compat_dict.get('max_human', '')),
)
version._compatible_apps = compatible_apps
version_serializer = self.fields.get('current_version') or None
if version_serializer:
version_serializer._attach_translations(
version, data, version_serializer.translated_fields)
version, data, version_serializer.translated_fields
)
if 'license' in data and version_serializer:
license_serializer = version_serializer.fields['license']
version.license = License(id=data['license']['id'])
license_serializer._attach_fields(
version.license, data['license'], ('builtin', 'url'))
version.license, data['license'], ('builtin', 'url')
)
# Can't use license_serializer._attach_translations() directly
# because 'name' is a SerializerMethodField, not an
# ESTranslatedField.
license_serializer.db_name.attach_translations(
version.license, data['license'], 'name')
version.license, data['license'], 'name'
)
else:
version.license = None
else:
@ -592,7 +654,9 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
# Attach base attributes that have the same name/format in ES and in
# the model.
self._attach_fields(
obj, data, (
obj,
data,
(
'average_daily_users',
'bayesian_rating',
'contributions',
@ -611,14 +675,15 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
'slug',
'status',
'type',
'weekly_downloads'
)
'weekly_downloads',
),
)
# Attach attributes that do not have the same name/format in ES.
obj.tag_list = data.get('tags', [])
obj.all_categories = [
CATEGORIES_BY_ID[cat_id] for cat_id in data.get('category', [])]
CATEGORIES_BY_ID[cat_id] for cat_id in data.get('category', [])
]
# Not entirely accurate, but enough in the context of the search API.
obj.disabled_by_user = data.get('is_disabled', False)
@ -631,23 +696,25 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
# begins with an underscore.
data_version = data.get('current_version') or {}
obj._current_version = self.fake_version_object(
obj, data_version, amo.RELEASE_CHANNEL_LISTED)
obj, data_version, amo.RELEASE_CHANNEL_LISTED
)
obj._current_version_id = data_version.get('id')
data_authors = data.get('listed_authors', [])
obj.listed_authors = [
UserProfile(
id=data_author['id'], display_name=data_author['name'],
id=data_author['id'],
display_name=data_author['name'],
username=data_author['username'],
is_public=data_author.get('is_public', False))
is_public=data_author.get('is_public', False),
)
for data_author in data_authors
]
is_static_theme = data.get('type') == amo.ADDON_STATICTHEME
preview_model_class = VersionPreview if is_static_theme else Preview
obj.current_previews = [
self.fake_preview_object(
obj, preview_data, model_class=preview_model_class)
self.fake_preview_object(obj, preview_data, model_class=preview_model_class)
for preview_data in data.get('previews', [])
]
@ -657,13 +724,16 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
# .approved_applications.
approved_for_apps = promoted.get('approved_for_apps')
obj.promoted = PromotedAddon(
addon=obj, approved_application_ids=approved_for_apps,
group_id=promoted['group_id'])
addon=obj,
approved_application_ids=approved_for_apps,
group_id=promoted['group_id'],
)
# we can safely regenerate these tuples because
# .appproved_applications only cares about the current group
obj._current_version.approved_for_groups = (
(obj.promoted.group, APP_IDS.get(app_id))
for app_id in approved_for_apps)
for app_id in approved_for_apps
)
else:
obj.promoted = None
@ -682,8 +752,11 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
def to_representation(self, obj):
data = super(ESAddonSerializer, self).to_representation(obj)
request = self.context.get('request')
if request and '_score' in data and not is_gate_active(
request, 'addons-search-_score-field'):
if (
request
and '_score' in data
and not is_gate_active(request, 'addons-search-_score-field')
):
data.pop('_score')
return data
@ -704,6 +777,7 @@ class ESAddonAutoCompleteSerializer(ESAddonSerializer):
class StaticCategorySerializer(serializers.Serializer):
"""Serializes a `StaticCategory` as found in constants.categories"""
id = serializers.IntegerField()
name = serializers.CharField()
slug = serializers.CharField()
@ -726,14 +800,22 @@ class LanguageToolsSerializer(AddonSerializer):
class Meta:
model = Addon
fields = ('id', 'current_compatible_version', 'default_locale', 'guid',
'name', 'slug', 'target_locale', 'type', 'url', )
fields = (
'id',
'current_compatible_version',
'default_locale',
'guid',
'name',
'slug',
'target_locale',
'type',
'url',
)
def get_current_compatible_version(self, obj):
compatible_versions = getattr(obj, 'compatible_versions', None)
if compatible_versions is not None:
data = MinimalVersionSerializer(
compatible_versions, many=True).data
data = MinimalVersionSerializer(compatible_versions, many=True).data
try:
# 99% of the cases there will only be one result, since most
# language packs are automatically uploaded for a given app
@ -750,11 +832,12 @@ class LanguageToolsSerializer(AddonSerializer):
def to_representation(self, obj):
data = super(LanguageToolsSerializer, self).to_representation(obj)
request = self.context['request']
if (AddonAppVersionQueryParam.query_param not in request.GET and
'current_compatible_version' in data):
if (
AddonAppVersionQueryParam.query_param not in request.GET
and 'current_compatible_version' in data
):
data.pop('current_compatible_version')
if request and is_gate_active(
request, 'addons-locale_disambiguation-shim'):
if request and is_gate_active(request, 'addons-locale_disambiguation-shim'):
data['locale_disambiguation'] = None
return data
@ -762,8 +845,7 @@ class LanguageToolsSerializer(AddonSerializer):
class VersionBasketSerializer(SimpleVersionSerializer):
class Meta:
model = Version
fields = ('id', 'compatibility', 'is_strict_compatibility_enabled',
'version')
fields = ('id', 'compatibility', 'is_strict_compatibility_enabled', 'version')
class AddonBasketSyncSerializer(AddonSerializerWithUnlistedData):
@ -777,11 +859,24 @@ class AddonBasketSyncSerializer(AddonSerializerWithUnlistedData):
class Meta:
model = Addon
fields = ('authors', 'average_daily_users', 'categories',
'current_version', 'default_locale', 'guid', 'id',
'is_disabled', 'is_recommended', 'last_updated',
'latest_unlisted_version', 'name', 'ratings', 'slug',
'status', 'type')
fields = (
'authors',
'average_daily_users',
'categories',
'current_version',
'default_locale',
'guid',
'id',
'is_disabled',
'is_recommended',
'last_updated',
'latest_unlisted_version',
'name',
'ratings',
'slug',
'status',
'type',
)
read_only_fields = fields
def get_name(self, obj):
@ -798,7 +893,8 @@ class ReplacementAddonSerializer(serializers.ModelSerializer):
replacement = serializers.SerializerMethodField()
ADDON_PATH_REGEX = r"""/addon/(?P<addon_id>[^/<>"']+)/$"""
COLLECTION_PATH_REGEX = (
r"""/collections/(?P<user_id>[^/<>"']+)/(?P<coll_slug>[^/]+)/$""")
r"""/collections/(?P<user_id>[^/<>"']+)/(?P<coll_slug>[^/]+)/$"""
)
class Meta:
model = ReplacementAddon
@ -822,8 +918,7 @@ class ReplacementAddonSerializer(serializers.ModelSerializer):
except Collection.DoesNotExist:
return []
valid_q = Addon.objects.get_queryset().valid_q([amo.STATUS_APPROVED])
return list(
collection.addons.filter(valid_q).values_list('guid', flat=True))
return list(collection.addons.filter(valid_q).values_list('guid', flat=True))
def get_replacement(self, obj):
if obj.has_external_url():
@ -836,5 +931,6 @@ class ReplacementAddonSerializer(serializers.ModelSerializer):
coll_match = re.search(self.COLLECTION_PATH_REGEX, obj.path)
if coll_match:
return self._get_collection_guids(
coll_match.group('user_id'), coll_match.group('coll_slug'))
coll_match.group('user_id'), coll_match.group('coll_slug')
)
return []

127
src/olympia/addons/tasks.py
Просмотреть файл

@ -9,16 +9,20 @@ import olympia.core
from olympia import amo
from olympia.addons.indexers import AddonIndexer
from olympia.addons.models import (
Addon, AppSupport, DeniedGuid, Preview, attach_tags,
attach_translations_dict)
Addon,
AppSupport,
DeniedGuid,
Preview,
attach_tags,
attach_translations_dict,
)
from olympia.amo.celery import task
from olympia.amo.decorators import use_primary_db
from olympia.amo.utils import LocalFileStorage, extract_colors_from_image
from olympia.files.utils import get_filepath, parse_addon
from olympia.lib.es.utils import index_objects
from olympia.tags.models import Tag
from olympia.versions.models import (
generate_static_theme_preview, VersionPreview)
from olympia.versions.models import generate_static_theme_preview, VersionPreview
log = olympia.core.logger.getLogger('z.task')
@ -36,8 +40,9 @@ def update_last_updated(addon_id):
try:
addon = Addon.objects.get(pk=addon_id)
except Addon.DoesNotExist:
log.info('[1@None] Updating last updated for %s failed, no addon found'
% addon_id)
log.info(
'[1@None] Updating last updated for %s failed, no addon found' % addon_id
)
return
log.info('[1@None] Updating last updated for %s.' % addon_id)
@ -68,8 +73,7 @@ def update_appsupport(ids, **kw):
else:
min_, max_ = appver.min.version_int, appver.max.version_int
support.append(AppSupport(addon=addon, app=app.id,
min=min_, max=max_))
support.append(AppSupport(addon=addon, app=app.id, min=min_, max=max_))
if not support:
return
@ -106,8 +110,14 @@ def delete_preview_files(id, **kw):
def index_addons(ids, **kw):
log.info('Indexing addons %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))
transforms = (attach_tags, attach_translations_dict)
index_objects(ids, Addon, AddonIndexer.extract_document,
kw.pop('index', None), transforms, Addon.unfiltered)
index_objects(
ids,
Addon,
AddonIndexer.extract_document,
kw.pop('index', None),
transforms,
Addon.unfiltered,
)
@task
@ -130,32 +140,46 @@ def find_inconsistencies_between_es_and_db(ids, **kw):
length = len(ids)
log.info(
'Searching for inconsistencies between db and es %d-%d [%d].',
ids[0], ids[-1], length)
ids[0],
ids[-1],
length,
)
db_addons = Addon.unfiltered.in_bulk(ids)
es_addons = Search(
doc_type=AddonIndexer.get_doctype_name(),
index=AddonIndexer.get_index_alias(),
using=amo.search.get_es()).filter('ids', values=ids)[:length].execute()
es_addons = (
Search(
doc_type=AddonIndexer.get_doctype_name(),
index=AddonIndexer.get_index_alias(),
using=amo.search.get_es(),
)
.filter('ids', values=ids)[:length]
.execute()
)
es_addons = es_addons
db_len = len(db_addons)
es_len = len(es_addons)
if db_len != es_len:
log.info('Inconsistency found: %d in db vs %d in es.',
db_len, es_len)
log.info('Inconsistency found: %d in db vs %d in es.', db_len, es_len)
for result in es_addons.hits.hits:
pk = result['_source']['id']
db_modified = db_addons[pk].modified.isoformat()
es_modified = result['_source']['modified']
if db_modified != es_modified:
log.info('Inconsistency found for addon %d: '
'modified is %s in db vs %s in es.',
pk, db_modified, es_modified)
log.info(
'Inconsistency found for addon %d: '
'modified is %s in db vs %s in es.',
pk,
db_modified,
es_modified,
)
db_status = db_addons[pk].status
es_status = result['_source']['status']
if db_status != es_status:
log.info('Inconsistency found for addon %d: '
'status is %s in db vs %s in es.',
pk, db_status, es_status)
log.info(
'Inconsistency found for addon %d: ' 'status is %s in db vs %s in es.',
pk,
db_status,
es_status,
)
@task
@ -163,8 +187,8 @@ def find_inconsistencies_between_es_and_db(ids, **kw):
def add_dynamic_theme_tag(ids, **kw):
"""Add dynamic theme tag to addons with the specified ids."""
log.info(
'Adding dynamic theme tag to addons %d-%d [%d].',
ids[0], ids[-1], len(ids))
'Adding dynamic theme tag to addons %d-%d [%d].', ids[0], ids[-1], len(ids)
)
addons = Addon.objects.filter(id__in=ids)
for addon in addons:
@ -178,8 +202,7 @@ def add_dynamic_theme_tag(ids, **kw):
@use_primary_db
def extract_colors_from_static_themes(ids, **kw):
"""Extract and store colors from existing static themes."""
log.info('Extracting static themes colors %d-%d [%d].', ids[0], ids[-1],
len(ids))
log.info('Extracting static themes colors %d-%d [%d].', ids[0], ids[-1], len(ids))
addons = Addon.objects.filter(id__in=ids)
extracted = []
for addon in addons:
@ -195,9 +218,10 @@ def extract_colors_from_static_themes(ids, **kw):
@task
@use_primary_db
def recreate_theme_previews(addon_ids, **kw):
log.info('[%s@%s] Recreating previews for themes starting at id: %s...'
% (len(addon_ids), recreate_theme_previews.rate_limit,
addon_ids[0]))
log.info(
'[%s@%s] Recreating previews for themes starting at id: %s...'
% (len(addon_ids), recreate_theme_previews.rate_limit, addon_ids[0])
)
addons = Addon.objects.filter(pk__in=addon_ids).no_transforms()
only_missing = kw.get('only_missing', False)
@ -207,8 +231,11 @@ def recreate_theme_previews(addon_ids, **kw):
continue
try:
if only_missing:
with_size = (VersionPreview.objects.filter(version=version)
.exclude(sizes={}).count())
with_size = (
VersionPreview.objects.filter(version=version)
.exclude(sizes={})
.count()
)
if with_size == len(amo.THEME_PREVIEW_SIZES):
continue
log.info('Recreating previews for theme: %s' % addon.id)
@ -235,19 +262,24 @@ def delete_addons(addon_ids, with_deleted=False, **kw):
STATUS_DELETED. *Addon.delete() only does a hard-delete where the Addon
has no versions or files - and has never had any versions or files.
"""
log.info('[%s@%s] %sDeleting addons starting at id: %s...'
% ('Hard ' if with_deleted else '', len(addon_ids),
delete_addons.rate_limit, addon_ids[0]))
log.info(
'[%s@%s] %sDeleting addons starting at id: %s...'
% (
'Hard ' if with_deleted else '',
len(addon_ids),
delete_addons.rate_limit,
addon_ids[0],
)
)
addons = Addon.unfiltered.filter(pk__in=addon_ids).no_transforms()
if with_deleted:
with transaction.atomic():
# Stop any of these guids from being reused
addon_guids = list(
addons.exclude(guid=None).values_list('guid', flat=True))
addon_guids = list(addons.exclude(guid=None).values_list('guid', flat=True))
denied = [
DeniedGuid(
guid=guid, comments='Hard deleted with delete_addons task')
for guid in addon_guids]
DeniedGuid(guid=guid, comments='Hard deleted with delete_addons task')
for guid in addon_guids
]
DeniedGuid.objects.bulk_create(denied, ignore_conflicts=True)
# Call QuerySet.delete rather than Addon.delete.
addons.delete()
@ -273,8 +305,11 @@ def update_addon_hotness(averages):
# See: https://github.com/mozilla/addons-server/issues/15525
if not average:
log.error('Averages not found for addon with id=%s and GUID=%s.',
addon.id, addon.guid)
log.error(
'Averages not found for addon with id=%s and GUID=%s.',
addon.id,
addon.guid,
)
continue
this = average['avg_this_week']
@ -302,8 +337,12 @@ def update_addon_weekly_downloads(data):
except Addon.DoesNotExist:
# The processing input comes from metrics which might be out of
# date in regards to currently existing add-ons.
log.info('Got a weekly_downloads update (%s) but the add-on '
'doesn\'t exist (hashed_guid=%s).', count, hashed_guid)
log.info(
'Got a weekly_downloads update (%s) but the add-on '
'doesn\'t exist (hashed_guid=%s).',
count,
hashed_guid,
)
continue
addon.update(weekly_downloads=int(float(count)))

11
src/olympia/addons/templatetags/jinja_helpers.py
Просмотреть файл

@ -6,9 +6,14 @@ from django_jinja import library
@library.global_function
@library.render_with('addons/impala/listing/sorter.html')
@jinja2.contextfunction
def impala_addon_listing_header(context, url_base, sort_opts=None,
selected=None, extra_sort_opts=None,
search_filter=None):
def impala_addon_listing_header(
context,
url_base,
sort_opts=None,
selected=None,
extra_sort_opts=None,
search_filter=None,
):
if sort_opts is None:
sort_opts = {}
if extra_sort_opts is None:

352
src/olympia/addons/tests/test_admin.py
Просмотреть файл

@ -4,17 +4,20 @@ from pyquery import PyQuery as pq
from django.conf import settings
from django.contrib import admin
from django.contrib.messages.storage import (
default_storage as default_messages_storage)
from django.contrib.messages.storage import default_storage as default_messages_storage
from django.core import mail
from olympia import amo, core
from olympia.activity.models import ActivityLog
from olympia.addons.admin import AddonAdmin, ReplacementAddonAdmin
from olympia.addons.models import (
Addon, AddonRegionalRestrictions, ReplacementAddon)
from olympia.addons.models import Addon, AddonRegionalRestrictions, ReplacementAddon
from olympia.amo.tests import (
TestCase, addon_factory, collection_factory, user_factory, version_factory)
TestCase,
addon_factory,
collection_factory,
user_factory,
version_factory,
)
from olympia.amo.urlresolvers import django_reverse, reverse
from olympia.blocklist.models import Block
@ -22,59 +25,69 @@ from olympia.blocklist.models import Block
class TestReplacementAddonForm(TestCase):
def test_valid_addon(self):
addon_factory(slug='bar')
form = ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': '/addon/bar/'})
form = ReplacementAddonAdmin(ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': '/addon/bar/'}
)
assert form.is_valid(), form.errors
assert form.cleaned_data['path'] == '/addon/bar/'
def test_invalid(self):
form = ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': '/invalid_url/'})
form = ReplacementAddonAdmin(ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': '/invalid_url/'}
)
assert not form.is_valid()
def test_valid_collection(self):
bagpuss = user_factory(username='bagpuss')
collection_factory(slug='stuff', author=bagpuss)
form = ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': '/collections/bagpuss/stuff/'})
form = ReplacementAddonAdmin(ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': '/collections/bagpuss/stuff/'}
)
assert form.is_valid(), form.errors
assert form.cleaned_data['path'] == '/collections/bagpuss/stuff/'
def test_url(self):
form = ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': 'https://google.com/'})
form = ReplacementAddonAdmin(ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': 'https://google.com/'}
)
assert form.is_valid()
assert form.cleaned_data['path'] == 'https://google.com/'
def test_invalid_urls(self):
assert not ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': 'ftp://google.com/'}).is_valid()
assert not ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': 'https://88999@~'}).is_valid()
assert not ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': 'https://www. rutrt/'}).is_valid()
assert (
not ReplacementAddonAdmin(ReplacementAddon, admin.site)
.get_form(None)({'guid': 'foo', 'path': 'ftp://google.com/'})
.is_valid()
)
assert (
not ReplacementAddonAdmin(ReplacementAddon, admin.site)
.get_form(None)({'guid': 'foo', 'path': 'https://88999@~'})
.is_valid()
)
assert (
not ReplacementAddonAdmin(ReplacementAddon, admin.site)
.get_form(None)({'guid': 'foo', 'path': 'https://www. rutrt/'})
.is_valid()
)
path = '/addon/bar/'
site = settings.SITE_URL
full_url = site + path
# path is okay
assert ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': path}).is_valid()
assert (
ReplacementAddonAdmin(ReplacementAddon, admin.site)
.get_form(None)({'guid': 'foo', 'path': path})
.is_valid()
)
# but we don't allow full urls for AMO paths
form = ReplacementAddonAdmin(
ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': full_url})
form = ReplacementAddonAdmin(ReplacementAddon, admin.site).get_form(None)(
{'guid': 'foo', 'path': full_url}
)
assert not form.is_valid()
assert ('Paths for [%s] should be relative, not full URLs including '
'the domain name' % site in form.errors['path'])
assert (
'Paths for [%s] should be relative, not full URLs including '
'the domain name' % site in form.errors['path']
)
class TestAddonAdmin(TestCase):
@ -143,9 +156,7 @@ class TestAddonAdmin(TestCase):
def test_can_edit_with_addons_edit_permission(self):
addon = addon_factory(guid='@foo')
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.client.login(email=user.email)
@ -206,13 +217,19 @@ class TestAddonAdmin(TestCase):
response = self.client.get(detail_url, follow=True)
content = response.content.decode('utf-8')
assert 'Reviewer Tools (listed)' in content
assert ('http://testserver{}'.format(
reverse('reviewers.review', args=('listed', addon.pk))
) in content)
assert (
'http://testserver{}'.format(
reverse('reviewers.review', args=('listed', addon.pk))
)
in content
)
assert 'Reviewer Tools (unlisted)' in content
assert ('http://testserver{}'.format(
reverse('reviewers.review', args=('unlisted', addon.pk))
) in content)
assert (
'http://testserver{}'.format(
reverse('reviewers.review', args=('unlisted', addon.pk))
)
in content
)
def test_can_not_list_without_addons_edit_permission(self):
addon = addon_factory()
@ -224,9 +241,7 @@ class TestAddonAdmin(TestCase):
def test_can_not_edit_without_addons_edit_permission(self):
addon = addon_factory(guid='@foo')
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.client.login(email=user.email)
response = self.client.get(self.detail_url, follow=True)
@ -254,12 +269,8 @@ class TestAddonAdmin(TestCase):
def test_access_using_slug(self):
addon = addon_factory(guid='@foo')
detail_url_by_slug = reverse(
'admin:addons_addon_change', args=(addon.slug,)
)
detail_url_final = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
detail_url_by_slug = reverse('admin:addons_addon_change', args=(addon.slug,))
detail_url_final = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.client.login(email=user.email)
@ -268,12 +279,8 @@ class TestAddonAdmin(TestCase):
def test_access_using_guid(self):
addon = addon_factory(guid='@foo')
detail_url_by_guid = reverse(
'admin:addons_addon_change', args=(addon.guid,)
)
detail_url_final = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
detail_url_by_guid = reverse('admin:addons_addon_change', args=(addon.guid,))
detail_url_final = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.client.login(email=user.email)
@ -283,9 +290,7 @@ class TestAddonAdmin(TestCase):
def test_can_edit_deleted_addon(self):
addon = addon_factory(guid='@foo')
addon.delete()
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.client.login(email=user.email)
@ -318,7 +323,6 @@ class TestAddonAdmin(TestCase):
'addonuser_set-0-role': amo.AUTHOR_ROLE_OWNER,
'addonuser_set-0-listed': 'on',
'addonuser_set-0-position': 0,
'files-TOTAL_FORMS': 1,
'files-INITIAL_FORMS': 1,
'files-MIN_NUM_FORMS': 0,
@ -331,9 +335,7 @@ class TestAddonAdmin(TestCase):
addon = addon_factory(guid='@foo', users=[user_factory()])
file = addon.current_version.all_files[0]
addonuser = addon.addonuser_set.get()
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.grant_permission(user, 'Admin:Advanced')
@ -342,11 +344,13 @@ class TestAddonAdmin(TestCase):
assert response.status_code == 200
assert addon.guid in response.content.decode('utf-8')
post_data = self._get_full_post_data(addon, addonuser)
post_data.update(**{
'type': amo.ADDON_STATICTHEME, # update it.
'addonuser_set-0-user': user.pk, # Different user than initial.
'files-0-status': amo.STATUS_AWAITING_REVIEW, # Different status.
})
post_data.update(
**{
'type': amo.ADDON_STATICTHEME, # update it.
'addonuser_set-0-user': user.pk, # Different user than initial.
'files-0-status': amo.STATUS_AWAITING_REVIEW, # Different status.
}
)
response = self.client.post(self.detail_url, post_data, follow=True)
assert response.status_code == 200
addon.reload()
@ -360,9 +364,7 @@ class TestAddonAdmin(TestCase):
addon = addon_factory(guid='@foo', users=[user_factory()])
file = addon.current_version.all_files[0]
addonuser = addon.addonuser_set.get()
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.client.login(email=user.email)
@ -371,11 +373,13 @@ class TestAddonAdmin(TestCase):
assert addon.guid in response.content.decode('utf-8')
post_data = self._get_full_post_data(addon, addonuser)
post_data.update(**{
'type': amo.ADDON_STATICTHEME, # update it.
'addonuser_set-0-user': user.pk, # Different user than initial.
'files-0-status': amo.STATUS_AWAITING_REVIEW, # Different status.
})
post_data.update(
**{
'type': amo.ADDON_STATICTHEME, # update it.
'addonuser_set-0-user': user.pk, # Different user than initial.
'files-0-status': amo.STATUS_AWAITING_REVIEW, # Different status.
}
)
response = self.client.post(self.detail_url, post_data, follow=True)
assert response.status_code == 200
addon.reload()
@ -388,12 +392,11 @@ class TestAddonAdmin(TestCase):
def test_can_manage_unlisted_versions_and_change_addon_status(self):
addon = addon_factory(guid='@foo', users=[user_factory()])
unlisted_version = version_factory(
addon=addon, channel=amo.RELEASE_CHANNEL_UNLISTED)
addon=addon, channel=amo.RELEASE_CHANNEL_UNLISTED
)
listed_version = addon.current_version
addonuser = addon.addonuser_set.get()
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.grant_permission(user, 'Admin:Advanced')
@ -403,24 +406,28 @@ class TestAddonAdmin(TestCase):
assert addon.guid in response.content.decode('utf-8')
doc = pq(response.content)
assert doc('#id_files-0-id').attr('value') == str(
unlisted_version.all_files[0].id)
unlisted_version.all_files[0].id
)
assert doc('#id_files-1-id').attr('value') == str(
addon.current_version.all_files[0].id)
addon.current_version.all_files[0].id
)
# pagination links aren't shown for less than page size (30) files.
next_url = self.detail_url + '?page=2'
assert next_url not in response.content.decode('utf-8')
post_data = self._get_full_post_data(addon, addonuser)
post_data.update(**{
'status': amo.STATUS_DISABLED,
'files-TOTAL_FORMS': 2,
'files-INITIAL_FORMS': 2,
'files-0-id': unlisted_version.all_files[0].pk,
'files-0-status': amo.STATUS_DISABLED,
'files-1-id': listed_version.all_files[0].pk,
'files-1-status': amo.STATUS_AWAITING_REVIEW, # Different status.
})
post_data.update(
**{
'status': amo.STATUS_DISABLED,
'files-TOTAL_FORMS': 2,
'files-INITIAL_FORMS': 2,
'files-0-id': unlisted_version.all_files[0].pk,
'files-0-status': amo.STATUS_DISABLED,
'files-1-id': listed_version.all_files[0].pk,
'files-1-status': amo.STATUS_AWAITING_REVIEW, # Different status.
}
)
# Confirm the original statuses so we know they're actually changing.
assert addon.status != amo.STATUS_DISABLED
assert listed_version.all_files[0].status != amo.STATUS_AWAITING_REVIEW
@ -430,8 +437,7 @@ class TestAddonAdmin(TestCase):
assert response.status_code == 200
addon.reload()
assert addon.status == amo.STATUS_DISABLED
assert ActivityLog.objects.filter(
action=amo.LOG.CHANGE_STATUS.id).exists()
assert ActivityLog.objects.filter(action=amo.LOG.CHANGE_STATUS.id).exists()
listed_version = addon.versions.get(id=listed_version.id)
assert listed_version.all_files[0].status == amo.STATUS_AWAITING_REVIEW
unlisted_version = addon.versions.get(id=unlisted_version.id)
@ -440,9 +446,7 @@ class TestAddonAdmin(TestCase):
def test_status_cannot_change_for_deleted_version(self):
addon = addon_factory(guid='@foo', users=[user_factory()])
file = addon.current_version.all_files[0]
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.grant_permission(user, 'Admin:Advanced')
@ -453,11 +457,12 @@ class TestAddonAdmin(TestCase):
response = self.client.get(self.detail_url, follow=True)
assert response.status_code == 200
assert f'{file.version} - Deleted' in response.content.decode('utf-8')
assert 'disabled' in (
pq(response.content)('#id_files-0-status')[0].attrib)
post_data.update(**{
'files-0-status': amo.STATUS_AWAITING_REVIEW, # Different status.
})
assert 'disabled' in (pq(response.content)('#id_files-0-status')[0].attrib)
post_data.update(
**{
'files-0-status': amo.STATUS_AWAITING_REVIEW, # Different status.
}
)
response = self.client.post(self.detail_url, post_data, follow=True)
assert response.status_code == 200
file.reload()
@ -465,9 +470,7 @@ class TestAddonAdmin(TestCase):
def test_block_status(self):
addon = addon_factory(guid='@foo', users=[user_factory()])
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,)
)
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.grant_permission(user, 'Admin:Advanced')
@ -478,21 +481,20 @@ class TestAddonAdmin(TestCase):
assert 'Blocked' not in response.content.decode('utf-8')
block = Block.objects.create(
addon=addon,
min_version=addon.current_version.version,
updated_by=user)
addon=addon, min_version=addon.current_version.version, updated_by=user
)
response = self.client.get(self.detail_url, follow=True)
assert response.status_code == 200
assert f'Blocked ({addon.current_version.version} - *)' in (
response.content.decode('utf-8'))
response.content.decode('utf-8')
)
link = pq(response.content)('.field-version__is_blocked a')[0]
assert link.attrib['href'] == block.get_admin_url_path()
def test_query_count(self):
addon = addon_factory(guid='@foo', users=[user_factory()])
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,))
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.grant_permission(user, 'Admin:Advanced')
@ -515,8 +517,7 @@ class TestAddonAdmin(TestCase):
addon = addon_factory(users=[user_factory()])
first_file = addon.current_version.all_files[0]
[version_factory(addon=addon) for i in range(0, 30)]
self.detail_url = reverse(
'admin:addons_addon_change', args=(addon.pk,))
self.detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.grant_permission(user, 'Admin:Advanced')
@ -532,14 +533,14 @@ class TestAddonAdmin(TestCase):
assert addon.guid in response.content.decode('utf-8')
assert len(pq(response.content)('.field-version__version')) == 1
assert pq(response.content)('#id_files-0-id')[0].attrib['value'] == (
str(first_file.id))
str(first_file.id)
)
def test_git_extract_action(self):
addon1 = addon_factory()
addon2 = addon_factory()
addons = Addon.objects.filter(
pk__in=(addon1.pk, addon2.pk))
addons = Addon.objects.filter(pk__in=(addon1.pk, addon2.pk))
addon_admin = AddonAdmin(Addon, admin.site)
request = RequestFactory().get('/')
request.user = user_factory()
@ -554,8 +555,7 @@ class TestAddonAdmin(TestCase):
def test_git_extract_button_in_change_view(self):
addon = addon_factory()
git_extract_url = reverse(
'admin:addons_git_extract', args=(addon.pk, ))
git_extract_url = reverse('admin:addons_git_extract', args=(addon.pk,))
detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
@ -567,10 +567,10 @@ class TestAddonAdmin(TestCase):
def test_git_extract(self):
addon = addon_factory()
git_extract_url = reverse(
'admin:addons_git_extract', args=(addon.pk, ))
git_extract_url = reverse('admin:addons_git_extract', args=(addon.pk,))
wrong_git_extract_url = reverse(
'admin:addons_git_extract', args=(addon.pk + 9, ))
'admin:addons_git_extract', args=(addon.pk + 9,)
)
detail_url = reverse('admin:addons_addon_change', args=(addon.pk,))
user = user_factory(email='someone@mozilla.com')
self.client.login(email=user.email)
@ -600,7 +600,8 @@ class TestReplacementAddonList(TestCase):
model_admin = ReplacementAddonAdmin(ReplacementAddon, admin.site)
self.assertEqual(
list(model_admin.get_list_display(None)),
['guid', 'path', 'guid_slug', '_url'])
['guid', 'path', 'guid_slug', '_url'],
)
def test_can_see_replacementaddon_module_in_admin_with_addons_edit(self):
user = user_factory(email='someone@mozilla.com')
@ -612,8 +613,7 @@ class TestReplacementAddonList(TestCase):
# Use django's reverse, since that's what the admin will use. Using our
# own would fail the assertion because of the locale that gets added.
self.list_url = django_reverse(
'admin:addons_replacementaddon_changelist')
self.list_url = django_reverse('admin:addons_replacementaddon_changelist')
assert self.list_url in response.content.decode('utf-8')
def test_can_see_replacementaddon_module_in_admin_with_admin_curate(self):
@ -626,13 +626,11 @@ class TestReplacementAddonList(TestCase):
# Use django's reverse, since that's what the admin will use. Using our
# own would fail the assertion because of the locale that gets added.
self.list_url = django_reverse(
'admin:addons_replacementaddon_changelist')
self.list_url = django_reverse('admin:addons_replacementaddon_changelist')
assert self.list_url in response.content.decode('utf-8')
def test_can_list_with_addons_edit_permission(self):
ReplacementAddon.objects.create(
guid='@bar', path='/addon/bar-replacement/')
ReplacementAddon.objects.create(guid='@bar', path='/addon/bar-replacement/')
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, 'Addons:Edit')
self.client.login(email=user.email)
@ -642,7 +640,8 @@ class TestReplacementAddonList(TestCase):
def test_can_not_edit_with_addons_edit_permission(self):
replacement = ReplacementAddon.objects.create(
guid='@bar', path='/addon/bar-replacement/')
guid='@bar', path='/addon/bar-replacement/'
)
self.detail_url = reverse(
'admin:addons_replacementaddon_change', args=(replacement.pk,)
)
@ -652,13 +651,14 @@ class TestReplacementAddonList(TestCase):
response = self.client.get(self.detail_url, follow=True)
assert response.status_code == 403
response = self.client.post(
self.detail_url, {'guid': '@bar', 'path': replacement.path},
follow=True)
self.detail_url, {'guid': '@bar', 'path': replacement.path}, follow=True
)
assert response.status_code == 403
def test_can_not_delete_with_addons_edit_permission(self):
replacement = ReplacementAddon.objects.create(
guid='@foo', path='/addon/foo-replacement/')
guid='@foo', path='/addon/foo-replacement/'
)
self.delete_url = reverse(
'admin:addons_replacementaddon_delete', args=(replacement.pk,)
)
@ -667,14 +667,14 @@ class TestReplacementAddonList(TestCase):
self.client.login(email=user.email)
response = self.client.get(self.delete_url, follow=True)
assert response.status_code == 403
response = self.client.post(
self.delete_url, data={'post': 'yes'}, follow=True)
response = self.client.post(self.delete_url, data={'post': 'yes'}, follow=True)
assert response.status_code == 403
assert ReplacementAddon.objects.filter(pk=replacement.pk).exists()
def test_can_edit_with_admin_curation_permission(self):
replacement = ReplacementAddon.objects.create(
guid='@foo', path='/addon/foo-replacement/')
guid='@foo', path='/addon/foo-replacement/'
)
self.detail_url = reverse(
'admin:addons_replacementaddon_change', args=(replacement.pk,)
)
@ -686,15 +686,16 @@ class TestReplacementAddonList(TestCase):
assert '/addon/foo-replacement/' in response.content.decode('utf-8')
response = self.client.post(
self.detail_url, {'guid': '@bar', 'path': replacement.path},
follow=True)
self.detail_url, {'guid': '@bar', 'path': replacement.path}, follow=True
)
assert response.status_code == 200
replacement.reload()
assert replacement.guid == '@bar'
def test_can_delete_with_admin_curation_permission(self):
replacement = ReplacementAddon.objects.create(
guid='@foo', path='/addon/foo-replacement/')
guid='@foo', path='/addon/foo-replacement/'
)
self.delete_url = reverse(
'admin:addons_replacementaddon_delete', args=(replacement.pk,)
)
@ -703,8 +704,7 @@ class TestReplacementAddonList(TestCase):
self.client.login(email=user.email)
response = self.client.get(self.delete_url, follow=True)
assert response.status_code == 200
response = self.client.post(
self.delete_url, data={'post': 'yes'}, follow=True)
response = self.client.post(self.delete_url, data={'post': 'yes'}, follow=True)
assert response.status_code == 200
assert not ReplacementAddon.objects.filter(pk=replacement.pk).exists()
@ -718,8 +718,10 @@ class TestReplacementAddonList(TestCase):
assert response.status_code == 200
assert '@foofoo&amp;foo' in response.content.decode('utf-8')
assert '/addon/bar/' in response.content.decode('utf-8')
test_url = str('<a href="%s">Test</a>' % (
reverse('addons.find_replacement') + '?guid=%40foofoo%26foo'))
test_url = str(
'<a href="%s">Test</a>'
% (reverse('addons.find_replacement') + '?guid=%40foofoo%26foo')
)
assert test_url in response.content.decode('utf-8')
# guid is not on AMO so no slug to show
assert '- Add-on not on AMO -' in response.content.decode('utf-8')
@ -732,8 +734,7 @@ class TestReplacementAddonList(TestCase):
class TestAddonRegionalRestrictionsAdmin(TestCase):
def setUp(self):
self.list_url = reverse(
'admin:addons_addonregionalrestrictions_changelist')
self.list_url = reverse('admin:addons_addonregionalrestrictions_changelist')
user = user_factory(email='someone@mozilla.com')
self.grant_permission(user, '*:*')
self.client.login(email=user.email)
@ -746,12 +747,14 @@ class TestAddonRegionalRestrictionsAdmin(TestCase):
# Use django's reverse, since that's what the admin will use. Using our
# own would fail the assertion because of the locale that gets added.
self.list_url = django_reverse(
'admin:addons_addonregionalrestrictions_changelist')
'admin:addons_addonregionalrestrictions_changelist'
)
assert self.list_url in response.content.decode('utf-8')
def test_can_list(self):
AddonRegionalRestrictions.objects.create(
addon=addon_factory(name='éléphant'), excluded_regions=['fr-FR'])
addon=addon_factory(name='éléphant'), excluded_regions=['fr-FR']
)
response = self.client.get(self.list_url, follow=True)
assert response.status_code == 200
assert b'fr-FR' in response.content
@ -766,28 +769,31 @@ class TestAddonRegionalRestrictionsAdmin(TestCase):
assert pq(response.content)('#id_addon') # addon input is editable
response = self.client.post(
self.add_url, {
self.add_url,
{
'excluded_regions': '["DE", "br"]', # should get uppercased
'addon': addon.id},
follow=True)
'addon': addon.id,
},
follow=True,
)
assert response.status_code == 200
restriction = AddonRegionalRestrictions.objects.get(addon=addon)
assert restriction.excluded_regions == ["DE", "BR"]
assert len(mail.outbox) == 1
assert mail.outbox[0].subject == (
'Regional Restriction added for Add-on')
assert mail.outbox[0].subject == ('Regional Restriction added for Add-on')
assert mail.outbox[0].body == (
f'Regional restriction for addon "Thíng" '
f"[{restriction.addon.id}] added: ['DE', 'BR']")
f"[{restriction.addon.id}] added: ['DE', 'BR']"
)
assert mail.outbox[0].to == ['amo-admins@mozilla.com']
def test_can_edit(self):
addon = addon_factory(name='Thíng')
restriction = AddonRegionalRestrictions.objects.create(
addon=addon, excluded_regions=['FR'])
addon=addon, excluded_regions=['FR']
)
self.detail_url = reverse(
'admin:addons_addonregionalrestrictions_change',
args=(restriction.pk,)
'admin:addons_addonregionalrestrictions_change', args=(restriction.pk,)
)
response = self.client.get(self.detail_url, follow=True)
assert response.status_code == 200
@ -795,40 +801,42 @@ class TestAddonRegionalRestrictionsAdmin(TestCase):
assert not pq(response.content)('#id_addon') # addon is readonly
response = self.client.post(
self.detail_url, {
'excluded_regions': '["de", "BR"]', # should get uppercased
self.detail_url,
{
'excluded_regions': '["de", "BR"]', # should get uppercased
# try to change the addon too
'addon': addon_factory().id},
follow=True)
'addon': addon_factory().id,
},
follow=True,
)
assert response.status_code == 200
restriction.reload()
assert restriction.excluded_regions == ["DE", "BR"]
assert restriction.addon == addon # didn't change
assert restriction.addon == addon # didn't change
assert len(mail.outbox) == 1
assert mail.outbox[0].subject == (
'Regional Restriction changed for Add-on')
assert mail.outbox[0].subject == ('Regional Restriction changed for Add-on')
assert mail.outbox[0].body == (
f'Regional restriction for addon "Thíng" '
f"[{restriction.addon.id}] changed: ['DE', 'BR']")
f"[{restriction.addon.id}] changed: ['DE', 'BR']"
)
assert mail.outbox[0].to == ['amo-admins@mozilla.com']
def test_can_delete(self):
restriction = AddonRegionalRestrictions.objects.create(
addon=addon_factory(name='Thíng'), excluded_regions=['FR'])
addon=addon_factory(name='Thíng'), excluded_regions=['FR']
)
self.delete_url = reverse(
'admin:addons_addonregionalrestrictions_delete',
args=(restriction.pk,)
'admin:addons_addonregionalrestrictions_delete', args=(restriction.pk,)
)
response = self.client.get(self.delete_url, follow=True)
assert response.status_code == 200
response = self.client.post(
self.delete_url, data={'post': 'yes'}, follow=True)
response = self.client.post(self.delete_url, data={'post': 'yes'}, follow=True)
assert response.status_code == 200
assert not AddonRegionalRestrictions.objects.exists()
assert len(mail.outbox) == 1
assert mail.outbox[0].subject == (
'Regional Restriction deleted for Add-on')
assert mail.outbox[0].subject == ('Regional Restriction deleted for Add-on')
assert mail.outbox[0].body == (
f'Regional restriction for addon "Thíng" '
f"[{restriction.addon.id}] deleted: ['FR']")
f"[{restriction.addon.id}] deleted: ['FR']"
)
assert mail.outbox[0].to == ['amo-admins@mozilla.com']

289
src/olympia/addons/tests/test_commands.py
Просмотреть файл

@ -12,17 +12,17 @@ import pytest
from olympia import amo
from olympia.addons.management.commands import (
fix_langpacks_with_max_version_star, process_addons)
fix_langpacks_with_max_version_star,
process_addons,
)
from olympia.addons.models import Addon, DeniedGuid
from olympia.abuse.models import AbuseReport
from olympia.amo.tests import (
TestCase, addon_factory, user_factory, version_factory)
from olympia.amo.tests import TestCase, addon_factory, user_factory, version_factory
from olympia.applications.models import AppVersion
from olympia.files.models import FileValidation, WebextPermission
from olympia.ratings.models import Rating
from olympia.reviewers.models import AutoApprovalSummary
from olympia.versions.models import (
ApplicationsVersions, Version, VersionPreview)
from olympia.versions.models import ApplicationsVersions, Version, VersionPreview
def id_function(fixture_value):
@ -38,16 +38,21 @@ def id_function(fixture_value):
unreviewed.
"""
addon_status, file_status, review_type = fixture_value
return '{0}-{1}-{2}'.format(amo.STATUS_CHOICES_API[addon_status],
amo.STATUS_CHOICES_API[file_status],
review_type)
return '{0}-{1}-{2}'.format(
amo.STATUS_CHOICES_API[addon_status],
amo.STATUS_CHOICES_API[file_status],
review_type,
)
@pytest.fixture(
params=[(amo.STATUS_NOMINATED, amo.STATUS_AWAITING_REVIEW, 'full'),
(amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW, 'full')],
params=[
(amo.STATUS_NOMINATED, amo.STATUS_AWAITING_REVIEW, 'full'),
(amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW, 'full'),
],
# ids are used to build better names for the tests using this fixture.
ids=id_function)
ids=id_function,
)
def use_case(request, db):
"""This fixture will return quadruples for different use cases.
@ -108,9 +113,7 @@ def count_subtask_calls(original_function):
@pytest.mark.django_db
def test_process_addons_limit_addons():
user_factory(id=settings.TASK_USER_ID)
addon_ids = [
addon_factory(status=amo.STATUS_APPROVED).id for _ in range(5)
]
addon_ids = [addon_factory(status=amo.STATUS_APPROVED).id for _ in range(5)]
assert Addon.objects.count() == 5
with count_subtask_calls(process_addons.sign_addons) as calls:
@ -127,9 +130,7 @@ def test_process_addons_limit_addons():
@pytest.mark.django_db
@mock.patch.object(process_addons.Command, 'get_pks')
def test_process_addons_batch_size(mock_get_pks):
addon_ids = [
random.randrange(1000) for _ in range(101)
]
addon_ids = [random.randrange(1000) for _ in range(101)]
mock_get_pks.return_value = addon_ids
with count_subtask_calls(process_addons.recreate_previews) as calls:
@ -139,9 +140,7 @@ def test_process_addons_batch_size(mock_get_pks):
assert calls[1]['kwargs']['args'] == [addon_ids[100:]]
with count_subtask_calls(process_addons.recreate_previews) as calls:
call_command(
'process_addons', task='recreate_previews',
**{'batch_size': 50})
call_command('process_addons', task='recreate_previews', **{'batch_size': 50})
assert len(calls) == 3
assert calls[0]['kwargs']['args'] == [addon_ids[:50]]
assert calls[1]['kwargs']['args'] == [addon_ids[50:100]]
@ -151,40 +150,36 @@ def test_process_addons_batch_size(mock_get_pks):
class TestAddDynamicThemeTagForThemeApiCommand(TestCase):
def test_affects_only_public_webextensions(self):
addon_factory()
addon_factory(file_kw={'is_webextension': True,
'status': amo.STATUS_AWAITING_REVIEW},
status=amo.STATUS_NOMINATED)
addon_factory(
file_kw={'is_webextension': True, 'status': amo.STATUS_AWAITING_REVIEW},
status=amo.STATUS_NOMINATED,
)
public_webextension = addon_factory(file_kw={'is_webextension': True})
with count_subtask_calls(
process_addons.add_dynamic_theme_tag) as calls:
call_command(
'process_addons', task='add_dynamic_theme_tag_for_theme_api')
with count_subtask_calls(process_addons.add_dynamic_theme_tag) as calls:
call_command('process_addons', task='add_dynamic_theme_tag_for_theme_api')
assert len(calls) == 1
assert calls[0]['kwargs']['args'] == [
[public_webextension.pk]
]
assert calls[0]['kwargs']['args'] == [[public_webextension.pk]]
def test_tag_added_for_is_dynamic_theme(self):
addon = addon_factory(file_kw={'is_webextension': True})
WebextPermission.objects.create(
file=addon.current_version.all_files[0],
permissions=['theme'])
file=addon.current_version.all_files[0], permissions=['theme']
)
assert addon.tags.all().count() == 0
# Add some more that shouldn't be tagged
no_perms = addon_factory(file_kw={'is_webextension': True})
not_a_theme = addon_factory(file_kw={'is_webextension': True})
WebextPermission.objects.create(
file=not_a_theme.current_version.all_files[0],
permissions=['downloads'])
file=not_a_theme.current_version.all_files[0], permissions=['downloads']
)
call_command(
'process_addons', task='add_dynamic_theme_tag_for_theme_api')
call_command('process_addons', task='add_dynamic_theme_tag_for_theme_api')
assert (
list(addon.tags.all().values_list('tag_text', flat=True)) ==
[u'dynamic theme'])
assert list(addon.tags.all().values_list('tag_text', flat=True)) == [
u'dynamic theme'
]
assert not no_perms.tags.all().exists()
assert not not_a_theme.tags.all().exists()
@ -198,14 +193,15 @@ class RecalculateWeightTestCase(TestCase):
# Non auto-approved add-on that has an AutoApprovalSummary entry,
# should not be considered.
AutoApprovalSummary.objects.create(
version=addon_factory().current_version,
verdict=amo.NOT_AUTO_APPROVED)
version=addon_factory().current_version, verdict=amo.NOT_AUTO_APPROVED
)
# Add-on with the current version not auto-approved, should not be
# considered.
extra_addon = addon_factory()
AutoApprovalSummary.objects.create(
version=extra_addon.current_version, verdict=amo.AUTO_APPROVED)
version=extra_addon.current_version, verdict=amo.AUTO_APPROVED
)
extra_addon.current_version.update(created=self.days_ago(1))
version_factory(addon=extra_addon)
@ -214,25 +210,26 @@ class RecalculateWeightTestCase(TestCase):
already_confirmed_addon = addon_factory()
AutoApprovalSummary.objects.create(
version=already_confirmed_addon.current_version,
verdict=amo.AUTO_APPROVED, confirmed=True)
verdict=amo.AUTO_APPROVED,
confirmed=True,
)
# Add-on that should be considered because it's current version is
# auto-approved.
auto_approved_addon = addon_factory()
AutoApprovalSummary.objects.create(
version=auto_approved_addon.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon.current_version, verdict=amo.AUTO_APPROVED
)
# Add some extra versions that should not have an impact.
version_factory(
addon=auto_approved_addon,
file_kw={'status': amo.STATUS_AWAITING_REVIEW})
version_factory(
addon=auto_approved_addon, channel=amo.RELEASE_CHANNEL_UNLISTED)
addon=auto_approved_addon, file_kw={'status': amo.STATUS_AWAITING_REVIEW}
)
version_factory(addon=auto_approved_addon, channel=amo.RELEASE_CHANNEL_UNLISTED)
with count_subtask_calls(
process_addons.recalculate_post_review_weight) as calls:
call_command(
'process_addons', task='recalculate_post_review_weight')
process_addons.recalculate_post_review_weight
) as calls:
call_command('process_addons', task='recalculate_post_review_weight')
assert len(calls) == 1
assert calls[0]['kwargs']['args'] == [[auto_approved_addon.pk]]
@ -240,14 +237,15 @@ class RecalculateWeightTestCase(TestCase):
def test_task_works_correctly(self):
addon = addon_factory(average_daily_users=100000)
FileValidation.objects.create(
file=addon.current_version.all_files[0], validation=u'{}')
file=addon.current_version.all_files[0], validation=u'{}'
)
addon = Addon.objects.get(pk=addon.pk)
summary = AutoApprovalSummary.objects.create(
version=addon.current_version, verdict=amo.AUTO_APPROVED)
version=addon.current_version, verdict=amo.AUTO_APPROVED
)
assert summary.weight == 0
call_command(
'process_addons', task='recalculate_post_review_weight')
call_command('process_addons', task='recalculate_post_review_weight')
summary.reload()
# Weight should be 10 because of average_daily_users / 10000.
@ -262,13 +260,14 @@ class ConstantlyRecalculateWeightTestCase(TestCase):
# *not considered* - Non auto-approved add-on that has an
# AutoApprovalSummary entry
AutoApprovalSummary.objects.create(
version=addon_factory().current_version,
verdict=amo.NOT_AUTO_APPROVED)
version=addon_factory().current_version, verdict=amo.NOT_AUTO_APPROVED
)
# *not considered* -Add-on with the current version not auto-approved
extra_addon = addon_factory()
AutoApprovalSummary.objects.create(
version=extra_addon.current_version, verdict=amo.AUTO_APPROVED)
version=extra_addon.current_version, verdict=amo.AUTO_APPROVED
)
extra_addon.current_version.update(created=self.days_ago(1))
version_factory(addon=extra_addon)
@ -276,87 +275,99 @@ class ConstantlyRecalculateWeightTestCase(TestCase):
# have recent abuse reports or low ratings
auto_approved_addon = addon_factory()
AutoApprovalSummary.objects.create(
version=auto_approved_addon.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon.current_version, verdict=amo.AUTO_APPROVED
)
# *considered* - current version is auto-approved and
# has a recent rating with rating <= 3
auto_approved_addon1 = addon_factory()
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon1.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon1.current_version, verdict=amo.AUTO_APPROVED
)
Rating.objects.create(
created=summary.modified + timedelta(days=3),
addon=auto_approved_addon1,
version=auto_approved_addon1.current_version,
rating=2, body='Apocalypse', user=user_factory()),
rating=2,
body='Apocalypse',
user=user_factory(),
),
# *not considered* - current version is auto-approved but
# has a recent rating with rating > 3
auto_approved_addon2 = addon_factory()
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon2.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon2.current_version, verdict=amo.AUTO_APPROVED
)
Rating.objects.create(
created=summary.modified + timedelta(days=3),
addon=auto_approved_addon2,
version=auto_approved_addon2.current_version,
rating=4, body='Apocalypse', user=user_factory()),
rating=4,
body='Apocalypse',
user=user_factory(),
),
# *not considered* - current version is auto-approved but
# has a recent rating with rating > 3
auto_approved_addon3 = addon_factory()
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon3.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon3.current_version, verdict=amo.AUTO_APPROVED
)
Rating.objects.create(
created=summary.modified + timedelta(days=3),
addon=auto_approved_addon3,
version=auto_approved_addon3.current_version,
rating=4, body='Apocalypse', user=user_factory()),
rating=4,
body='Apocalypse',
user=user_factory(),
),
# *not considered* - current version is auto-approved but
# has a low rating that isn't recent enough
auto_approved_addon4 = addon_factory()
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon4.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon4.current_version, verdict=amo.AUTO_APPROVED
)
Rating.objects.create(
created=summary.modified - timedelta(days=3),
addon=auto_approved_addon4,
version=auto_approved_addon4.current_version,
rating=1, body='Apocalypse', user=user_factory()),
rating=1,
body='Apocalypse',
user=user_factory(),
),
# *considered* - current version is auto-approved and
# has a recent abuse report
auto_approved_addon5 = addon_factory()
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon5.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon5.current_version, verdict=amo.AUTO_APPROVED
)
AbuseReport.objects.create(
addon=auto_approved_addon5,
created=summary.modified + timedelta(days=3))
addon=auto_approved_addon5, created=summary.modified + timedelta(days=3)
)
# *not considered* - current version is auto-approved but
# has an abuse report that isn't recent enough
auto_approved_addon6 = addon_factory()
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon6.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon6.current_version, verdict=amo.AUTO_APPROVED
)
AbuseReport.objects.create(
addon=auto_approved_addon6,
created=summary.modified - timedelta(days=3))
addon=auto_approved_addon6, created=summary.modified - timedelta(days=3)
)
# *considered* - current version is auto-approved and
# has an abuse report through it's author that is recent enough
author = user_factory()
auto_approved_addon7 = addon_factory(users=[author])
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon7.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon7.current_version, verdict=amo.AUTO_APPROVED
)
AbuseReport.objects.create(
user=author,
created=summary.modified + timedelta(days=3))
user=author, created=summary.modified + timedelta(days=3)
)
# *not considered* - current version is auto-approved and
# has an abuse report through it's author that is recent enough
@ -364,26 +375,30 @@ class ConstantlyRecalculateWeightTestCase(TestCase):
author = user_factory()
auto_approved_addon8 = addon_factory(users=[author])
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon8.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon8.current_version, verdict=amo.AUTO_APPROVED
)
AbuseReport.objects.create(
user=author,
state=AbuseReport.STATES.DELETED,
created=summary.modified + timedelta(days=3))
created=summary.modified + timedelta(days=3),
)
# *not considered* - current version is auto-approved and
# has a recent rating with rating <= 3
# but the rating is deleted.
auto_approved_addon9 = addon_factory()
summary = AutoApprovalSummary.objects.create(
version=auto_approved_addon9.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon9.current_version, verdict=amo.AUTO_APPROVED
)
Rating.objects.create(
created=summary.modified + timedelta(days=3),
addon=auto_approved_addon9,
version=auto_approved_addon9.current_version,
deleted=True,
rating=2, body='Apocalypse', user=user_factory()),
rating=2,
body='Apocalypse',
user=user_factory(),
),
# *considered* - current version is auto-approved and
# has an abuse report through it's author that is recent enough
@ -391,39 +406,42 @@ class ConstantlyRecalculateWeightTestCase(TestCase):
# the most recent version
author = user_factory()
auto_approved_addon8 = addon_factory(
users=[author], version_kw={'version': '0.1'})
users=[author], version_kw={'version': '0.1'}
)
AutoApprovalSummary.objects.create(
version=auto_approved_addon8.current_version,
verdict=amo.AUTO_APPROVED)
version=auto_approved_addon8.current_version, verdict=amo.AUTO_APPROVED
)
# Let's create a new `current_version` and summary
current_version = version_factory(
addon=auto_approved_addon8, version='0.2')
current_version = version_factory(addon=auto_approved_addon8, version='0.2')
summary = AutoApprovalSummary.objects.create(
version=current_version,
verdict=amo.AUTO_APPROVED)
version=current_version, verdict=amo.AUTO_APPROVED
)
AbuseReport.objects.create(
user=author,
created=summary.modified + timedelta(days=3))
user=author, created=summary.modified + timedelta(days=3)
)
mod = 'olympia.reviewers.tasks.AutoApprovalSummary.calculate_weight'
with mock.patch(mod) as calc_weight_mock:
with count_subtask_calls(
process_addons.recalculate_post_review_weight) as calls:
process_addons.recalculate_post_review_weight
) as calls:
call_command(
'process_addons',
task='constantly_recalculate_post_review_weight')
'process_addons', task='constantly_recalculate_post_review_weight'
)
assert len(calls) == 1
assert calls[0]['kwargs']['args'] == [[
auto_approved_addon1.pk,
auto_approved_addon5.pk,
auto_approved_addon7.pk,
auto_approved_addon8.pk,
]]
assert calls[0]['kwargs']['args'] == [
[
auto_approved_addon1.pk,
auto_approved_addon5.pk,
auto_approved_addon7.pk,
auto_approved_addon8.pk,
]
]
# Only 4 calls for each add-on, doesn't consider the extra version
# that got created for addon 8
@ -436,14 +454,11 @@ class TestExtractColorsFromStaticThemes(TestCase):
addon = addon_factory(type=amo.ADDON_STATICTHEME)
preview = VersionPreview.objects.create(version=addon.current_version)
extract_colors_from_image_mock.return_value = [
{'h': 4, 's': 8, 'l': 15, 'ratio': .16}
{'h': 4, 's': 8, 'l': 15, 'ratio': 0.16}
]
call_command(
'process_addons', task='extract_colors_from_static_themes')
call_command('process_addons', task='extract_colors_from_static_themes')
preview.reload()
assert preview.colors == [
{'h': 4, 's': 8, 'l': 15, 'ratio': .16}
]
assert preview.colors == [{'h': 4, 's': 8, 'l': 15, 'ratio': 0.16}]
class TestResignAddonsForCose(TestCase):
@ -504,8 +519,7 @@ class TestDeleteObsoleteAddons(TestCase):
DeniedGuid.objects.create(guid=self.xul_theme.guid)
DeniedGuid.objects.all().count() == 1
call_command(
'process_addons', task='delete_obsolete_addons', with_deleted=True)
call_command('process_addons', task='delete_obsolete_addons', with_deleted=True)
assert Addon.unfiltered.count() == 3
assert Addon.unfiltered.get(id=self.extension.id)
@ -525,8 +539,7 @@ class TestDeleteObsoleteAddons(TestCase):
self.test_hard()
def test_normal(self):
call_command(
'process_addons', task='delete_obsolete_addons')
call_command('process_addons', task='delete_obsolete_addons')
assert Addon.unfiltered.count() == 8
assert Addon.objects.count() == 3
@ -538,37 +551,49 @@ class TestDeleteObsoleteAddons(TestCase):
class TestFixLangpacksWithMaxVersionStar(TestCase):
def setUp(self):
addon = addon_factory( # Should autocreate the AppVersions for Firefox
type=amo.ADDON_LPAPP, version_kw={
type=amo.ADDON_LPAPP,
version_kw={
'min_app_version': '77.0',
'max_app_version': '*',
}
},
)
# Add the missing AppVersions for Android, and assign them to the addon
min_android = AppVersion.objects.get_or_create(
application=amo.ANDROID.id, version='77.0')[0]
application=amo.ANDROID.id, version='77.0'
)[0]
max_android_star = AppVersion.objects.get_or_create(
application=amo.ANDROID.id, version='*')[0]
application=amo.ANDROID.id, version='*'
)[0]
ApplicationsVersions.objects.create(
application=amo.ANDROID.id, version=addon.current_version,
min=min_android, max=max_android_star)
application=amo.ANDROID.id,
version=addon.current_version,
min=min_android,
max=max_android_star,
)
addon_factory( # Same kind of langpack, but without android compat.
type=amo.ADDON_LPAPP, version_kw={
type=amo.ADDON_LPAPP,
version_kw={
'min_app_version': '77.0',
'max_app_version': '*',
}
},
)
addon = addon_factory( # Shouldn't be touched, its max is not '*'.
type=amo.ADDON_LPAPP, version_kw={
type=amo.ADDON_LPAPP,
version_kw={
'min_app_version': '77.0',
'max_app_version': '77.*',
}
},
)
max_android = AppVersion.objects.get_or_create(
application=amo.ANDROID.id, version='77.*')[0]
application=amo.ANDROID.id, version='77.*'
)[0]
ApplicationsVersions.objects.create(
application=amo.ANDROID.id, version=addon.current_version,
min=min_android, max=max_android)
application=amo.ANDROID.id,
version=addon.current_version,
min=min_android,
max=max_android,
)
def test_find_affected_langpacks(self):
command = fix_langpacks_with_max_version_star.Command()

105
src/olympia/addons/tests/test_cron.py
Просмотреть файл

@ -26,13 +26,13 @@ class TestLastUpdated(TestCase):
"""Make sure the catch-all last_updated is stable and accurate."""
# Nullify all datestatuschanged so the public add-ons hit the
# catch-all.
(File.objects.filter(status=amo.STATUS_APPROVED)
.update(datestatuschanged=None))
(File.objects.filter(status=amo.STATUS_APPROVED).update(datestatuschanged=None))
Addon.objects.update(last_updated=None)
cron.addon_last_updated()
for addon in Addon.objects.filter(status=amo.STATUS_APPROVED,
type=amo.ADDON_EXTENSION):
for addon in Addon.objects.filter(
status=amo.STATUS_APPROVED, type=amo.ADDON_EXTENSION
):
assert addon.last_updated == addon.created
# Make sure it's stable.
@ -53,8 +53,7 @@ class TestLastUpdated(TestCase):
AppSupport.objects.all().delete()
assert AppSupport.objects.filter(addon=3723).count() == 0
cron.update_addon_appsupport()
assert AppSupport.objects.filter(
addon=3723, app=amo.FIREFOX.id).count() == 0
assert AppSupport.objects.filter(addon=3723, app=amo.FIREFOX.id).count() == 0
class TestHideDisabledFiles(TestCase):
@ -64,16 +63,20 @@ class TestHideDisabledFiles(TestCase):
super(TestHideDisabledFiles, self).setUp()
self.addon = Addon.objects.create(type=amo.ADDON_EXTENSION)
self.version = Version.objects.create(addon=self.addon)
self.f1 = File.objects.create(version=self.version, filename='f1',
platform=amo.PLATFORM_ALL.id)
self.f2 = File.objects.create(version=self.version, filename='f2',
platform=amo.PLATFORM_ALL.id)
self.f1 = File.objects.create(
version=self.version, filename='f1', platform=amo.PLATFORM_ALL.id
)
self.f2 = File.objects.create(
version=self.version, filename='f2', platform=amo.PLATFORM_ALL.id
)
@mock.patch('olympia.files.models.os')
def test_leave_nondisabled_files(self, os_mock):
# All these addon/file status pairs should stay.
stati = ((amo.STATUS_APPROVED, amo.STATUS_APPROVED),
(amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW))
stati = (
(amo.STATUS_APPROVED, amo.STATUS_APPROVED),
(amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW),
)
for addon_status, file_status in stati:
self.addon.update(status=addon_status)
File.objects.update(status=file_status)
@ -86,52 +89,44 @@ class TestHideDisabledFiles(TestCase):
def test_move_user_disabled_addon(self, mv_mock):
# Use Addon.objects.update so the signal handler isn't called.
Addon.objects.filter(id=self.addon.id).update(
status=amo.STATUS_APPROVED, disabled_by_user=True)
status=amo.STATUS_APPROVED, disabled_by_user=True
)
File.objects.update(status=amo.STATUS_APPROVED)
cron.hide_disabled_files()
# Check that f2 was moved.
f2 = self.f2
mv_mock.assert_called_with(f2.file_path, f2.guarded_file_path,
self.msg)
mv_mock.assert_called_with(f2.file_path, f2.guarded_file_path, self.msg)
# Check that f1 was moved as well.
f1 = self.f1
mv_mock.call_args = mv_mock.call_args_list[0]
mv_mock.assert_called_with(f1.file_path, f1.guarded_file_path,
self.msg)
mv_mock.assert_called_with(f1.file_path, f1.guarded_file_path, self.msg)
# There's only 2 files, both should have been moved.
assert mv_mock.call_count == 2
@mock.patch('olympia.files.models.File.move_file')
def test_move_admin_disabled_addon(self, mv_mock):
Addon.objects.filter(id=self.addon.id).update(
status=amo.STATUS_DISABLED)
Addon.objects.filter(id=self.addon.id).update(status=amo.STATUS_DISABLED)
File.objects.update(status=amo.STATUS_APPROVED)
cron.hide_disabled_files()
# Check that f2 was moved.
f2 = self.f2
mv_mock.assert_called_with(f2.file_path, f2.guarded_file_path,
self.msg)
mv_mock.assert_called_with(f2.file_path, f2.guarded_file_path, self.msg)
# Check that f1 was moved as well.
f1 = self.f1
mv_mock.call_args = mv_mock.call_args_list[0]
mv_mock.assert_called_with(f1.file_path, f1.guarded_file_path,
self.msg)
mv_mock.assert_called_with(f1.file_path, f1.guarded_file_path, self.msg)
# There's only 2 files, both should have been moved.
assert mv_mock.call_count == 2
@mock.patch('olympia.files.models.File.move_file')
def test_move_disabled_file(self, mv_mock):
Addon.objects.filter(id=self.addon.id).update(
status=amo.STATUS_APPROVED)
File.objects.filter(id=self.f1.id).update(
status=amo.STATUS_DISABLED)
File.objects.filter(id=self.f2.id).update(
status=amo.STATUS_AWAITING_REVIEW)
Addon.objects.filter(id=self.addon.id).update(status=amo.STATUS_APPROVED)
File.objects.filter(id=self.f1.id).update(status=amo.STATUS_DISABLED)
File.objects.filter(id=self.f2.id).update(status=amo.STATUS_AWAITING_REVIEW)
cron.hide_disabled_files()
# Only f1 should have been moved.
f1 = self.f1
mv_mock.assert_called_with(f1.file_path, f1.guarded_file_path,
self.msg)
mv_mock.assert_called_with(f1.file_path, f1.guarded_file_path, self.msg)
assert mv_mock.call_count == 1
@mock.patch('olympia.files.models.storage.exists')
@ -144,7 +139,8 @@ class TestHideDisabledFiles(TestCase):
# Use Addon.objects.update so the signal handler isn't called.
Addon.objects.filter(id=self.addon.id).update(
status=amo.STATUS_APPROVED, disabled_by_user=True)
status=amo.STATUS_APPROVED, disabled_by_user=True
)
File.objects.update(status=amo.STATUS_APPROVED)
cron.hide_disabled_files()
@ -171,7 +167,8 @@ class TestUnhideDisabledFiles(TestCase):
self.addon = Addon.objects.create(type=amo.ADDON_EXTENSION)
self.version = Version.objects.create(addon=self.addon)
self.file_ = File.objects.create(
version=self.version, platform=amo.PLATFORM_ALL.id, filename=u'fé')
version=self.version, platform=amo.PLATFORM_ALL.id, filename=u'fé'
)
@mock.patch('olympia.files.models.os')
def test_leave_disabled_files(self, os_mock):
@ -201,7 +198,8 @@ class TestUnhideDisabledFiles(TestCase):
self.file_.update(status=amo.STATUS_APPROVED)
cron.unhide_disabled_files()
mv_mock.assert_called_with(
self.file_.guarded_file_path, self.file_.file_path, self.msg)
self.file_.guarded_file_path, self.file_.file_path, self.msg
)
assert mv_mock.call_count == 1
def test_cleans_up_empty_directories_after_moving(self):
@ -217,14 +215,14 @@ class TestUnhideDisabledFiles(TestCase):
assert storage.exists(self.file_.file_path)
assert not storage.exists(self.file_.guarded_file_path)
# Empty dir also removed:
assert not storage.exists(
os.path.dirname(self.file_.guarded_file_path))
assert not storage.exists(os.path.dirname(self.file_.guarded_file_path))
def test_doesnt_remove_non_empty_directories(self):
# Add an extra disabled file. The approved one should move, but not the
# other, so the directory should be left intact.
self.disabled_file = file_factory(
version=self.version, status=amo.STATUS_DISABLED)
version=self.version, status=amo.STATUS_DISABLED
)
self.addon.update(status=amo.STATUS_APPROVED)
self.file_.update(status=amo.STATUS_APPROVED)
with storage.open(self.file_.guarded_file_path, 'wb') as fp:
@ -267,9 +265,7 @@ class TestAvgDailyUserCountTestCase(TestCase):
def setUp(self):
super().setUp()
@mock.patch(
'olympia.addons.cron.get_addons_and_average_daily_users_from_bigquery'
)
@mock.patch('olympia.addons.cron.get_addons_and_average_daily_users_from_bigquery')
def test_update_addon_average_daily_users_with_bigquery(self, get_mock):
addon = Addon.objects.get(pk=3615)
addon.update(average_daily_users=0)
@ -278,8 +274,7 @@ class TestAvgDailyUserCountTestCase(TestCase):
langpack_count = 12345
dictionary = addon_factory(type=amo.ADDON_DICT, average_daily_users=0)
dictionary_count = 5567
addon_without_count = addon_factory(type=amo.ADDON_DICT,
average_daily_users=2)
addon_without_count = addon_factory(type=amo.ADDON_DICT, average_daily_users=2)
deleted_addon = addon_factory(average_daily_users=0)
deleted_addon_count = 23456
deleted_addon.delete()
@ -312,9 +307,7 @@ class TestAvgDailyUserCountTestCase(TestCase):
assert addon_without_count.average_daily_users == 0
@mock.patch('olympia.addons.cron.create_chunked_tasks_signatures')
@mock.patch(
'olympia.addons.cron.get_addons_and_average_daily_users_from_bigquery'
)
@mock.patch('olympia.addons.cron.get_addons_and_average_daily_users_from_bigquery')
def test_update_addon_average_daily_users_values_with_bigquery(
self, get_mock, create_chunked_mock
):
@ -326,8 +319,7 @@ class TestAvgDailyUserCountTestCase(TestCase):
langpack_count = 12345
dictionary = addon_factory(type=amo.ADDON_DICT, average_daily_users=0)
dictionary_count = 6789
addon_without_count = addon_factory(type=amo.ADDON_DICT,
average_daily_users=2)
addon_without_count = addon_factory(type=amo.ADDON_DICT, average_daily_users=2)
# This one should be ignored.
addon_factory(guid=None, type=amo.ADDON_LPAPP)
# This one should be ignored as well.
@ -356,7 +348,7 @@ class TestAvgDailyUserCountTestCase(TestCase):
(dictionary.guid, dictionary_count),
(deleted_addon.guid, deleted_addon_count),
],
chunk_size
chunk_size,
)
@ -445,12 +437,8 @@ class TestUpdateAddonHotness(TestCase):
class TestUpdateAddonWeeklyDownloads(TestCase):
@mock.patch('olympia.addons.cron.create_chunked_tasks_signatures')
@mock.patch(
'olympia.addons.cron.get_addons_and_weekly_downloads_from_bigquery'
)
def test_calls_create_chunked_tasks_signatures(
self, get_mock, create_chunked_mock
):
@mock.patch('olympia.addons.cron.get_addons_and_weekly_downloads_from_bigquery')
def test_calls_create_chunked_tasks_signatures(self, get_mock, create_chunked_mock):
create_chunked_mock.return_value = group([])
addon = addon_factory(weekly_downloads=0)
count = 56789
@ -458,8 +446,7 @@ class TestUpdateAddonWeeklyDownloads(TestCase):
langpack_count = 12345
dictionary = addon_factory(type=amo.ADDON_DICT, weekly_downloads=0)
dictionary_count = 6789
addon_without_count = addon_factory(type=amo.ADDON_DICT,
weekly_downloads=2)
addon_without_count = addon_factory(type=amo.ADDON_DICT, weekly_downloads=2)
# This one should be ignored.
addon_factory(guid=None, type=amo.ADDON_LPAPP)
# This one should be ignored as well.
@ -481,12 +468,10 @@ class TestUpdateAddonWeeklyDownloads(TestCase):
(langpack.addonguid.hashed_guid, langpack_count),
(dictionary.addonguid.hashed_guid, dictionary_count),
],
chunk_size
chunk_size,
)
@mock.patch(
'olympia.addons.cron.get_addons_and_weekly_downloads_from_bigquery'
)
@mock.patch('olympia.addons.cron.get_addons_and_weekly_downloads_from_bigquery')
def test_update_weekly_downloads(self, get_mock):
addon = addon_factory(weekly_downloads=0)
count = 56789

69
src/olympia/addons/tests/test_decorators.py
Просмотреть файл

@ -9,7 +9,6 @@ from olympia.amo.tests import TestCase, addon_factory
class TestAddonView(TestCase):
def setUp(self):
super(TestAddonView, self).setUp()
self.addon = addon_factory()
@ -18,11 +17,12 @@ class TestAddonView(TestCase):
self.func.__name__ = 'mock_function'
self.view = dec.addon_view(self.func)
self.request = mock.Mock()
self.slug_path = (
'http://testserver/addon/%s/reviews' %
quote(self.addon.slug.encode('utf-8')))
self.slug_path = 'http://testserver/addon/%s/reviews' % quote(
self.addon.slug.encode('utf-8')
)
self.request.path = self.id_path = (
u'http://testserver/addon/%s/reviews' % self.addon.id)
u'http://testserver/addon/%s/reviews' % self.addon.id
)
self.request.GET = {}
def test_301_by_id(self):
@ -30,8 +30,7 @@ class TestAddonView(TestCase):
self.assert3xx(res, self.slug_path, 301)
def test_301_by_guid(self):
self.request.path = (
u'http://testserver/addon/%s/reviews' % self.addon.guid)
self.request.path = u'http://testserver/addon/%s/reviews' % self.addon.guid
res = self.view(self.request, str(self.addon.guid))
self.assert3xx(res, self.slug_path, 301)
@ -40,10 +39,9 @@ class TestAddonView(TestCase):
self.request.path = path.format(id=self.addon.id)
res = self.view(self.request, str(self.addon.id))
redirection = (
u'http://testserver/addon/{slug}/reviews/{id}345/path'.format(
id=self.addon.id,
slug=quote(self.addon.slug.encode('utf8'))))
redirection = u'http://testserver/addon/{slug}/reviews/{id}345/path'.format(
id=self.addon.id, slug=quote(self.addon.slug.encode('utf8'))
)
self.assert3xx(res, redirection, 301)
def test_301_with_querystring(self):
@ -109,20 +107,20 @@ class TestAddonView(TestCase):
request, addon_ = self.func.call_args[0]
assert addon_ == addon
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer',
lambda r: False)
@mock.patch('olympia.access.acl.check_addon_ownership',
lambda *args, **kwargs: False)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer', lambda r: False)
@mock.patch(
'olympia.access.acl.check_addon_ownership', lambda *args, **kwargs: False
)
def test_no_versions_404(self):
self.addon.current_version.delete()
view = dec.addon_view_factory(qs=Addon.objects.all)(self.func)
with self.assertRaises(http.Http404):
view(self.request, self.addon.slug)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer',
lambda r: False)
@mock.patch('olympia.access.acl.check_addon_ownership',
lambda *args, **kwargs: True)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer', lambda r: False)
@mock.patch(
'olympia.access.acl.check_addon_ownership', lambda *args, **kwargs: True
)
def test_no_versions_developer(self):
self.addon.current_version.delete()
res = self.view(self.request, self.addon.slug)
@ -131,34 +129,31 @@ class TestAddonView(TestCase):
def test_no_versions_include_deleted_when_checking(self):
self.addon.current_version.delete()
view = dec.addon_view( # Not available on the factory
self.func,
qs=Addon.objects.all,
include_deleted_when_checking_versions=True)
self.func, qs=Addon.objects.all, include_deleted_when_checking_versions=True
)
res = view(self.request, self.addon.slug)
assert res == mock.sentinel.OK
class TestAddonViewWithUnlisted(TestAddonView):
def setUp(self):
super(TestAddonViewWithUnlisted, self).setUp()
self.view = dec.addon_view_factory(
qs=Addon.objects.all)(self.func)
self.view = dec.addon_view_factory(qs=Addon.objects.all)(self.func)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer',
lambda r: False)
@mock.patch('olympia.access.acl.check_addon_ownership',
lambda *args, **kwargs: False)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer', lambda r: False)
@mock.patch(
'olympia.access.acl.check_addon_ownership', lambda *args, **kwargs: False
)
def test_unlisted_addon(self):
"""Return a 404 for non authorized access."""
self.make_addon_unlisted(self.addon)
with self.assertRaises(http.Http404):
self.view(self.request, self.addon.slug)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer',
lambda r: False)
@mock.patch('olympia.access.acl.check_addon_ownership',
lambda *args, **kwargs: True)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer', lambda r: False)
@mock.patch(
'olympia.access.acl.check_addon_ownership', lambda *args, **kwargs: True
)
def test_unlisted_addon_owner(self):
"""Addon owners have access."""
self.make_addon_unlisted(self.addon)
@ -166,10 +161,10 @@ class TestAddonViewWithUnlisted(TestAddonView):
request, addon = self.func.call_args[0]
assert addon == self.addon
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer',
lambda r: True)
@mock.patch('olympia.access.acl.check_addon_ownership',
lambda *args, **kwargs: False)
@mock.patch('olympia.access.acl.check_unlisted_addons_reviewer', lambda r: True)
@mock.patch(
'olympia.access.acl.check_addon_ownership', lambda *args, **kwargs: False
)
def test_unlisted_addon_unlisted_admin(self):
"""Unlisted addon reviewers have access."""
self.make_addon_unlisted(self.addon)

278
src/olympia/addons/tests/test_indexers.py
Просмотреть файл

@ -6,8 +6,7 @@ from django.conf import settings
from olympia import amo
from olympia.addons.indexers import AddonIndexer
from olympia.addons.models import (
Addon, Preview, attach_tags, attach_translations_dict)
from olympia.addons.models import Addon, Preview, attach_tags, attach_translations_dict
from olympia.amo.models import SearchMixin
from olympia.amo.tests import addon_factory, ESTestCase, TestCase, file_factory
from olympia.bandwagon.models import Collection
@ -27,10 +26,24 @@ class TestAddonIndexer(TestCase):
# This only contains the fields for which we use the value directly,
# see expected_fields() for the rest.
simple_fields = [
'average_daily_users', 'bayesian_rating', 'contributions', 'created',
'default_locale', 'guid', 'hotness', 'icon_hash', 'icon_type', 'id',
'is_disabled', 'is_experimental', 'last_updated',
'modified', 'requires_payment', 'slug', 'status', 'type',
'average_daily_users',
'bayesian_rating',
'contributions',
'created',
'default_locale',
'guid',
'hotness',
'icon_hash',
'icon_type',
'id',
'is_disabled',
'is_experimental',
'last_updated',
'modified',
'requires_payment',
'slug',
'status',
'type',
'weekly_downloads',
]
@ -53,10 +66,22 @@ class TestAddonIndexer(TestCase):
# exist on the model, or it has a different name, or the value we need
# to store in ES differs from the one in the db.
complex_fields = [
'app', 'boost', 'category', 'colors', 'current_version',
'description', 'has_eula', 'has_privacy_policy', 'is_recommended',
'app',
'boost',
'category',
'colors',
'current_version',
'description',
'has_eula',
'has_privacy_policy',
'is_recommended',
'listed_authors',
'name', 'platforms', 'previews', 'promoted', 'ratings', 'summary',
'name',
'platforms',
'previews',
'promoted',
'ratings',
'summary',
'tags',
]
@ -67,26 +92,43 @@ class TestAddonIndexer(TestCase):
# For each translated field that needs to be indexed, we store one
# version for each language we have an analyzer for.
_indexed_translated_fields = ('name', 'description', 'summary')
analyzer_fields = list(chain.from_iterable(
[['%s_l10n_%s' % (field, lang) for lang, analyzer
in SEARCH_LANGUAGE_TO_ANALYZER.items()]
for field in _indexed_translated_fields]))
analyzer_fields = list(
chain.from_iterable(
[
[
'%s_l10n_%s' % (field, lang)
for lang, analyzer in SEARCH_LANGUAGE_TO_ANALYZER.items()
]
for field in _indexed_translated_fields
]
)
)
# It'd be annoying to hardcode `analyzer_fields`, so we generate it,
# but to make sure the test is correct we still do a simple check of
# the length to make sure we properly flattened the list.
assert len(analyzer_fields) == (len(SEARCH_LANGUAGE_TO_ANALYZER) *
len(_indexed_translated_fields))
assert len(analyzer_fields) == (
len(SEARCH_LANGUAGE_TO_ANALYZER) * len(_indexed_translated_fields)
)
# Each translated field that we want to return to the API.
raw_translated_fields = [
'%s_translations' % field for field in
['name', 'description', 'developer_comments', 'homepage',
'summary', 'support_email', 'support_url']]
'%s_translations' % field
for field in [
'name',
'description',
'developer_comments',
'homepage',
'summary',
'support_email',
'support_url',
]
]
# Return a list with the base fields and the dynamic ones added.
fields = (cls.simple_fields + complex_fields + analyzer_fields +
raw_translated_fields)
fields = (
cls.simple_fields + complex_fields + analyzer_fields + raw_translated_fields
)
if include_nullable:
fields += nullable_fields
return fields
@ -111,17 +153,33 @@ class TestAddonIndexer(TestCase):
assert mapping_properties['current_version']['properties']
version_mapping = mapping_properties['current_version']['properties']
expected_version_keys = (
'id', 'compatible_apps', 'files', 'license',
'release_notes_translations', 'reviewed', 'version')
'id',
'compatible_apps',
'files',
'license',
'release_notes_translations',
'reviewed',
'version',
)
assert set(version_mapping.keys()) == set(expected_version_keys)
# Make sure files mapping is set inside current_version.
files_mapping = version_mapping['files']['properties']
expected_file_keys = (
'id', 'created', 'filename', 'hash', 'is_webextension',
'is_restart_required', 'is_mozilla_signed_extension', 'platform',
'size', 'status', 'strict_compatibility',
'permissions', 'optional_permissions')
'id',
'created',
'filename',
'hash',
'is_webextension',
'is_restart_required',
'is_mozilla_signed_extension',
'platform',
'size',
'status',
'strict_compatibility',
'permissions',
'optional_permissions',
)
assert set(files_mapping.keys()) == set(expected_file_keys)
def test_index_setting_boolean(self):
@ -138,20 +196,24 @@ class TestAddonIndexer(TestCase):
assert all(
isinstance(prop['index'], bool)
for prop in mapping_properties.values()
if 'index' in prop)
if 'index' in prop
)
# Make sure our version_mapping is setup correctly too.
props = mapping_properties['current_version']['properties']
assert all(
isinstance(prop['index'], bool)
for prop in props.values() if 'index' in prop)
for prop in props.values()
if 'index' in prop
)
# As well as for current_version.files
assert all(
isinstance(prop['index'], bool)
for prop in props['files']['properties'].values()
if 'index' in prop)
if 'index' in prop
)
def _extract(self):
qs = Addon.unfiltered.filter(id__in=[self.addon.pk])
@ -167,19 +229,25 @@ class TestAddonIndexer(TestCase):
# Make sure the method does not return fields we did not expect to be
# present, or omitted fields we want.
assert set(extracted.keys()) == set(
self.expected_fields(include_nullable=False))
self.expected_fields(include_nullable=False)
)
# Check base fields values. Other tests below check the dynamic ones.
for field_name in self.simple_fields:
assert extracted[field_name] == getattr(self.addon, field_name)
assert extracted['app'] == [FIREFOX.id]
assert extracted['boost'] == self.addon.average_daily_users ** .2 * 4
assert extracted['boost'] == self.addon.average_daily_users ** 0.2 * 4
assert extracted['category'] == [1, 22, 71] # From fixture.
assert extracted['current_version']
assert extracted['listed_authors'] == [
{'name': u'55021 التطب', 'id': 55021, 'username': '55021',
'is_public': True}]
{
'name': u'55021 التطب',
'id': 55021,
'username': '55021',
'is_public': True,
}
]
assert extracted['platforms'] == [PLATFORM_ALL.id]
assert extracted['ratings'] == {
'average': self.addon.average_rating,
@ -215,17 +283,19 @@ class TestAddonIndexer(TestCase):
# Make the version a webextension and add a bunch of things to it to
# test different scenarios.
version.all_files[0].update(is_webextension=True)
file_factory(
version=version, platform=PLATFORM_MAC.id, is_webextension=True)
file_factory(version=version, platform=PLATFORM_MAC.id, is_webextension=True)
del version.all_files
version.license = License.objects.create(
name=u'My licensé',
url='http://example.com/',
builtin=0)
[WebextPermission.objects.create(
file=file_, permissions=permissions,
optional_permissions=optional_permissions
) for file_ in version.all_files]
name=u'My licensé', url='http://example.com/', builtin=0
)
[
WebextPermission.objects.create(
file=file_,
permissions=permissions,
optional_permissions=optional_permissions,
)
for file_ in version.all_files
]
version.save()
# Now we can run the extraction and start testing.
@ -247,12 +317,15 @@ class TestAddonIndexer(TestCase):
'builtin': 0,
'id': version.license.pk,
'name_translations': [{'lang': u'en-US', 'string': u'My licensé'}],
'url': u'http://example.com/'
'url': u'http://example.com/',
}
assert extracted['current_version']['release_notes_translations'] == [
{'lang': 'en-US', 'string': u'Fix for an important bug'},
{'lang': 'fr', 'string': u"Quelque chose en fran\xe7ais."
u"\n\nQuelque chose d'autre."},
{
'lang': 'fr',
'string': u"Quelque chose en fran\xe7ais."
u"\n\nQuelque chose d'autre.",
},
]
assert extracted['current_version']['reviewed'] == version.reviewed
assert extracted['current_version']['version'] == version.version
@ -263,28 +336,23 @@ class TestAddonIndexer(TestCase):
assert extracted_file['filename'] == file_.filename
assert extracted_file['hash'] == file_.hash
assert extracted_file['is_webextension'] == file_.is_webextension
assert extracted_file['is_restart_required'] == (
file_.is_restart_required)
assert extracted_file['is_restart_required'] == (file_.is_restart_required)
assert extracted_file['is_mozilla_signed_extension'] == (
file_.is_mozilla_signed_extension)
file_.is_mozilla_signed_extension
)
assert extracted_file['platform'] == file_.platform
assert extracted_file['size'] == file_.size
assert extracted_file['status'] == file_.status
assert (
extracted_file['permissions'] ==
permissions)
assert (
extracted_file['optional_permissions'] ==
optional_permissions)
assert extracted_file['permissions'] == permissions
assert extracted_file['optional_permissions'] == optional_permissions
assert set(extracted['platforms']) == set([PLATFORM_MAC.id,
PLATFORM_ALL.id])
assert set(extracted['platforms']) == set([PLATFORM_MAC.id, PLATFORM_ALL.id])
def test_version_compatibility_with_strict_compatibility_enabled(self):
version = self.addon.current_version
file_factory(
version=version, platform=PLATFORM_MAC.id,
strict_compatibility=True)
version=version, platform=PLATFORM_MAC.id, strict_compatibility=True
)
extracted = self._extract()
assert extracted['current_version']['compatible_apps'] == {
@ -320,24 +388,17 @@ class TestAddonIndexer(TestCase):
assert extracted['description_translations'] == [
{'lang': 'en-US', 'string': translations_description['en-US']},
{'lang': 'es', 'string': translations_description['es']},
{'lang': 'it', 'string': '&lt;script&gt;alert(42)&lt;/script&gt;'}
{'lang': 'it', 'string': '&lt;script&gt;alert(42)&lt;/script&gt;'},
]
assert extracted['name_l10n_en-us'] == translations_name['en-US']
assert extracted['name_l10n_en-gb'] == ''
assert extracted['name_l10n_es'] == translations_name['es']
assert extracted['name_l10n_it'] == ''
assert (
extracted['description_l10n_en-us'] ==
translations_description['en-US']
)
assert (
extracted['description_l10n_es'] ==
translations_description['es']
)
assert extracted['description_l10n_en-us'] == translations_description['en-US']
assert extracted['description_l10n_es'] == translations_description['es']
assert extracted['description_l10n_fr'] == ''
assert (
extracted['description_l10n_it'] ==
'&lt;script&gt;alert(42)&lt;/script&gt;'
extracted['description_l10n_it'] == '&lt;script&gt;alert(42)&lt;/script&gt;'
)
assert extracted['summary_l10n_en-us'] == ''
# The non-l10n fields are fallbacks in the addon's default locale, they
@ -359,8 +420,7 @@ class TestAddonIndexer(TestCase):
self.addon = Addon.objects.create(**kwargs)
self.addon.name = {'es': 'Banana Bonkers espanole'}
self.addon.description = {
'es': 'Deje que su navegador coma sus plátanos'}
self.addon.description = {'es': 'Deje que su navegador coma sus plátanos'}
self.addon.summary = {'es': 'resumen banana'}
self.addon.save()
@ -372,28 +432,26 @@ class TestAddonIndexer(TestCase):
]
assert extracted['description_translations'] == [
{'lang': 'en-GB', 'string': 'Let your browser eat your bananas'},
{
'lang': 'es',
'string': 'Deje que su navegador coma sus plátanos'
},
{'lang': 'es', 'string': 'Deje que su navegador coma sus plátanos'},
]
assert extracted['name_l10n_en-gb'] == 'Banana Bonkers'
assert extracted['name_l10n_en-us'] == ''
assert extracted['name_l10n_es'] == 'Banana Bonkers espanole'
assert (
extracted['description_l10n_en-gb'] ==
'Let your browser eat your bananas'
extracted['description_l10n_en-gb'] == 'Let your browser eat your bananas'
)
assert (
extracted['description_l10n_es'] ==
'Deje que su navegador coma sus plátanos'
extracted['description_l10n_es']
== 'Deje que su navegador coma sus plátanos'
)
def test_extract_previews(self):
second_preview = Preview.objects.create(
addon=self.addon, position=2,
addon=self.addon,
position=2,
caption={'en-US': u'My câption', 'fr': u'Mön tîtré'},
sizes={'thumbnail': [199, 99], 'image': [567, 780]})
sizes={'thumbnail': [199, 99], 'image': [567, 780]},
)
first_preview = Preview.objects.create(addon=self.addon, position=1)
first_preview.reload()
second_preview.reload()
@ -408,9 +466,13 @@ class TestAddonIndexer(TestCase):
assert extracted['previews'][1]['modified'] == second_preview.modified
assert extracted['previews'][1]['caption_translations'] == [
{'lang': 'en-US', 'string': u'My câption'},
{'lang': 'fr', 'string': u'Mön tîtré'}]
assert extracted['previews'][1]['sizes'] == second_preview.sizes == {
'thumbnail': [199, 99], 'image': [567, 780]}
{'lang': 'fr', 'string': u'Mön tîtré'},
]
assert (
extracted['previews'][1]['sizes']
== second_preview.sizes
== {'thumbnail': [199, 99], 'image': [567, 780]}
)
# Only raw translations dict should exist, since we don't need the
# to search against preview captions.
@ -422,23 +484,33 @@ class TestAddonIndexer(TestCase):
current_preview = VersionPreview.objects.create(
version=self.addon.current_version,
colors=[{'h': 1, 's': 2, 'l': 3, 'ratio': 0.9}],
sizes={'thumbnail': [56, 78], 'image': [91, 234]}, position=1)
sizes={'thumbnail': [56, 78], 'image': [91, 234]},
position=1,
)
second_preview = VersionPreview.objects.create(
version=self.addon.current_version,
sizes={'thumbnail': [12, 34], 'image': [56, 78]}, position=2)
sizes={'thumbnail': [12, 34], 'image': [56, 78]},
position=2,
)
extracted = self._extract()
assert extracted['previews']
assert len(extracted['previews']) == 2
assert 'caption_translations' not in extracted['previews'][0]
assert extracted['previews'][0]['id'] == current_preview.pk
assert extracted['previews'][0]['modified'] == current_preview.modified
assert extracted['previews'][0]['sizes'] == current_preview.sizes == {
'thumbnail': [56, 78], 'image': [91, 234]}
assert (
extracted['previews'][0]['sizes']
== current_preview.sizes
== {'thumbnail': [56, 78], 'image': [91, 234]}
)
assert 'caption_translations' not in extracted['previews'][1]
assert extracted['previews'][1]['id'] == second_preview.pk
assert extracted['previews'][1]['modified'] == second_preview.modified
assert extracted['previews'][1]['sizes'] == second_preview.sizes == {
'thumbnail': [12, 34], 'image': [56, 78]}
assert (
extracted['previews'][1]['sizes']
== second_preview.sizes
== {'thumbnail': [12, 34], 'image': [56, 78]}
)
# Make sure we extract colors from the first preview.
assert extracted['colors'] == [{'h': 1, 's': 2, 'l': 3, 'ratio': 0.9}]
@ -464,26 +536,30 @@ class TestAddonIndexer(TestCase):
assert extracted['promoted']
assert extracted['promoted']['group_id'] == RECOMMENDED.id
assert extracted['promoted']['approved_for_apps'] == [
amo.FIREFOX.id, amo.ANDROID.id]
amo.FIREFOX.id,
amo.ANDROID.id,
]
assert extracted['is_recommended'] is True
# Specific application.
self.addon.promotedaddon.update(application_id=amo.FIREFOX.id)
extracted = self._extract()
assert extracted['promoted']['approved_for_apps'] == [
amo.FIREFOX.id]
assert extracted['promoted']['approved_for_apps'] == [amo.FIREFOX.id]
assert extracted['is_recommended'] is True
# Promoted theme.
self.addon = addon_factory(type=amo.ADDON_STATICTHEME)
featured_collection, _ = Collection.objects.get_or_create(
id=settings.COLLECTION_FEATURED_THEMES_ID)
id=settings.COLLECTION_FEATURED_THEMES_ID
)
featured_collection.add_addon(self.addon)
extracted = self._extract()
assert extracted['promoted']
assert extracted['promoted']['group_id'] == RECOMMENDED.id
assert extracted['promoted']['approved_for_apps'] == [
amo.FIREFOX.id, amo.ANDROID.id]
amo.FIREFOX.id,
amo.ANDROID.id,
]
assert extracted['is_recommended'] is True
@mock.patch('olympia.addons.indexers.create_chunked_tasks_signatures')
@ -495,12 +571,15 @@ class TestAddonIndexer(TestCase):
addon_factory(status=amo.STATUS_DELETED).pk,
addon_factory(
status=amo.STATUS_NULL,
version_kw={'channel': amo.RELEASE_CHANNEL_UNLISTED}).pk,
version_kw={'channel': amo.RELEASE_CHANNEL_UNLISTED},
).pk,
]
rval = AddonIndexer.reindex_tasks_group('addons')
assert create_chunked_tasks_signatures_mock.call_count == 1
assert create_chunked_tasks_signatures_mock.call_args[0] == (
index_addons, expected_ids, 150
index_addons,
expected_ids,
150,
)
assert rval == create_chunked_tasks_signatures_mock.return_value
@ -519,8 +598,9 @@ class TestAddonIndexerWithES(ESTestCase):
indexer = AddonIndexer()
doc_name = indexer.get_doctype_name()
real_index_name = self.get_index_name(SearchMixin.ES_ALIAS_KEY)
mappings = self.es.indices.get_mapping(
indexer.get_index_alias())[real_index_name]['mappings']
mappings = self.es.indices.get_mapping(indexer.get_index_alias())[
real_index_name
]['mappings']
actual_properties = mappings[doc_name]['properties']
indexer_properties = indexer.get_mapping()[doc_name]['properties']

825
src/olympia/addons/tests/test_models.py
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

578
src/olympia/addons/tests/test_serializers.py
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

75
src/olympia/addons/tests/test_tasks.py
Просмотреть файл

@ -7,10 +7,12 @@ from django.conf import settings
from waffle.testutils import override_switch
from olympia import amo
from olympia.addons.tasks import (recreate_theme_previews,
update_addon_average_daily_users,
update_addon_hotness,
update_addon_weekly_downloads)
from olympia.addons.tasks import (
recreate_theme_previews,
update_addon_average_daily_users,
update_addon_hotness,
update_addon_weekly_downloads,
)
from olympia.amo.storage_utils import copy_stored_file
from olympia.amo.tests import addon_factory
from olympia.versions.models import VersionPreview
@ -19,36 +21,42 @@ from olympia.versions.models import VersionPreview
@pytest.mark.django_db
def test_recreate_theme_previews():
xpi_path = os.path.join(
settings.ROOT,
'src/olympia/devhub/tests/addons/mozilla_static_theme.zip')
settings.ROOT, 'src/olympia/devhub/tests/addons/mozilla_static_theme.zip'
)
addon_without_previews = addon_factory(type=amo.ADDON_STATICTHEME)
copy_stored_file(
xpi_path,
addon_without_previews.current_version.all_files[0].file_path)
xpi_path, addon_without_previews.current_version.all_files[0].file_path
)
addon_with_previews = addon_factory(type=amo.ADDON_STATICTHEME)
copy_stored_file(
xpi_path,
addon_with_previews.current_version.all_files[0].file_path)
xpi_path, addon_with_previews.current_version.all_files[0].file_path
)
VersionPreview.objects.create(
version=addon_with_previews.current_version,
sizes={'image': [123, 456], 'thumbnail': [34, 45]})
sizes={'image': [123, 456], 'thumbnail': [34, 45]},
)
assert addon_without_previews.current_previews.count() == 0
assert addon_with_previews.current_previews.count() == 1
recreate_theme_previews(
[addon_without_previews.id, addon_with_previews.id])
recreate_theme_previews([addon_without_previews.id, addon_with_previews.id])
assert addon_without_previews.reload().current_previews.count() == 3
assert addon_with_previews.reload().current_previews.count() == 3
sizes = addon_without_previews.current_previews.values_list(
'sizes', flat=True)
sizes = addon_without_previews.current_previews.values_list('sizes', flat=True)
assert list(sizes) == [
{'image': list(amo.THEME_PREVIEW_SIZES['header']['full']),
'thumbnail': list(amo.THEME_PREVIEW_SIZES['header']['thumbnail'])},
{'image': list(amo.THEME_PREVIEW_SIZES['list']['full']),
'thumbnail': list(amo.THEME_PREVIEW_SIZES['list']['thumbnail'])},
{'image': list(amo.THEME_PREVIEW_SIZES['single']['full']),
'thumbnail': list(amo.THEME_PREVIEW_SIZES['single']['thumbnail'])}]
{
'image': list(amo.THEME_PREVIEW_SIZES['header']['full']),
'thumbnail': list(amo.THEME_PREVIEW_SIZES['header']['thumbnail']),
},
{
'image': list(amo.THEME_PREVIEW_SIZES['list']['full']),
'thumbnail': list(amo.THEME_PREVIEW_SIZES['list']['thumbnail']),
},
{
'image': list(amo.THEME_PREVIEW_SIZES['single']['full']),
'thumbnail': list(amo.THEME_PREVIEW_SIZES['single']['thumbnail']),
},
]
@pytest.mark.django_db
@ -58,13 +66,16 @@ def test_create_missing_theme_previews(parse_addon_mock):
theme = addon_factory(type=amo.ADDON_STATICTHEME)
preview = VersionPreview.objects.create(
version=theme.current_version,
sizes={'image': [123, 456], 'thumbnail': [34, 45]})
sizes={'image': [123, 456], 'thumbnail': [34, 45]},
)
VersionPreview.objects.create(
version=theme.current_version,
sizes={'image': [123, 456], 'thumbnail': [34, 45]})
sizes={'image': [123, 456], 'thumbnail': [34, 45]},
)
VersionPreview.objects.create(
version=theme.current_version,
sizes={'image': [123, 456], 'thumbnail': [34, 45]})
sizes={'image': [123, 456], 'thumbnail': [34, 45]},
)
# addon has 3 complete previews already so skip when only_missing=True
with mock.patch('olympia.addons.tasks.generate_static_theme_preview') as p:
@ -117,23 +128,15 @@ def test_update_deleted_addon_average_daily_users():
@pytest.mark.django_db
def test_update_addon_hotness():
addon1 = addon_factory(hotness=0, status=amo.STATUS_APPROVED)
addon2 = addon_factory(hotness=123,
status=amo.STATUS_APPROVED)
addon3 = addon_factory(hotness=123,
status=amo.STATUS_AWAITING_REVIEW)
addon2 = addon_factory(hotness=123, status=amo.STATUS_APPROVED)
addon3 = addon_factory(hotness=123, status=amo.STATUS_AWAITING_REVIEW)
averages = {
addon1.guid: {
'avg_this_week': 213467,
'avg_three_weeks_before': 123467
},
addon1.guid: {'avg_this_week': 213467, 'avg_three_weeks_before': 123467},
addon2.guid: {
'avg_this_week': 1,
'avg_three_weeks_before': 1,
},
addon3.guid: {
'avg_this_week': 213467,
'avg_three_weeks_before': 123467
},
addon3.guid: {'avg_this_week': 213467, 'avg_three_weeks_before': 123467},
}
update_addon_hotness(averages=averages.items())

158
src/olympia/addons/tests/test_update.py
Просмотреть файл

@ -20,7 +20,6 @@ from olympia.versions.models import ApplicationsVersions, Version
class VersionCheckMixin(object):
def get_update_instance(self, data):
instance = update.Update(data)
instance.cursor = connection.cursor()
@ -142,8 +141,10 @@ class TestLookup(VersionCheckMixin, TestCase):
assert instance.is_valid()
instance.data['version_int'] = args[1]
instance.get_update()
return (instance.data['row'].get('version_id'),
instance.data['row'].get('file_id'))
return (
instance.data['row'].get('version_id'),
instance.data['row'].get('file_id'),
)
def change_status(self, version, status):
version = Version.objects.get(pk=version)
@ -161,7 +162,8 @@ class TestLookup(VersionCheckMixin, TestCase):
add-on is returned.
"""
version, file = self.get_update_instance(
'', '3000000001100', self.app, self.platform)
'', '3000000001100', self.app, self.platform
)
assert version == self.version_1_0_2
def test_new_client(self):
@ -170,7 +172,8 @@ class TestLookup(VersionCheckMixin, TestCase):
add-on is returned.
"""
version, file = self.get_update_instance(
'', self.version_int, self.app, self.platform)
'', self.version_int, self.app, self.platform
)
assert version == self.version_1_2_2
def test_min_client(self):
@ -185,7 +188,8 @@ class TestLookup(VersionCheckMixin, TestCase):
appversion.save()
version, file = self.get_update_instance(
'', '3070000005000', self.app, self.platform) # 3.7a5pre
'', '3070000005000', self.app, self.platform
) # 3.7a5pre
assert version == self.version_1_1_3
def test_new_client_ordering(self):
@ -206,7 +210,8 @@ class TestLookup(VersionCheckMixin, TestCase):
application_version.save()
version, file = self.get_update_instance(
'', self.version_int, self.app, self.platform)
'', self.version_int, self.app, self.platform
)
assert version == self.version_1_2_2
def test_public(self):
@ -217,7 +222,8 @@ class TestLookup(VersionCheckMixin, TestCase):
self.addon.reload()
assert self.addon.status == amo.STATUS_APPROVED
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, self.platform)
'1.2', self.version_int, self.app, self.platform
)
assert version == self.version_1_2_1
def test_no_unlisted(self):
@ -225,11 +231,13 @@ class TestLookup(VersionCheckMixin, TestCase):
Unlisted versions are always ignored, never served as updates.
"""
Version.objects.get(pk=self.version_1_2_2).update(
channel=amo.RELEASE_CHANNEL_UNLISTED)
channel=amo.RELEASE_CHANNEL_UNLISTED
)
self.addon.reload()
assert self.addon.status == amo.STATUS_APPROVED
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, self.platform)
'1.2', self.version_int, self.app, self.platform
)
assert version == self.version_1_2_1
def test_can_downgrade(self):
@ -241,7 +249,8 @@ class TestLookup(VersionCheckMixin, TestCase):
for v in Version.objects.filter(pk__gte=self.version_1_2_1):
v.delete()
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, self.platform)
'1.2', self.version_int, self.app, self.platform
)
assert version == self.version_1_1_3
@ -257,7 +266,8 @@ class TestLookup(VersionCheckMixin, TestCase):
self.change_version(self.version_1_2_0, '1.2beta')
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, self.platform)
'1.2', self.version_int, self.app, self.platform
)
assert version == self.version_1_2_1
@ -272,7 +282,8 @@ class TestLookup(VersionCheckMixin, TestCase):
Version.objects.get(pk=self.version_1_2_0).files.all().delete()
version, file = self.get_update_instance(
'1.2beta', self.version_int, self.app, self.platform)
'1.2beta', self.version_int, self.app, self.platform
)
dest = Version.objects.get(pk=self.version_1_2_2)
assert dest.addon.status == amo.STATUS_APPROVED
assert dest.files.all()[0].status == amo.STATUS_APPROVED
@ -286,7 +297,8 @@ class TestLookup(VersionCheckMixin, TestCase):
self.change_status(self.version_1_2_2, amo.STATUS_NULL)
self.addon.update(status=amo.STATUS_NULL)
version, file = self.get_update_instance(
'1.2.1', self.version_int, self.app, self.platform)
'1.2.1', self.version_int, self.app, self.platform
)
assert version == self.version_1_2_1
def test_platform_does_not_exist(self):
@ -297,7 +309,8 @@ class TestLookup(VersionCheckMixin, TestCase):
file.save()
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, self.platform)
'1.2', self.version_int, self.app, self.platform
)
assert version == self.version_1_2_1
def test_platform_exists(self):
@ -308,7 +321,8 @@ class TestLookup(VersionCheckMixin, TestCase):
file.save()
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, amo.PLATFORM_LINUX)
'1.2', self.version_int, self.app, amo.PLATFORM_LINUX
)
assert version == self.version_1_2_2
def test_file_for_platform(self):
@ -318,17 +332,23 @@ class TestLookup(VersionCheckMixin, TestCase):
file_one.platform = amo.PLATFORM_LINUX.id
file_one.save()
file_two = File(version=version, filename='foo', hash='bar',
platform=amo.PLATFORM_WIN.id,
status=amo.STATUS_APPROVED)
file_two = File(
version=version,
filename='foo',
hash='bar',
platform=amo.PLATFORM_WIN.id,
status=amo.STATUS_APPROVED,
)
file_two.save()
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, amo.PLATFORM_LINUX)
'1.2', self.version_int, self.app, amo.PLATFORM_LINUX
)
assert version == self.version_1_2_2
assert file == file_one.pk
version, file = self.get_update_instance(
'1.2', self.version_int, self.app, amo.PLATFORM_WIN)
'1.2', self.version_int, self.app, amo.PLATFORM_WIN
)
assert version == self.version_1_2_2
assert file == file_two.pk
@ -337,6 +357,7 @@ class TestDefaultToCompat(VersionCheckMixin, TestCase):
"""
Test default to compatible with all the various combinations of input.
"""
fixtures = ['addons/default-to-compat']
def setUp(self):
@ -356,7 +377,9 @@ class TestDefaultToCompat(VersionCheckMixin, TestCase):
self.ver_1_3 = 1268884
self.expected = {
'3.0-strict': None, '3.0-normal': None, '3.0-ignore': None,
'3.0-strict': None,
'3.0-normal': None,
'3.0-ignore': None,
'4.0-strict': self.ver_1_0,
'4.0-normal': self.ver_1_0,
'4.0-ignore': self.ver_1_0,
@ -380,13 +403,15 @@ class TestDefaultToCompat(VersionCheckMixin, TestCase):
file.update(**kw)
def get_update_instance(self, **kw):
instance = super(TestDefaultToCompat, self).get_update_instance({
'reqVersion': 1,
'id': self.addon.guid,
'version': kw.get('item_version', '1.0'),
'appID': self.app.guid,
'appVersion': kw.get('app_version', '3.0'),
})
instance = super(TestDefaultToCompat, self).get_update_instance(
{
'reqVersion': 1,
'id': self.addon.guid,
'version': kw.get('item_version', '1.0'),
'appID': self.app.guid,
'appVersion': kw.get('app_version', '3.0'),
}
)
assert instance.is_valid()
instance.compat_mode = kw.get('compat_mode', 'strict')
instance.get_update()
@ -403,9 +428,8 @@ class TestDefaultToCompat(VersionCheckMixin, TestCase):
for version in versions:
for mode in modes:
assert (
self.get_update_instance(
app_version=version, compat_mode=mode) ==
expected['-'.join([version, mode])]
self.get_update_instance(app_version=version, compat_mode=mode)
== expected['-'.join([version, mode])]
)
def test_baseline(self):
@ -415,17 +439,21 @@ class TestDefaultToCompat(VersionCheckMixin, TestCase):
def test_binary_components(self):
# Tests add-on with binary_components flag.
self.update_files(binary_components=True)
self.expected.update({
'8.0-normal': None,
})
self.expected.update(
{
'8.0-normal': None,
}
)
self.check(self.expected)
def test_strict_opt_in(self):
# Tests add-on with opt-in strict compatibility
self.update_files(strict_compatibility=True)
self.expected.update({
'8.0-normal': None,
})
self.expected.update(
{
'8.0-normal': None,
}
)
self.check(self.expected)
def test_min_max_version(self):
@ -434,17 +462,19 @@ class TestDefaultToCompat(VersionCheckMixin, TestCase):
av.min_id = 233 # Firefox 3.0.
av.max_id = 268 # Firefox 3.5.
av.save()
self.expected.update({
'3.0-strict': self.ver_1_3,
'3.0-ignore': self.ver_1_3,
'4.0-ignore': self.ver_1_3,
'5.0-ignore': self.ver_1_3,
'6.0-strict': self.ver_1_2,
'6.0-normal': self.ver_1_2,
'7.0-strict': self.ver_1_2,
'7.0-normal': self.ver_1_2,
'8.0-normal': self.ver_1_2,
})
self.expected.update(
{
'3.0-strict': self.ver_1_3,
'3.0-ignore': self.ver_1_3,
'4.0-ignore': self.ver_1_3,
'5.0-ignore': self.ver_1_3,
'6.0-strict': self.ver_1_2,
'6.0-normal': self.ver_1_2,
'7.0-strict': self.ver_1_2,
'7.0-normal': self.ver_1_2,
'8.0-normal': self.ver_1_2,
}
)
self.check(self.expected)
@ -483,9 +513,7 @@ class TestResponse(VersionCheckMixin, TestCase):
data['appOS'] = self.mac.api_name
instance = self.get_update_instance(data)
assert (
json.loads(instance.get_output()) ==
instance.get_no_updates_output())
assert json.loads(instance.get_output()) == instance.get_no_updates_output()
def test_different_platform(self):
file = File.objects.get(pk=67442)
@ -554,8 +582,7 @@ class TestResponse(VersionCheckMixin, TestCase):
# way lies pain with broken tests later.
instance = self.get_update_instance(self.data)
headers = dict(instance.get_headers(1))
last_modified = datetime(
*utils.parsedate_tz(headers['Last-Modified'])[:7])
last_modified = datetime(*utils.parsedate_tz(headers['Last-Modified'])[:7])
expires = datetime(*utils.parsedate_tz(headers['Expires'])[:7])
assert (expires - last_modified).seconds == 3600
@ -565,7 +592,8 @@ class TestResponse(VersionCheckMixin, TestCase):
'http://testserver/user-media/addons/3615/'
'delicious_bookmarks-2.1.072-fx.xpi?'
'filehash=sha256%3A3808b13ef8341378b9c8305ca648200954ee7dcd8dc'
'e09fef55f2673458bc31f')
'e09fef55f2673458bc31f'
)
def test_url(self):
instance = self.get_update_instance(self.data)
@ -611,17 +639,13 @@ class TestResponse(VersionCheckMixin, TestCase):
def test_no_updates_at_all(self):
self.addon_one.versions.all().delete()
instance = self.get_update_instance(self.data)
assert (
json.loads(instance.get_output()) ==
instance.get_no_updates_output())
assert json.loads(instance.get_output()) == instance.get_no_updates_output()
def test_no_updates_my_fx(self):
data = self.data.copy()
data['appVersion'] = '5.0.1'
instance = self.get_update_instance(data)
assert (
json.loads(instance.get_output()) ==
instance.get_no_updates_output())
assert json.loads(instance.get_output()) == instance.get_no_updates_output()
def test_application(self):
# Basic test making sure application() is returning the output of
@ -630,14 +654,10 @@ class TestResponse(VersionCheckMixin, TestCase):
# settings_test.py, we wouldn't see results because the data wouldn't
# exist with the cursor the update service is using, which is different
# from the one used by django tests.
environ = {
'QUERY_STRING': ''
}
environ = {'QUERY_STRING': ''}
self.start_response_call_count = 0
expected_headers = [
('FakeHeader', 'FakeHeaderValue')
]
expected_headers = [('FakeHeader', 'FakeHeaderValue')]
expected_output = b'{"fake": "output"}'
@ -660,6 +680,7 @@ class TestResponse(VersionCheckMixin, TestCase):
@mock.patch('services.update.Update')
def test_exception_handling(self, UpdateMock, log_mock):
"""Test ensuring exceptions are raised and logged properly."""
class CustomException(Exception):
pass
@ -676,8 +697,7 @@ class TestResponse(VersionCheckMixin, TestCase):
# The log should be present.
assert log_mock.exception.call_count == 1
log_mock.exception.assert_called_with(
update_instance.get_output.side_effect)
log_mock.exception.assert_called_with(update_instance.get_output.side_effect)
# This test needs to be a TransactionTestCase because we want to test the

93
src/olympia/addons/tests/test_utils_.py
Просмотреть файл

@ -6,38 +6,46 @@ from django.conf import settings
from django.forms import ValidationError
from olympia.addons.utils import (
get_addon_recommendations, get_addon_recommendations_invalid,
get_addon_recommendations,
get_addon_recommendations_invalid,
is_outcome_recommended,
TAAR_LITE_FALLBACK_REASON_EMPTY, TAAR_LITE_FALLBACK_REASON_TIMEOUT,
TAAR_LITE_FALLBACKS, TAAR_LITE_OUTCOME_CURATED,
TAAR_LITE_OUTCOME_REAL_FAIL, TAAR_LITE_OUTCOME_REAL_SUCCESS,
TAAR_LITE_FALLBACK_REASON_EMPTY,
TAAR_LITE_FALLBACK_REASON_TIMEOUT,
TAAR_LITE_FALLBACKS,
TAAR_LITE_OUTCOME_CURATED,
TAAR_LITE_OUTCOME_REAL_FAIL,
TAAR_LITE_OUTCOME_REAL_SUCCESS,
TAAR_LITE_FALLBACK_REASON_INVALID,
verify_mozilla_trademark)
verify_mozilla_trademark,
)
from olympia.amo.tests import TestCase, addon_factory, user_factory
from olympia.users.models import Group, GroupUser
@pytest.mark.django_db
@pytest.mark.parametrize('name, allowed, give_permission', (
('Fancy new Add-on', True, False),
# We allow the 'for ...' postfix to be used
('Fancy new Add-on for Firefox', True, False),
('Fancy new Add-on for Mozilla', True, False),
# But only the postfix
('Fancy new Add-on for Firefox Browser', False, False),
('For Firefox fancy new add-on', False, False),
# But users with the TRADEMARK_BYPASS permission are allowed
('Firefox makes everything better', False, False),
('Firefox makes everything better', True, True),
('Mozilla makes everything better', True, True),
# A few more test-cases...
('Firefox add-on for Firefox', False, False),
('Firefox add-on for Firefox', True, True),
('Foobarfor Firefox', False, False),
('Better Privacy for Firefox!', True, False),
('Firefox awesome for Mozilla', False, False),
('Firefox awesome for Mozilla', True, True),
))
@pytest.mark.parametrize(
'name, allowed, give_permission',
(
('Fancy new Add-on', True, False),
# We allow the 'for ...' postfix to be used
('Fancy new Add-on for Firefox', True, False),
('Fancy new Add-on for Mozilla', True, False),
# But only the postfix
('Fancy new Add-on for Firefox Browser', False, False),
('For Firefox fancy new add-on', False, False),
# But users with the TRADEMARK_BYPASS permission are allowed
('Firefox makes everything better', False, False),
('Firefox makes everything better', True, True),
('Mozilla makes everything better', True, True),
# A few more test-cases...
('Firefox add-on for Firefox', False, False),
('Firefox add-on for Firefox', True, True),
('Foobarfor Firefox', False, False),
('Better Privacy for Firefox!', True, False),
('Firefox awesome for Mozilla', False, False),
('Firefox awesome for Mozilla', True, True),
),
)
def test_verify_mozilla_trademark(name, allowed, give_permission):
user = user_factory()
if give_permission:
@ -48,8 +56,7 @@ def test_verify_mozilla_trademark(name, allowed, give_permission):
with pytest.raises(ValidationError) as exc:
verify_mozilla_trademark(name, user)
assert exc.value.message == (
'Add-on names cannot contain the Mozilla or Firefox '
'trademarks.'
'Add-on names cannot contain the Mozilla or Firefox ' 'trademarks.'
)
else:
verify_mozilla_trademark(name, user)
@ -57,8 +64,7 @@ def test_verify_mozilla_trademark(name, allowed, give_permission):
class TestGetAddonRecommendations(TestCase):
def setUp(self):
patcher = mock.patch(
'olympia.addons.utils.call_recommendation_server')
patcher = mock.patch('olympia.addons.utils.call_recommendation_server')
self.recommendation_server_mock = patcher.start()
self.addCleanup(patcher.stop)
self.a101 = addon_factory(id=101, guid='101@mozilla')
@ -67,43 +73,44 @@ class TestGetAddonRecommendations(TestCase):
addon_factory(id=104, guid='104@mozilla')
self.recommendation_guids = [
'101@mozilla', '102@mozilla', '103@mozilla', '104@mozilla'
'101@mozilla',
'102@mozilla',
'103@mozilla',
'104@mozilla',
]
self.recommendation_server_mock.return_value = (
self.recommendation_guids)
self.recommendation_server_mock.return_value = self.recommendation_guids
def test_recommended(self):
recommendations, outcome, reason = get_addon_recommendations(
'a@b', True)
recommendations, outcome, reason = get_addon_recommendations('a@b', True)
assert recommendations == self.recommendation_guids
assert outcome == TAAR_LITE_OUTCOME_REAL_SUCCESS
assert reason is None
self.recommendation_server_mock.assert_called_with(
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, 'a@b', {})
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, 'a@b', {}
)
def test_recommended_no_results(self):
self.recommendation_server_mock.return_value = []
recommendations, outcome, reason = get_addon_recommendations(
'a@b', True)
recommendations, outcome, reason = get_addon_recommendations('a@b', True)
assert recommendations == TAAR_LITE_FALLBACKS
assert outcome == TAAR_LITE_OUTCOME_REAL_FAIL
assert reason is TAAR_LITE_FALLBACK_REASON_EMPTY
self.recommendation_server_mock.assert_called_with(
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, 'a@b', {})
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, 'a@b', {}
)
def test_recommended_timeout(self):
self.recommendation_server_mock.return_value = None
recommendations, outcome, reason = get_addon_recommendations(
'a@b', True)
recommendations, outcome, reason = get_addon_recommendations('a@b', True)
assert recommendations == TAAR_LITE_FALLBACKS
assert outcome == TAAR_LITE_OUTCOME_REAL_FAIL
assert reason is TAAR_LITE_FALLBACK_REASON_TIMEOUT
self.recommendation_server_mock.assert_called_with(
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, 'a@b', {})
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, 'a@b', {}
)
def test_not_recommended(self):
recommendations, outcome, reason = get_addon_recommendations(
'a@b', False)
recommendations, outcome, reason = get_addon_recommendations('a@b', False)
assert not self.recommendation_server_mock.called
assert recommendations == TAAR_LITE_FALLBACKS
assert outcome == TAAR_LITE_OUTCOME_CURATED

1165
src/olympia/addons/tests/test_views.py
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

16
src/olympia/addons/urls.py
Просмотреть файл

@ -11,9 +11,7 @@ ADDON_ID = r"""(?P<addon_id>[^/<>"']+)"""
# These will all start with /addon/<addon_id>/
detail_patterns = [
re_path(r'^$', frontend_view, name='addons.detail'),
re_path(r'^license/(?P<version>[^/]+)?', frontend_view,
name='addons.license'),
re_path(r'^license/(?P<version>[^/]+)?', frontend_view, name='addons.license'),
re_path(r'^reviews/', include('olympia.ratings.urls')),
re_path(r'^statistics/', include(stats_patterns)),
re_path(r'^versions/', include('olympia.versions.urls')),
@ -22,11 +20,11 @@ detail_patterns = [
urlpatterns = [
# URLs for a single add-on.
re_path(r'^addon/%s/' % ADDON_ID, include(detail_patterns)),
re_path(r'^find-replacement/$', views.find_replacement_addon,
name='addons.find_replacement'),
re_path(
r'^find-replacement/$',
views.find_replacement_addon,
name='addons.find_replacement',
),
# frontend block view
re_path(r'^blocked-addon/%s/' % ADDON_ID, frontend_view,
name='blocklist.block'),
re_path(r'^blocked-addon/%s/' % ADDON_ID, frontend_view, name='blocklist.block'),
]

53
src/olympia/addons/utils.py
Просмотреть файл

@ -17,22 +17,26 @@ def generate_addon_guid():
def verify_mozilla_trademark(name, user, form=None):
skip_trademark_check = (
user and user.is_authenticated and action_allowed_user(
user, amo.permissions.TRADEMARK_BYPASS))
user
and user.is_authenticated
and action_allowed_user(user, amo.permissions.TRADEMARK_BYPASS)
)
def _check(name):
name = normalize_string(name, strip_punctuation=True).lower()
for symbol in amo.MOZILLA_TRADEMARK_SYMBOLS:
violates_trademark = (
name.count(symbol) > 1 or (
name.count(symbol) >= 1 and not
name.endswith(' for {}'.format(symbol))))
violates_trademark = name.count(symbol) > 1 or (
name.count(symbol) >= 1 and not name.endswith(' for {}'.format(symbol))
)
if violates_trademark:
raise forms.ValidationError(ugettext(
u'Add-on names cannot contain the Mozilla or '
u'Firefox trademarks.'))
raise forms.ValidationError(
ugettext(
u'Add-on names cannot contain the Mozilla or '
u'Firefox trademarks.'
)
)
if not skip_trademark_check:
if not isinstance(name, dict):
@ -45,7 +49,8 @@ def verify_mozilla_trademark(name, user, form=None):
if form is not None:
for message in exc.messages:
error_message = LocaleErrorMessage(
message=message, locale=locale)
message=message, locale=locale
)
form.add_error('name', error_message)
else:
raise
@ -54,9 +59,10 @@ def verify_mozilla_trademark(name, user, form=None):
TAAR_LITE_FALLBACKS = [
'enhancerforyoutube@maximerf.addons.mozilla.org', # /enhancer-for-youtube/
'{2e5ff8c8-32fe-46d0-9fc8-6b8986621f3c}', # /search_by_image/
'uBlock0@raymondhill.net', # /ublock-origin/
'newtaboverride@agenedia.com'] # /new-tab-override/
'{2e5ff8c8-32fe-46d0-9fc8-6b8986621f3c}', # /search_by_image/
'uBlock0@raymondhill.net', # /ublock-origin/
'newtaboverride@agenedia.com',
] # /new-tab-override/
TAAR_LITE_OUTCOME_REAL_SUCCESS = 'recommended'
TAAR_LITE_OUTCOME_REAL_FAIL = 'recommended_fallback'
@ -71,12 +77,17 @@ def get_addon_recommendations(guid_param, taar_enable):
fail_reason = None
if taar_enable:
guids = call_recommendation_server(
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, guid_param, {})
outcome = (TAAR_LITE_OUTCOME_REAL_SUCCESS if guids
else TAAR_LITE_OUTCOME_REAL_FAIL)
settings.TAAR_LITE_RECOMMENDATION_ENGINE_URL, guid_param, {}
)
outcome = (
TAAR_LITE_OUTCOME_REAL_SUCCESS if guids else TAAR_LITE_OUTCOME_REAL_FAIL
)
if not guids:
fail_reason = (TAAR_LITE_FALLBACK_REASON_EMPTY if guids == []
else TAAR_LITE_FALLBACK_REASON_TIMEOUT)
fail_reason = (
TAAR_LITE_FALLBACK_REASON_EMPTY
if guids == []
else TAAR_LITE_FALLBACK_REASON_TIMEOUT
)
else:
outcome = TAAR_LITE_OUTCOME_CURATED
if not guids:
@ -90,5 +101,7 @@ def is_outcome_recommended(outcome):
def get_addon_recommendations_invalid():
return (
TAAR_LITE_FALLBACKS, TAAR_LITE_OUTCOME_REAL_FAIL,
TAAR_LITE_FALLBACK_REASON_INVALID)
TAAR_LITE_FALLBACKS,
TAAR_LITE_OUTCOME_REAL_FAIL,
TAAR_LITE_FALLBACK_REASON_INVALID,
)

284
src/olympia/addons/views.py
Просмотреть файл

@ -26,15 +26,27 @@ from olympia.amo.urlresolvers import get_outgoing_url
from olympia.api.exceptions import UnavailableForLegalReasons
from olympia.api.pagination import ESPageNumberPagination
from olympia.api.permissions import (
AllowAddonAuthor, AllowReadOnlyIfPublic, AllowRelatedObjectPermissions,
AllowReviewer, AllowReviewerUnlisted, AnyOf, GroupPermission,
RegionalRestriction)
AllowAddonAuthor,
AllowReadOnlyIfPublic,
AllowRelatedObjectPermissions,
AllowReviewer,
AllowReviewerUnlisted,
AnyOf,
GroupPermission,
RegionalRestriction,
)
from olympia.constants.categories import CATEGORIES_BY_ID
from olympia.search.filters import (
AddonAppQueryParam, AddonAppVersionQueryParam, AddonAuthorQueryParam,
AddonTypeQueryParam, AutoCompleteSortFilter,
ReviewedContentFilter, SearchParameterFilter, SearchQueryFilter,
SortingFilter)
AddonAppQueryParam,
AddonAppVersionQueryParam,
AddonAuthorQueryParam,
AddonTypeQueryParam,
AutoCompleteSortFilter,
ReviewedContentFilter,
SearchParameterFilter,
SearchQueryFilter,
SortingFilter,
)
from olympia.translations.query import order_by_translation
from olympia.versions.models import Version
@ -43,12 +55,20 @@ from .indexers import AddonIndexer
from .models import Addon, ReplacementAddon
from .serializers import (
AddonEulaPolicySerializer,
AddonSerializer, AddonSerializerWithUnlistedData,
ESAddonAutoCompleteSerializer, ESAddonSerializer, LanguageToolsSerializer,
ReplacementAddonSerializer, StaticCategorySerializer, VersionSerializer)
AddonSerializer,
AddonSerializerWithUnlistedData,
ESAddonAutoCompleteSerializer,
ESAddonSerializer,
LanguageToolsSerializer,
ReplacementAddonSerializer,
StaticCategorySerializer,
VersionSerializer,
)
from .utils import (
get_addon_recommendations, get_addon_recommendations_invalid,
is_outcome_recommended)
get_addon_recommendations,
get_addon_recommendations_invalid,
is_outcome_recommended,
)
log = olympia.core.logger.getLogger('z.addons')
@ -80,8 +100,9 @@ class BaseFilter(object):
def options(self, request, key, default):
"""Get the (option, title) pair we want according to the request."""
if key in request.GET and (request.GET[key] in self.opts_dict or
request.GET[key] in self.extras_dict):
if key in request.GET and (
request.GET[key] in self.opts_dict or request.GET[key] in self.extras_dict
):
opt = request.GET[key]
else:
opt = default
@ -149,12 +170,16 @@ def find_replacement_addon(request):
class AddonViewSet(RetrieveModelMixin, GenericViewSet):
permission_classes = [
AnyOf(AllowReadOnlyIfPublic, AllowAddonAuthor,
AllowReviewer, AllowReviewerUnlisted),
AnyOf(
AllowReadOnlyIfPublic,
AllowAddonAuthor,
AllowReviewer,
AllowReviewerUnlisted,
),
]
georestriction_classes = [
RegionalRestriction |
GroupPermission(amo.permissions.ADDONS_EDIT)]
RegionalRestriction | GroupPermission(amo.permissions.ADDONS_EDIT)
]
serializer_class = AddonSerializer
serializer_class_with_unlisted_data = AddonSerializerWithUnlistedData
lookup_value_regex = '[^/]+' # Allow '.' for email-like guids.
@ -164,9 +189,9 @@ class AddonViewSet(RetrieveModelMixin, GenericViewSet):
# Special case: admins - and only admins - can see deleted add-ons.
# This is handled outside a permission class because that condition
# would pollute all other classes otherwise.
if (self.request.user.is_authenticated and
acl.action_allowed(self.request,
amo.permissions.ADDONS_VIEW_DELETED)):
if self.request.user.is_authenticated and acl.action_allowed(
self.request, amo.permissions.ADDONS_VIEW_DELETED
):
qs = Addon.unfiltered.all()
else:
# Permission classes disallow access to non-public/unlisted add-ons
@ -187,9 +212,11 @@ class AddonViewSet(RetrieveModelMixin, GenericViewSet):
# we are allowed to access unlisted data.
obj = getattr(self, 'instance')
request = self.request
if (acl.check_unlisted_addons_reviewer(request) or
(obj and request.user.is_authenticated and
obj.authors.filter(pk=request.user.pk).exists())):
if acl.check_unlisted_addons_reviewer(request) or (
obj
and request.user.is_authenticated
and obj.authors.filter(pk=request.user.pk).exists()
):
return self.serializer_class_with_unlisted_data
return self.serializer_class
@ -248,15 +275,17 @@ class AddonViewSet(RetrieveModelMixin, GenericViewSet):
def eula_policy(self, request, pk=None):
obj = self.get_object()
serializer = AddonEulaPolicySerializer(
obj, context=self.get_serializer_context())
obj, context=self.get_serializer_context()
)
return Response(serializer.data)
class AddonChildMixin(object):
"""Mixin containing method to retrieve the parent add-on object."""
def get_addon_object(self, permission_classes=None,
georestriction_classes=None, lookup='addon_pk'):
def get_addon_object(
self, permission_classes=None, georestriction_classes=None, lookup='addon_pk'
):
"""Return the parent Addon object using the URL parameter passed
to the view.
@ -281,12 +310,14 @@ class AddonChildMixin(object):
permission_classes=permission_classes,
georestriction_classes=georestriction_classes,
kwargs={'pk': self.kwargs[lookup]},
action='retrieve_from_related').get_object()
action='retrieve_from_related',
).get_object()
return self.addon_object
class AddonVersionViewSet(AddonChildMixin, RetrieveModelMixin,
ListModelMixin, GenericViewSet):
class AddonVersionViewSet(
AddonChildMixin, RetrieveModelMixin, ListModelMixin, GenericViewSet
):
# Permissions are always checked against the parent add-on in
# get_addon_object() using AddonViewSet.permission_classes so we don't need
# to set any here. Some extra permission classes are added dynamically
@ -301,33 +332,37 @@ class AddonVersionViewSet(AddonChildMixin, RetrieveModelMixin,
if requested == 'all_with_deleted':
# To see deleted versions, you need Addons:ViewDeleted.
self.permission_classes = [
GroupPermission(amo.permissions.ADDONS_VIEW_DELETED)]
GroupPermission(amo.permissions.ADDONS_VIEW_DELETED)
]
elif requested == 'all_with_unlisted':
# To see unlisted versions, you need to be add-on author or
# unlisted reviewer.
self.permission_classes = [AnyOf(
AllowReviewerUnlisted, AllowAddonAuthor)]
self.permission_classes = [
AnyOf(AllowReviewerUnlisted, AllowAddonAuthor)
]
elif requested == 'all_without_unlisted':
# To see all listed versions (not just public ones) you need to
# be add-on author or reviewer.
self.permission_classes = [AnyOf(
AllowReviewer, AllowReviewerUnlisted, AllowAddonAuthor)]
self.permission_classes = [
AnyOf(AllowReviewer, AllowReviewerUnlisted, AllowAddonAuthor)
]
# When listing, we can't use AllowRelatedObjectPermissions() with
# check_permissions(), because AllowAddonAuthor needs an author to
# do the actual permission check. To work around that, we call
# super + check_object_permission() ourselves, passing down the
# addon object directly.
return super(AddonVersionViewSet, self).check_object_permissions(
request, self.get_addon_object())
request, self.get_addon_object()
)
super(AddonVersionViewSet, self).check_permissions(request)
def check_object_permissions(self, request, obj):
# If the instance is marked as deleted and the client is not allowed to
# see deleted instances, we want to return a 404, behaving as if it
# does not exist.
if (obj.deleted and
not GroupPermission(amo.permissions.ADDONS_VIEW_DELETED).
has_object_permission(request, self, obj)):
if obj.deleted and not GroupPermission(
amo.permissions.ADDONS_VIEW_DELETED
).has_object_permission(request, self, obj):
raise http.Http404
if obj.channel == amo.RELEASE_CHANNEL_UNLISTED:
@ -335,13 +370,15 @@ class AddonVersionViewSet(AddonChildMixin, RetrieveModelMixin,
# authors..
self.permission_classes = [
AllowRelatedObjectPermissions(
'addon', [AnyOf(AllowReviewerUnlisted, AllowAddonAuthor)])
'addon', [AnyOf(AllowReviewerUnlisted, AllowAddonAuthor)]
)
]
elif not obj.is_public():
# If the instance is disabled, only allow reviewers and authors.
self.permission_classes = [
AllowRelatedObjectPermissions(
'addon', [AnyOf(AllowReviewer, AllowAddonAuthor)])
'addon', [AnyOf(AllowReviewer, AllowAddonAuthor)]
)
]
super(AddonVersionViewSet, self).check_object_permissions(request, obj)
@ -356,10 +393,12 @@ class AddonVersionViewSet(AddonChildMixin, RetrieveModelMixin,
if requested is not None:
if self.action != 'list':
raise serializers.ValidationError(
'The "filter" parameter is not valid in this context.')
'The "filter" parameter is not valid in this context.'
)
elif requested not in valid_filters:
raise serializers.ValidationError(
'Invalid "filter" parameter specified.')
'Invalid "filter" parameter specified.'
)
# When listing, by default we only want to return listed, approved
# versions, matching frontend needs - this can be overridden by the
# filter in use. When fetching a single instance however, we use the
@ -375,12 +414,11 @@ class AddonVersionViewSet(AddonChildMixin, RetrieveModelMixin,
elif requested == 'all_with_unlisted':
queryset = addon.versions.all()
elif requested == 'all_without_unlisted':
queryset = addon.versions.filter(
channel=amo.RELEASE_CHANNEL_LISTED)
queryset = addon.versions.filter(channel=amo.RELEASE_CHANNEL_LISTED)
else:
queryset = addon.versions.filter(
files__status=amo.STATUS_APPROVED,
channel=amo.RELEASE_CHANNEL_LISTED).distinct()
files__status=amo.STATUS_APPROVED, channel=amo.RELEASE_CHANNEL_LISTED
).distinct()
return queryset
@ -388,7 +426,9 @@ class AddonVersionViewSet(AddonChildMixin, RetrieveModelMixin,
class AddonSearchView(ListAPIView):
authentication_classes = []
filter_backends = [
ReviewedContentFilter, SearchQueryFilter, SearchParameterFilter,
ReviewedContentFilter,
SearchQueryFilter,
SearchParameterFilter,
SortingFilter,
]
pagination_class = ESPageNumberPagination
@ -396,12 +436,15 @@ class AddonSearchView(ListAPIView):
serializer_class = ESAddonSerializer
def get_queryset(self):
qset = Search(
using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name()).extra(
_source={'excludes': AddonIndexer.hidden_fields}).params(
search_type='dfs_query_then_fetch')
qset = (
Search(
using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name(),
)
.extra(_source={'excludes': AddonIndexer.hidden_fields})
.params(search_type='dfs_query_then_fetch')
)
return qset
@ -422,7 +465,9 @@ class AddonAutoCompleteSearchView(AddonSearchView):
pagination_class = None
serializer_class = ESAddonAutoCompleteSerializer
filter_backends = [
ReviewedContentFilter, SearchQueryFilter, SearchParameterFilter,
ReviewedContentFilter,
SearchQueryFilter,
SearchParameterFilter,
AutoCompleteSortFilter,
]
@ -443,12 +488,11 @@ class AddonAutoCompleteSearchView(AddonSearchView):
'type', # Needed to attach the Persona for icon_url (sadly).
)
qset = (
Search(
using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name())
.extra(_source={'includes': included_fields}))
qset = Search(
using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name(),
).extra(_source={'includes': included_fields})
return qset
@ -463,18 +507,19 @@ class AddonAutoCompleteSearchView(AddonSearchView):
class AddonFeaturedView(AddonSearchView):
"""Featuring is gone, so this view is a hollowed out shim that returns
recommended addons for api/v3."""
# We accept the 'page_size' parameter but we do not allow pagination for
# this endpoint since the order is random.
pagination_class = None
filter_backends = [
ReviewedContentFilter, SearchParameterFilter,
ReviewedContentFilter,
SearchParameterFilter,
]
def get(self, request, *args, **kwargs):
try:
page_size = int(
self.request.GET.get('page_size', api_settings.PAGE_SIZE))
page_size = int(self.request.GET.get('page_size', api_settings.PAGE_SIZE))
except ValueError:
raise exceptions.ParseError('Invalid page_size parameter')
@ -486,9 +531,8 @@ class AddonFeaturedView(AddonSearchView):
def filter_queryset(self, qs):
qs = super().filter_queryset(qs)
qs = qs.query(query.Bool(filter=[Q('term', is_recommended=True)]))
return (
qs.query('function_score', functions=[query.SF('random_score')])
.sort('_score')
return qs.query('function_score', functions=[query.SF('random_score')]).sort(
'_score'
)
@ -508,7 +552,8 @@ class StaticCategoryView(ListAPIView):
def finalize_response(self, request, response, *args, **kwargs):
response = super(StaticCategoryView, self).finalize_response(
request, response, *args, **kwargs)
request, response, *args, **kwargs
)
patch_cache_control(response, max_age=60 * 60 * 6)
return response
@ -522,8 +567,7 @@ class LanguageToolsView(ListAPIView):
@classmethod
def as_view(cls, **initkwargs):
"""The API is read-only so we can turn off atomic requests."""
return non_atomic_requests(
super(LanguageToolsView, cls).as_view(**initkwargs))
return non_atomic_requests(super(LanguageToolsView, cls).as_view(**initkwargs))
def get_query_params(self):
"""
@ -548,12 +592,8 @@ class LanguageToolsView(ListAPIView):
# appversion parameter is optional.
if AddonAppVersionQueryParam.query_param in self.request.GET:
try:
value = AddonAppVersionQueryParam(
self.request.GET).get_values()
appversions = {
'min': value[1],
'max': value[2]
}
value = AddonAppVersionQueryParam(self.request.GET).get_values()
appversions = {'min': value[1], 'max': value[2]}
except ValueError:
raise exceptions.ParseError('Invalid appversion parameter.')
else:
@ -565,12 +605,12 @@ class LanguageToolsView(ListAPIView):
# to filter by type if they want appversion filtering.
if AddonTypeQueryParam.query_param in self.request.GET or appversions:
try:
addon_types = tuple(
AddonTypeQueryParam(self.request.GET).get_values())
addon_types = tuple(AddonTypeQueryParam(self.request.GET).get_values())
except ValueError:
raise exceptions.ParseError(
'Invalid or missing type parameter while appversion '
'parameter is set.')
'parameter is set.'
)
else:
addon_types = (amo.ADDON_LPAPP, amo.ADDON_DICT)
@ -596,7 +636,8 @@ class LanguageToolsView(ListAPIView):
params = self.get_query_params()
if params['types'] == (amo.ADDON_LPAPP,) and params['appversions']:
qs = self.get_language_packs_queryset_with_appversions(
params['application'], params['appversions'])
params['application'], params['appversions']
)
else:
# appversions filtering only makes sense for language packs only,
# so it's ignored here.
@ -604,8 +645,8 @@ class LanguageToolsView(ListAPIView):
if params['authors']:
qs = qs.filter(
addonuser__user__username__in=params['authors'],
addonuser__listed=True).distinct()
addonuser__user__username__in=params['authors'], addonuser__listed=True
).distinct()
return qs
def get_queryset_base(self, application, addon_types):
@ -615,23 +656,25 @@ class LanguageToolsView(ListAPIView):
"""
return (
Addon.objects.public()
.filter(appsupport__app=application, type__in=addon_types,
target_locale__isnull=False)
.exclude(target_locale='')
.filter(
appsupport__app=application,
type__in=addon_types,
target_locale__isnull=False,
)
.exclude(target_locale='')
# Deactivate default transforms which fetch a ton of stuff we
# don't need here like authors, previews or current version.
# It would be nice to avoid translations entirely, because the
# translations transformer is going to fetch a lot of translations
# we don't need, but some language packs or dictionaries have
# custom names, so we can't use a generic one for them...
.only_translations()
.only_translations()
# Since we're fetching everything with no pagination, might as well
# not order it.
.order_by()
.order_by()
)
def get_language_packs_queryset_with_appversions(
self, application, appversions):
def get_language_packs_queryset_with_appversions(self, application, appversions):
"""
Return queryset to use specifically when requesting language packs
compatible with a given app + versions.
@ -647,18 +690,23 @@ class LanguageToolsView(ListAPIView):
# re-applying the default one that takes care of the files and compat
# info.
versions_qs = (
Version.objects
.latest_public_compatible_with(application, appversions)
.no_transforms().transform(Version.transformer))
Version.objects.latest_public_compatible_with(application, appversions)
.no_transforms()
.transform(Version.transformer)
)
return (
qs.prefetch_related(Prefetch('versions',
to_attr='compatible_versions',
queryset=versions_qs))
.filter(versions__apps__application=application,
versions__apps__min__version_int__lte=appversions['min'],
versions__apps__max__version_int__gte=appversions['max'],
versions__channel=amo.RELEASE_CHANNEL_LISTED,
versions__files__status=amo.STATUS_APPROVED)
qs.prefetch_related(
Prefetch(
'versions', to_attr='compatible_versions', queryset=versions_qs
)
)
.filter(
versions__apps__application=application,
versions__apps__min__version_int__lte=appversions['min'],
versions__apps__max__version_int__gte=appversions['max'],
versions__channel=amo.RELEASE_CHANNEL_LISTED,
versions__files__status=amo.STATUS_APPROVED,
)
.distinct()
)
@ -702,29 +750,37 @@ class AddonRecommendationView(AddonSearchView):
def get_paginated_response(self, data):
data = data[:4] # taar is only supposed to return 4 anyway.
return Response(OrderedDict([
('outcome', self.ab_outcome),
('fallback_reason', self.fallback_reason),
('page_size', 1),
('page_count', 1),
('count', len(data)),
('next', None),
('previous', None),
('results', data),
]))
return Response(
OrderedDict(
[
('outcome', self.ab_outcome),
('fallback_reason', self.fallback_reason),
('page_size', 1),
('page_count', 1),
('count', len(data)),
('next', None),
('previous', None),
('results', data),
]
)
)
def filter_queryset(self, qs):
qs = super(AddonRecommendationView, self).filter_queryset(qs)
guid_param = self.request.GET.get('guid')
taar_enable = self.request.GET.get('recommended', '').lower() == 'true'
guids, self.ab_outcome, self.fallback_reason = (
get_addon_recommendations(guid_param, taar_enable))
guids, self.ab_outcome, self.fallback_reason = get_addon_recommendations(
guid_param, taar_enable
)
results_qs = qs.query(query.Bool(must=[Q('terms', guid=guids)]))
results_qs.execute() # To cache the results.
if results_qs.count() != 4 and is_outcome_recommended(self.ab_outcome):
guids, self.ab_outcome, self.fallback_reason = (
get_addon_recommendations_invalid())
(
guids,
self.ab_outcome,
self.fallback_reason,
) = get_addon_recommendations_invalid()
return qs.query(query.Bool(must=[Q('terms', guid=guids)]))
return results_qs

14
src/olympia/amo/__init__.py
Просмотреть файл

@ -4,9 +4,17 @@ Miscellaneous helpers that make Django compatible with AMO.
from olympia.constants import permissions # noqa
from olympia.constants import promoted # noqa
from olympia.constants.activity import ( # noqa
LOG, LOG_BY_ID, LOG_ADMINS, LOG_REVIEWER_REVIEW_ACTION,
LOG_RATING_MODERATION, LOG_HIDE_DEVELOPER, LOG_KEEP, LOG_REVIEW_QUEUE,
LOG_REVIEW_QUEUE_DEVELOPER, LOG_REVIEW_EMAIL_USER)
LOG,
LOG_BY_ID,
LOG_ADMINS,
LOG_REVIEWER_REVIEW_ACTION,
LOG_RATING_MODERATION,
LOG_HIDE_DEVELOPER,
LOG_KEEP,
LOG_REVIEW_QUEUE,
LOG_REVIEW_QUEUE_DEVELOPER,
LOG_REVIEW_EMAIL_USER,
)
from olympia.constants.applications import * # noqa
from olympia.constants.base import * # noqa
from olympia.constants.licenses import * # noqa

24
src/olympia/amo/admin.py
Просмотреть файл

@ -31,8 +31,10 @@ class CommaSearchInAdminMixin:
# Not pretty but looking up the actual field would require truly
# resolving the field name, walking to any relations we find up until
# the last one, that would be a lot of work for a simple edge case.
if any(field_name in lookup_fields for field_name in
('localized_string', 'localized_string_clean')):
if any(
field_name in lookup_fields
for field_name in ('localized_string', 'localized_string_clean')
):
rval = True
return rval
@ -103,24 +105,24 @@ class CommaSearchInAdminMixin:
queryset = queryset.filter(**{orm_lookup: search_terms})
else:
orm_lookups = [
construct_search(str(search_field))
for search_field in search_fields]
construct_search(str(search_field)) for search_field in search_fields
]
for bit in search_terms:
or_queries = [models.Q(**{orm_lookup: bit})
for orm_lookup in orm_lookups]
or_queries = [
models.Q(**{orm_lookup: bit}) for orm_lookup in orm_lookups
]
q_for_this_term = models.Q(
functools.reduce(operator.or_, or_queries))
q_for_this_term = models.Q(functools.reduce(operator.or_, or_queries))
filters.append(q_for_this_term)
use_distinct |= any(
# Use our own lookup_needs_distinct(), not django's.
self.lookup_needs_distinct(self.opts, search_spec)
for search_spec in orm_lookups)
for search_spec in orm_lookups
)
if filters:
queryset = queryset.filter(
functools.reduce(joining_operator, filters))
queryset = queryset.filter(functools.reduce(joining_operator, filters))
return queryset, use_distinct

86
src/olympia/amo/celery.py
Просмотреть файл

@ -17,7 +17,10 @@ from celery.signals import task_failure, task_postrun, task_prerun
from django_statsd.clients import statsd
from kombu import serialization
from post_request_task.task import (
PostRequestTask, _start_queuing_tasks, _send_tasks_and_stop_queuing)
PostRequestTask,
_start_queuing_tasks,
_send_tasks_and_stop_queuing,
)
import olympia.core.logger
@ -38,35 +41,33 @@ class AMOTask(PostRequestTask):
that would cause them to try to serialize data that has already been
serialized...
"""
abstract = True
def _serialize_args_and_kwargs_for_eager_mode(
self, args=None, kwargs=None, **options):
self, args=None, kwargs=None, **options
):
producer = options.get('producer')
with app.producer_or_acquire(producer) as eager_producer:
serializer = options.get(
'serializer', eager_producer.serializer
)
serializer = options.get('serializer', eager_producer.serializer)
body = args, kwargs
content_type, content_encoding, data = serialization.dumps(
body, serializer
)
args, kwargs = serialization.loads(
data, content_type, content_encoding
)
content_type, content_encoding, data = serialization.dumps(body, serializer)
args, kwargs = serialization.loads(data, content_type, content_encoding)
return args, kwargs
def apply_async(self, args=None, kwargs=None, **options):
if app.conf.task_always_eager:
args, kwargs = self._serialize_args_and_kwargs_for_eager_mode(
args=args, kwargs=kwargs, **options)
args=args, kwargs=kwargs, **options
)
return super().apply_async(args=args, kwargs=kwargs, **options)
def apply(self, args=None, kwargs=None, **options):
if app.conf.task_always_eager:
args, kwargs = self._serialize_args_and_kwargs_for_eager_mode(
args=args, kwargs=kwargs, **options)
args=args, kwargs=kwargs, **options
)
return super().apply(args=args, kwargs=kwargs, **options)
@ -79,8 +80,9 @@ app.autodiscover_tasks()
@task_failure.connect
def process_failure_signal(exception, traceback, sender, task_id,
signal, args, kwargs, einfo, **kw):
def process_failure_signal(
exception, traceback, sender, task_id, signal, args, kwargs, einfo, **kw
):
"""Catch any task failure signals from within our worker processes and log
them as exceptions, so they appear in Sentry and ordinary logging
output."""
@ -94,18 +96,21 @@ def process_failure_signal(exception, traceback, sender, task_id,
'task_id': task_id,
'sender': sender,
'args': args,
'kwargs': kwargs
'kwargs': kwargs,
}
})
},
)
@task_prerun.connect
def start_task_timer(task_id, task, **kw):
timer = TaskTimer()
log.info('starting task timer; id={id}; name={name}; '
'current_dt={current_dt}'
.format(id=task_id, name=task.name,
current_dt=timer.current_datetime))
log.info(
'starting task timer; id={id}; name={name}; '
'current_dt={current_dt}'.format(
id=task_id, name=task.name, current_dt=timer.current_datetime
)
)
# Cache start time for one hour. This will allow us to catch crazy long
# tasks. Currently, stats indexing tasks run around 20-30 min.
@ -119,35 +124,41 @@ def track_task_run_time(task_id, task, **kw):
timer = TaskTimer()
start_time = cache.get(timer.cache_key(task_id))
if start_time is None:
log.info('could not track task run time; id={id}; name={name}; '
'current_dt={current_dt}'
.format(id=task_id, name=task.name,
current_dt=timer.current_datetime))
log.info(
'could not track task run time; id={id}; name={name}; '
'current_dt={current_dt}'.format(
id=task_id, name=task.name, current_dt=timer.current_datetime
)
)
else:
run_time = timer.current_epoch_ms - start_time
log.info('tracking task run time; id={id}; name={name}; '
'run_time={run_time}; current_dt={current_dt}'
.format(id=task_id, name=task.name,
current_dt=timer.current_datetime,
run_time=run_time))
log.info(
'tracking task run time; id={id}; name={name}; '
'run_time={run_time}; current_dt={current_dt}'.format(
id=task_id,
name=task.name,
current_dt=timer.current_datetime,
run_time=run_time,
)
)
statsd.timing('tasks.{}'.format(task.name), run_time)
cache.delete(timer.cache_key(task_id))
class TaskTimer(object):
def __init__(self):
from olympia.amo.utils import utc_millesecs_from_epoch
self.current_datetime = datetime.datetime.now()
self.current_epoch_ms = utc_millesecs_from_epoch(
self.current_datetime)
self.current_epoch_ms = utc_millesecs_from_epoch(self.current_datetime)
def cache_key(self, task_id):
return 'task_start_time.{}'.format(task_id)
def create_chunked_tasks_signatures(
task, items, chunk_size, task_args=None, task_kwargs=None):
task, items, chunk_size, task_args=None, task_kwargs=None
):
"""
Splits a task depending on a list of items into a bunch of tasks of the
specified chunk_size, passing a chunked queryset and optional additional
@ -155,6 +166,7 @@ def create_chunked_tasks_signatures(
Return the group of task signatures without executing it."""
from olympia.amo.utils import chunked
if task_args is None:
task_args = ()
if task_kwargs is None:
@ -164,11 +176,7 @@ def create_chunked_tasks_signatures(
task.si(chunk, *task_args, **task_kwargs)
for chunk in chunked(items, chunk_size)
]
log.info(
'Created a group of %s tasks for task "%s".',
len(tasks),
str(task.name)
)
log.info('Created a group of %s tasks for task "%s".', len(tasks), str(task.name))
return group(tasks)

105
src/olympia/amo/context_processors.py
Просмотреть файл

@ -9,15 +9,16 @@ from olympia.amo.urlresolvers import reverse
def static_url(request):
return {'CDN_HOST': settings.CDN_HOST,
'STATIC_URL': settings.STATIC_URL}
return {'CDN_HOST': settings.CDN_HOST, 'STATIC_URL': settings.STATIC_URL}
def i18n(request):
lang = get_language()
return {'LANGUAGES': settings.LANGUAGES,
'LANG': settings.LANGUAGE_URL_MAP.get(lang) or lang,
'DIR': 'rtl' if get_language_bidi() else 'ltr'}
return {
'LANGUAGES': settings.LANGUAGES,
'LANG': settings.LANGUAGE_URL_MAP.get(lang) or lang,
'DIR': 'rtl' if get_language_bidi() else 'ltr',
}
def global_settings(request):
@ -37,58 +38,84 @@ def global_settings(request):
if getattr(request, 'user', AnonymousUser()).is_authenticated:
is_reviewer = acl.is_user_any_kind_of_reviewer(request.user)
account_links.append({'text': ugettext('My Profile'),
'href': request.user.get_url_path()})
account_links.append(
{'text': ugettext('My Profile'), 'href': request.user.get_url_path()}
)
account_links.append({'text': ugettext('Account Settings'),
'href': reverse('users.edit')})
account_links.append({
'text': ugettext('My Collections'),
'href': reverse('collections.list')})
account_links.append(
{'text': ugettext('Account Settings'), 'href': reverse('users.edit')}
)
account_links.append(
{'text': ugettext('My Collections'), 'href': reverse('collections.list')}
)
if request.user.favorite_addons:
account_links.append(
{'text': ugettext('My Favorites'),
'href': reverse('collections.detail',
args=[request.user.id, 'favorites'])})
{
'text': ugettext('My Favorites'),
'href': reverse(
'collections.detail', args=[request.user.id, 'favorites']
),
}
)
account_links.append({
'text': ugettext('Log out'),
'href': reverse('devhub.logout') + '?to=' + urlquote(request.path),
})
account_links.append(
{
'text': ugettext('Log out'),
'href': reverse('devhub.logout') + '?to=' + urlquote(request.path),
}
)
if request.user.is_developer:
tools_links.append({'text': ugettext('Manage My Submissions'),
'href': reverse('devhub.addons')})
tools_links.append(
{
'text': ugettext('Manage My Submissions'),
'href': reverse('devhub.addons'),
}
)
tools_links.append(
{'text': ugettext('Submit a New Add-on'),
'href': reverse('devhub.submit.agreement')})
{
'text': ugettext('Submit a New Add-on'),
'href': reverse('devhub.submit.agreement'),
}
)
tools_links.append(
{'text': ugettext('Submit a New Theme'),
'href': reverse('devhub.submit.agreement')})
{
'text': ugettext('Submit a New Theme'),
'href': reverse('devhub.submit.agreement'),
}
)
tools_links.append(
{'text': ugettext('Developer Hub'),
'href': reverse('devhub.index')})
{'text': ugettext('Developer Hub'), 'href': reverse('devhub.index')}
)
tools_links.append(
{'text': ugettext('Manage API Keys'),
'href': reverse('devhub.api_key')}
{'text': ugettext('Manage API Keys'), 'href': reverse('devhub.api_key')}
)
if is_reviewer:
tools_links.append({'text': ugettext('Reviewer Tools'),
'href': reverse('reviewers.dashboard')})
tools_links.append(
{
'text': ugettext('Reviewer Tools'),
'href': reverse('reviewers.dashboard'),
}
)
if acl.action_allowed(request, amo.permissions.ANY_ADMIN):
tools_links.append({'text': ugettext('Admin Tools'),
'href': reverse('admin:index')})
tools_links.append(
{'text': ugettext('Admin Tools'), 'href': reverse('admin:index')}
)
context['user'] = request.user
else:
context['user'] = AnonymousUser()
context.update({'account_links': account_links,
'settings': settings,
'amo': amo,
'tools_links': tools_links,
'tools_title': tools_title,
'is_reviewer': is_reviewer})
context.update(
{
'account_links': account_links,
'settings': settings,
'amo': amo,
'tools_links': tools_links,
'tools_title': tools_title,
'is_reviewer': is_reviewer,
}
)
return context

41
src/olympia/amo/cron.py
Просмотреть файл

@ -22,18 +22,21 @@ log = olympia.core.logger.getLogger('z.cron')
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.info('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))
logs = (
ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP)
.values_list('id', flat=True)
)
collections_to_delete = (
Collection.objects.filter(created__lt=days_ago(2),
type=amo.COLLECTION_ANONYMOUS)
.values_list('id', flat=True))
collections_to_delete = Collection.objects.filter(
created__lt=days_ago(2), type=amo.COLLECTION_ANONYMOUS
).values_list('id', flat=True)
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
@ -44,19 +47,20 @@ def gc(test_result=True):
# Delete stale add-ons with no versions. Should soft-delete add-ons that
# are somehow not in incomplete status, hard-delete the rest. No email
# should be sent in either case.
versionless_addons = (
Addon.objects.filter(versions__pk=None, created__lte=a_week_ago)
.values_list('pk', flat=True)
)
versionless_addons = Addon.objects.filter(
versions__pk=None, created__lte=a_week_ago
).values_list('pk', flat=True)
for chunk in chunked(versionless_addons, 100):
delete_addons.delay(chunk)
# Delete stale FileUploads.
stale_uploads = FileUpload.objects.filter(
created__lte=a_week_ago).order_by('id')
stale_uploads = FileUpload.objects.filter(created__lte=a_week_ago).order_by('id')
for file_upload in stale_uploads:
log.info(u'[FileUpload:{uuid}] Removing file: {path}'
.format(uuid=file_upload.uuid, path=file_upload.path))
log.info(
u'[FileUpload:{uuid}] Removing file: {path}'.format(
uuid=file_upload.uuid, path=file_upload.path
)
)
if file_upload.path:
try:
storage.delete(file_upload.path)
@ -77,7 +81,8 @@ def category_totals():
file_statuses = ",".join(['%s'] * len(VALID_FILE_STATUSES))
with connection.cursor() as cursor:
cursor.execute("""
cursor.execute(
"""
UPDATE categories AS t INNER JOIN (
SELECT at.category_id, COUNT(DISTINCT Addon.id) AS ct
FROM addons AS Addon
@ -93,5 +98,7 @@ def category_totals():
GROUP BY at.category_id)
AS j ON (t.id = j.category_id)
SET t.count = j.ct
""" % (file_statuses, addon_statuses),
VALID_FILE_STATUSES + VALID_ADDON_STATUSES)
"""
% (file_statuses, addon_statuses),
VALID_FILE_STATUSES + VALID_ADDON_STATUSES,
)

44
src/olympia/amo/decorators.py
Просмотреть файл

@ -21,11 +21,13 @@ def login_required(f=None, redirect=True):
If redirect=False then we return 401 instead of redirecting to the
login page. That's nice for ajax views.
"""
def decorator(func):
@functools.wraps(func)
def wrapper(request, *args, **kw):
# Prevent circular ref in accounts.utils
from olympia.accounts.utils import redirect_for_login
if request.user.is_authenticated:
return func(request, *args, **kw)
else:
@ -33,7 +35,9 @@ def login_required(f=None, redirect=True):
return redirect_for_login(request)
else:
return http.HttpResponse(status=401)
return wrapper
if f:
return decorator(f)
else:
@ -47,6 +51,7 @@ def post_required(f):
return http.HttpResponseNotAllowed(['POST'])
else:
return f(request, *args, **kw)
return wrapper
@ -56,11 +61,14 @@ def permission_required(permission):
@login_required
def wrapper(request, *args, **kw):
from olympia.access import acl
if acl.action_allowed(request, permission):
return f(request, *args, **kw)
else:
raise PermissionDenied
return wrapper
return decorator
@ -71,13 +79,14 @@ def json_response(response, has_trans=False, status_code=200):
"""
# to avoid circular imports with users.models
from .utils import AMOJSONEncoder
if has_trans:
response = json.dumps(response, cls=AMOJSONEncoder)
else:
response = json.dumps(response)
return http.HttpResponse(response,
content_type='application/json',
status=status_code)
return http.HttpResponse(
response, content_type='application/json', status=status_code
)
def json_view(f=None, has_trans=False, status_code=200):
@ -88,9 +97,12 @@ def json_view(f=None, has_trans=False, status_code=200):
if isinstance(response, http.HttpResponse):
return response
else:
return json_response(response, has_trans=has_trans,
status_code=status_code)
return json_response(
response, has_trans=has_trans, status_code=status_code
)
return wrapper
if f:
return decorator(f)
else:
@ -98,7 +110,8 @@ def json_view(f=None, has_trans=False, status_code=200):
json_view.error = lambda s: http.HttpResponseBadRequest(
json.dumps(s), content_type='application/json')
json.dumps(s), content_type='application/json'
)
def use_primary_db(f):
@ -106,6 +119,7 @@ def use_primary_db(f):
def wrapper(*args, **kw):
with context.use_primary_db():
return f(*args, **kw)
return wrapper
@ -130,8 +144,10 @@ def set_modified_on(f):
# kwargs to the set_modified_on_object task. Useful to set
# things like icon hashes.
kwargs_from_result = result if isinstance(result, dict) else {}
task_log.info('Delaying setting modified on object: %s, %s' %
(obj_info[0], obj_info[1]))
task_log.info(
'Delaying setting modified on object: %s, %s'
% (obj_info[0], obj_info[1])
)
# Execute set_modified_on_object in NFS_LAG_DELAY seconds. This
# allows us to make sure any changes have been written to disk
# before changing modification date and/or image hashes stored
@ -140,15 +156,20 @@ def set_modified_on(f):
set_modified_on_object.apply_async(
args=obj_info,
kwargs=kwargs_from_result,
eta=(datetime.datetime.now() +
datetime.timedelta(seconds=settings.NFS_LAG_DELAY)))
eta=(
datetime.datetime.now()
+ datetime.timedelta(seconds=settings.NFS_LAG_DELAY)
),
)
return result
return wrapper
def allow_cross_site_request(f):
"""Allow other sites to access this resource, see
https://developer.mozilla.org/en/HTTP_access_control."""
@functools.wraps(f)
def wrapper(request, *args, **kw):
response = f(request, *args, **kw)
@ -158,6 +179,7 @@ def allow_cross_site_request(f):
response['Access-Control-Allow-Origin'] = '*'
response['Access-Control-Allow-Methods'] = 'GET'
return response
return wrapper
@ -170,9 +192,11 @@ def allow_mine(f):
"""
# Prevent circular ref in accounts.utils
from olympia.accounts.utils import redirect_for_login
if user_id == 'mine':
if not request.user.is_authenticated:
return redirect_for_login(request)
user_id = request.user.id
return f(request, user_id, *args, **kw)
return wrapper

6
src/olympia/amo/feeds.py
Просмотреть файл

@ -10,6 +10,7 @@ class BaseFeed(Feed):
A base feed class that does not use transactions and tries to avoid raising
exceptions on unserializable content.
"""
# Regexp controlling which characters to strip from the items because they
# would raise UnserializableContentError. Pretty much all control chars
# except things like line feed, carriage return etc which are fine.
@ -26,8 +27,7 @@ class BaseFeed(Feed):
# we're returning, so we can use it to strip XML control chars before they
# are being used. This avoid raising UnserializableContentError later.
def _get_dynamic_attr(self, attname, obj, default=None):
data = super(BaseFeed, self)._get_dynamic_attr(
attname, obj, default=default)
data = super(BaseFeed, self)._get_dynamic_attr(attname, obj, default=default)
# Limite the search to the item types we know can potentially contain
# some weird characters.
@ -38,7 +38,7 @@ class BaseFeed(Feed):
'item_author_name',
'item_description',
'item_title',
'title'
'title',
)
if data and attname in problematic_keys:
data = re.sub(self.CONTROL_CHARS_REGEXP, '', str(data))

21
src/olympia/amo/fields.py
Просмотреть файл

@ -19,6 +19,7 @@ class PositiveAutoField(models.AutoField):
Because AutoFields are special we need a custom database backend to support
using them. See olympia.core.db.mysql.base for that."""
description = _("Positive integer")
def get_internal_type(self):
@ -29,7 +30,6 @@ class PositiveAutoField(models.AutoField):
class HttpHttpsOnlyURLField(fields.URLField):
def __init__(self, *args, **kwargs):
super(HttpHttpsOnlyURLField, self).__init__(*args, **kwargs)
@ -42,10 +42,11 @@ class HttpHttpsOnlyURLField(fields.URLField):
message=_(
'This field can only be used to link to external websites.'
' URLs on %(domain)s are not allowed.',
) % {'domain': settings.DOMAIN},
)
% {'domain': settings.DOMAIN},
code='no_amo_url',
inverse_match=True
)
inverse_match=True,
),
]
@ -62,15 +63,14 @@ def validate_cidr(value):
ipaddress.ip_network(value)
except ValueError:
raise exceptions.ValidationError(
_('Enter a valid IP4 or IP6 network.'), code='invalid')
_('Enter a valid IP4 or IP6 network.'), code='invalid'
)
class CIDRField(models.Field):
empty_strings_allowed = False
description = _('CIDR')
default_error_messages = {
'invalid': _('Enter a valid IP4 or IP6 network.')
}
default_error_messages = {'invalid': _('Enter a valid IP4 or IP6 network.')}
def __init__(self, verbose_name=None, name=None, *args, **kwargs):
self.validators = [validate_cidr]
@ -112,9 +112,6 @@ class CIDRField(models.Field):
return str(value)
def formfield(self, **kwargs):
defaults = {
'form_class': fields.CharField,
'validators': self.validators
}
defaults = {'form_class': fields.CharField, 'validators': self.validators}
defaults.update(kwargs)
return super(CIDRField, self).formfield(**defaults)

7
src/olympia/amo/forms.py
Просмотреть файл

@ -5,7 +5,6 @@ from django.utils.functional import cached_property
class AMOModelForm(forms.ModelForm):
@cached_property
def changed_data(self):
"""
@ -19,9 +18,9 @@ class AMOModelForm(forms.ModelForm):
changed_data = forms.ModelForm.changed_data.__get__(self)[:]
changed_translation_fields = [
field.name for field in Model._meta.get_fields()
if isinstance(field, TranslatedField) and
field.name in changed_data
field.name
for field in Model._meta.get_fields()
if isinstance(field, TranslatedField) and field.name in changed_data
]
# If there are translated fields, pull the model from the database

22
src/olympia/amo/indexers.py
Просмотреть файл

@ -25,6 +25,7 @@ class BaseSearchIndexer(object):
- create_new_index(cls, index_name)
- reindex_tasks_group(cls, index_name)
"""
@classmethod
def get_index_alias(cls):
return settings.ES_INDEXES.get(cls.get_model().ES_ALIAS_KEY)
@ -51,8 +52,9 @@ class BaseSearchIndexer(object):
for field_name in field_names:
# _translations is the suffix in TranslationSerializer.
mapping[doc_name]['properties']['%s_translations' % field_name] = (
cls.get_translations_definition())
mapping[doc_name]['properties'][
'%s_translations' % field_name
] = cls.get_translations_definition()
@classmethod
def get_translations_definition(cls):
@ -65,8 +67,8 @@ class BaseSearchIndexer(object):
'type': 'object',
'properties': {
'lang': {'type': 'text', 'index': False},
'string': {'type': 'text', 'index': False}
}
'string': {'type': 'text', 'index': False},
},
}
@classmethod
@ -108,8 +110,7 @@ class BaseSearchIndexer(object):
}
@classmethod
def attach_language_specific_analyzers_with_raw_variant(
cls, mapping, field_names):
def attach_language_specific_analyzers_with_raw_variant(cls, mapping, field_names):
"""
Like attach_language_specific_analyzers() but with an extra field to
storethe "raw" variant of the value, for exact matches.
@ -124,7 +125,7 @@ class BaseSearchIndexer(object):
'analyzer': analyzer,
'fields': {
'raw': cls.get_raw_field_definition(),
}
},
}
@classmethod
@ -137,7 +138,8 @@ class BaseSearchIndexer(object):
db_field = '%s_id' % field
extend_with_me = {
'%s_translations' % field: [
'%s_translations'
% field: [
{'lang': to_language(lang), 'string': str(string)}
for lang, string in obj.translations[getattr(obj, db_field)]
if string
@ -174,9 +176,7 @@ class BaseSearchIndexer(object):
if db_field is None:
db_field = '%s_id' % field
translations = dict(
obj.translations[getattr(obj, db_field)]
)
translations = dict(obj.translations[getattr(obj, db_field)])
return {
'%s_l10n_%s' % (field, lang): translations.get(lang) or ''

13
src/olympia/amo/management/commands/addnewversion.py
Просмотреть файл

@ -8,8 +8,10 @@ from olympia.applications.models import AppVersion
class Command(BaseCommand):
help = ('Add a new version of an application. Syntax: \n'
' ./manage.py addnewversion <application_name> <version>')
help = (
'Add a new version of an application. Syntax: \n'
' ./manage.py addnewversion <application_name> <version>'
)
log = olympia.core.logger.getLogger('z.appversions')
def add_arguments(self, parser):
@ -23,7 +25,9 @@ class Command(BaseCommand):
raise CommandError(self.help)
msg = 'Adding version %r to application %r\n' % (
options['version'], options['application_name'])
options['version'],
options['application_name'],
)
self.log.info(msg)
self.stdout.write(msg)
@ -32,7 +36,6 @@ def do_addnewversion(application, version):
if application not in amo.APPS:
raise CommandError('Application %r does not exist.' % application)
try:
AppVersion.objects.create(application=amo.APPS[application].id,
version=version)
AppVersion.objects.create(application=amo.APPS[application].id, version=version)
except IntegrityError as e:
raise CommandError('Version %r already exists: %r' % (version, e))

99
src/olympia/amo/management/commands/compress_assets.py
Просмотреть файл

@ -16,7 +16,8 @@ from olympia.lib.jingo_minify_helpers import ensure_path_exists
def run_command(command):
"""Run a command and correctly poll the output and write that to stdout"""
process = subprocess.Popen(
command, stdout=subprocess.PIPE, shell=True, universal_newlines=True)
command, stdout=subprocess.PIPE, shell=True, universal_newlines=True
)
while True:
output = process.stdout.readline()
if output == '' and process.poll() is not None:
@ -27,7 +28,7 @@ def run_command(command):
class Command(BaseCommand):
help = ('Compresses css and js assets defined in settings.MINIFY_BUNDLES')
help = 'Compresses css and js assets defined in settings.MINIFY_BUNDLES'
# This command must not do any system checks because Django runs db-field
# related checks since 1.10 which require a working MySQL connection.
@ -44,19 +45,19 @@ class Command(BaseCommand):
def add_arguments(self, parser):
"""Handle command arguments."""
parser.add_argument(
'--force', action='store_true',
help='Ignores modified/created dates and forces compression.')
'--force',
action='store_true',
help='Ignores modified/created dates and forces compression.',
)
def generate_build_id(self):
return uuid.uuid4().hex[:8]
def update_hashes(self):
# Adds a time based hash on to the build id.
self.build_id = '%s-%s' % (
self.generate_build_id(), hex(int(time.time()))[2:])
self.build_id = '%s-%s' % (self.generate_build_id(), hex(int(time.time()))[2:])
build_id_file = os.path.realpath(
os.path.join(settings.ROOT, 'build.py'))
build_id_file = os.path.realpath(os.path.join(settings.ROOT, 'build.py'))
with open(build_id_file, 'w') as f:
f.write('BUILD_ID_CSS = "%s"\n' % self.build_id)
@ -75,11 +76,25 @@ class Command(BaseCommand):
for name, files in bundle.items():
# Set the paths to the files.
concatted_file = os.path.join(
settings.ROOT, 'static',
ftype, '%s-all.%s' % (name, ftype,))
settings.ROOT,
'static',
ftype,
'%s-all.%s'
% (
name,
ftype,
),
)
compressed_file = os.path.join(
settings.ROOT, 'static',
ftype, '%s-min.%s' % (name, ftype,))
settings.ROOT,
'static',
ftype,
'%s-min.%s'
% (
name,
ftype,
),
)
ensure_path_exists(concatted_file)
ensure_path_exists(compressed_file)
@ -96,13 +111,13 @@ class Command(BaseCommand):
if len(files_all) == 0:
raise CommandError(
'No input files specified in '
'MINIFY_BUNDLES["%s"]["%s"] in settings.py!' %
(ftype, name)
'MINIFY_BUNDLES["%s"]["%s"] in settings.py!' % (ftype, name)
)
run_command('cat {files} > {tmp}'.format(
files=' '.join(files_all),
tmp=tmp_concatted
))
run_command(
'cat {files} > {tmp}'.format(
files=' '.join(files_all), tmp=tmp_concatted
)
)
# Cache bust individual images in the CSS.
if ftype == 'css':
@ -116,8 +131,8 @@ class Command(BaseCommand):
self._minify(ftype, concatted_file, compressed_file)
else:
print(
'File unchanged, skipping minification of %s' % (
concatted_file))
'File unchanged, skipping minification of %s' % (concatted_file)
)
self.minify_skipped += 1
# Write out the hashes
@ -125,8 +140,8 @@ class Command(BaseCommand):
if self.minify_skipped:
print(
'Unchanged files skipped for minification: %s' % (
self.minify_skipped))
'Unchanged files skipped for minification: %s' % (self.minify_skipped)
)
def _preprocess_file(self, filename):
"""Preprocess files and return new filenames."""
@ -135,10 +150,11 @@ class Command(BaseCommand):
target = source
if css_bin:
target = '%s.css' % source
run_command('{lessc} {source} {target}'.format(
lessc=css_bin,
source=str(source),
target=str(target)))
run_command(
'{lessc} {source} {target}'.format(
lessc=css_bin, source=str(source), target=str(target)
)
)
return target
def _is_changed(self, concatted_file):
@ -147,9 +163,9 @@ class Command(BaseCommand):
return True
tmp_concatted = '%s.tmp' % concatted_file
file_exists = (
os.path.exists(concatted_file) and
os.path.getsize(concatted_file) == os.path.getsize(tmp_concatted))
file_exists = os.path.exists(concatted_file) and os.path.getsize(
concatted_file
) == os.path.getsize(tmp_concatted)
if file_exists:
orig_hash = self._file_hash(concatted_file)
temp_hash = self._file_hash(tmp_concatted)
@ -166,7 +182,8 @@ class Command(BaseCommand):
def _cachebust(self, css_file, bundle_name):
"""Cache bust images. Return a new bundle hash."""
self.stdout.write(
'Cache busting images in %s\n' % re.sub('.tmp$', '', css_file))
'Cache busting images in %s\n' % re.sub('.tmp$', '', css_file)
)
if not os.path.exists(css_file):
return
@ -188,8 +205,7 @@ class Command(BaseCommand):
self.checked_hash[css_file] = file_hash
if self.missing_files:
self.stdout.write(
' - Error finding %s images\n' % (self.missing_files,))
self.stdout.write(' - Error finding %s images\n' % (self.missing_files,))
self.missing_files = 0
return file_hash
@ -200,20 +216,18 @@ class Command(BaseCommand):
opts = {'method': 'terser', 'bin': settings.JS_MINIFIER_BIN}
run_command(
'{bin} --compress --mangle -o {target} {source} -m'.format(
bin=opts['bin'],
target=file_out,
source=file_in
bin=opts['bin'], target=file_out, source=file_in
)
)
elif ftype == 'css' and hasattr(settings, 'CLEANCSS_BIN'):
opts = {'method': 'clean-css', 'bin': settings.CLEANCSS_BIN}
run_command('{cleancss} -o {target} {source}'.format(
cleancss=opts['bin'],
target=file_out,
source=file_in))
run_command(
'{cleancss} -o {target} {source}'.format(
cleancss=opts['bin'], target=file_out, source=file_in
)
)
self.stdout.write(
'Minifying %s (using %s)\n' % (file_in, opts['method']))
self.stdout.write('Minifying %s (using %s)\n' % (file_in, opts['method']))
def _file_hash(self, url):
"""Open the file and get a hash of it."""
@ -238,7 +252,6 @@ class Command(BaseCommand):
return 'url(%s)' % url
url = url.split('?')[0]
full_url = os.path.join(
settings.ROOT, os.path.dirname(parent), url)
full_url = os.path.join(settings.ROOT, os.path.dirname(parent), url)
return 'url(%s?%s)' % (url, self._file_hash(full_url))

9
src/olympia/amo/management/commands/create_db.py
Просмотреть файл

@ -11,13 +11,12 @@ class Command(BaseCommand):
"""Based on django_extension's reset_db command but simplifed and with
support for all character sets defined in settings."""
help = ('Creates the database for this project.')
help = 'Creates the database for this project.'
def add_arguments(self, parser):
super(Command, self).add_arguments(parser)
parser.add_argument(
'--force', action='store_true',
help='Drops any existing database first.'
'--force', action='store_true', help='Drops any existing database first.'
)
def handle(self, *args, **options):
@ -47,7 +46,9 @@ class Command(BaseCommand):
character_set = db_info.get('OPTIONS').get('charset', 'utf8mb4')
create_query = 'CREATE DATABASE `%s` CHARACTER SET %s' % (
database_name, character_set)
database_name,
character_set,
)
if drop_query:
logging.info('Executing... "' + drop_query + '"')
connection.query(drop_query)

24
src/olympia/amo/management/commands/cron.py
Просмотреть файл

@ -24,20 +24,26 @@ class Command(BaseCommand):
def handle(self, *args, **options):
if not options['name']:
log.error("Cron called without args")
raise CommandError('These jobs are available:\n%s' % '\n'.join(
sorted(settings.CRON_JOBS.keys())))
raise CommandError(
'These jobs are available:\n%s'
% '\n'.join(sorted(settings.CRON_JOBS.keys()))
)
name, args_and_kwargs = options['name'], options['cron_args']
args = [arg for arg in args_and_kwargs if '=' not in arg]
kwargs = dict(
(kwarg.split('=', maxsplit=1) for kwarg in args_and_kwargs
if kwarg not in args))
(
kwarg.split('=', maxsplit=1)
for kwarg in args_and_kwargs
if kwarg not in args
)
)
path = settings.CRON_JOBS.get(name)
if not path:
log.error(
'Cron called with an unknown cron job: '
f'{name} {args} {kwargs}')
'Cron called with an unknown cron job: ' f'{name} {args} {kwargs}'
)
raise CommandError(f'Unrecognized job name: {name}')
module = import_module(path)
@ -46,8 +52,10 @@ class Command(BaseCommand):
log.info(
f'Beginning job: {name} {args} {kwargs} '
f'(start timestamp: {current_millis})')
f'(start timestamp: {current_millis})'
)
getattr(module, name)(*args, **kwargs)
log.info(
f'Ending job: {name} {args} {kwargs} '
f'(start timestamp: {current_millis})')
f'(start timestamp: {current_millis})'
)

3
src/olympia/amo/management/commands/generate_jsi18n_files.py
Просмотреть файл

@ -18,7 +18,8 @@ class Command(BaseCommand):
fake_request.method = 'GET'
for lang in settings.AMO_LANGUAGES:
filename = os.path.join(
settings.STATICFILES_DIRS[0], 'js', 'i18n', '%s.js' % lang)
settings.STATICFILES_DIRS[0], 'js', 'i18n', '%s.js' % lang
)
with translation.override(lang):
response = JavaScriptCatalog.as_view()(fake_request)
with open(filename, 'w') as f:

22
src/olympia/amo/messages.py
Просмотреть файл

@ -33,11 +33,13 @@ class DoubleSafe(safestring.SafeText, jinja2.Markup):
"""
def _make_message(title=None, message=None, title_safe=False,
message_safe=False):
def _make_message(title=None, message=None, title_safe=False, message_safe=False):
context = {
'title': title, 'message': message,
'title_safe': title_safe, 'message_safe': message_safe}
'title': title,
'message': message,
'title_safe': title_safe,
'message_safe': message_safe,
}
tpl = loader.get_template('message_content.html').render(context)
return DoubleSafe(tpl)
@ -67,8 +69,16 @@ def _is_dupe(msg, request):
return is_dupe
def _file_message(type_, request, title, message=None, extra_tags='',
fail_silently=False, title_safe=False, message_safe=False):
def _file_message(
type_,
request,
title,
message=None,
extra_tags='',
fail_silently=False,
title_safe=False,
message_safe=False,
):
msg = _make_message(title, message, title_safe, message_safe)
# Don't save duplicates.
if _is_dupe(msg, request):

51
src/olympia/amo/middleware.py
Просмотреть файл

@ -12,8 +12,10 @@ from django.contrib.sessions.middleware import SessionMiddleware
from django.db import transaction
from django.urls import is_valid_path
from django.http import (
HttpResponsePermanentRedirect, HttpResponseRedirect,
JsonResponse)
HttpResponsePermanentRedirect,
HttpResponseRedirect,
JsonResponse,
)
from django.middleware import common
from django.utils.cache import patch_cache_control, patch_vary_headers
from django.utils.deprecation import MiddlewareMixin
@ -53,13 +55,14 @@ class LocaleAndAppURLMiddleware(MiddlewareMixin):
urlresolvers.set_url_prefix(prefixer)
full_path = prefixer.fix(prefixer.shortened_path)
if (prefixer.app == amo.MOBILE.short and
request.path.rstrip('/').endswith('/' + amo.MOBILE.short)):
if prefixer.app == amo.MOBILE.short and request.path.rstrip('/').endswith(
'/' + amo.MOBILE.short
):
return redirect_type(request.path.replace('/mobile', '/android'))
if ('lang' in request.GET and not re.match(
settings.SUPPORTED_NONAPPS_NONLOCALES_REGEX,
prefixer.shortened_path)):
if 'lang' in request.GET and not re.match(
settings.SUPPORTED_NONAPPS_NONLOCALES_REGEX, prefixer.shortened_path
):
# Blank out the locale so that we can set a new one. Remove lang
# from query params so we don't have an infinite loop.
prefixer.locale = ''
@ -103,13 +106,14 @@ class AuthenticationMiddlewareWithoutAPI(AuthenticationMiddleware):
Like AuthenticationMiddleware, but disabled for the API, which uses its
own authentication mechanism.
"""
def process_request(self, request):
if request.is_api and not auth_path.match(request.path):
request.user = AnonymousUser()
else:
return super(
AuthenticationMiddlewareWithoutAPI,
self).process_request(request)
return super(AuthenticationMiddlewareWithoutAPI, self).process_request(
request
)
class NoVarySessionMiddleware(SessionMiddleware):
@ -122,6 +126,7 @@ class NoVarySessionMiddleware(SessionMiddleware):
We skip the cache in Zeus if someone has an AMOv3+ cookie, so varying on
Cookie at this level only hurts us.
"""
def process_response(self, request, response):
if settings.READ_ONLY:
return response
@ -132,9 +137,9 @@ class NoVarySessionMiddleware(SessionMiddleware):
if hasattr(response, 'get'):
vary = response.get('Vary', None)
new_response = (
super(NoVarySessionMiddleware, self)
.process_response(request, response))
new_response = super(NoVarySessionMiddleware, self).process_response(
request, response
)
if vary:
new_response['Vary'] = vary
@ -152,10 +157,12 @@ class RemoveSlashMiddleware(MiddlewareMixin):
"""
def process_response(self, request, response):
if (response.status_code == 404 and
request.path_info.endswith('/') and
not is_valid_path(request.path_info) and
is_valid_path(request.path_info[:-1])):
if (
response.status_code == 404
and request.path_info.endswith('/')
and not is_valid_path(request.path_info)
and is_valid_path(request.path_info[:-1])
):
# Use request.path because we munged app/locale in path_info.
newurl = request.path[:-1]
if request.GET:
@ -183,7 +190,6 @@ def safe_query_string(request):
class CommonMiddleware(common.CommonMiddleware):
def process_request(self, request):
with safe_query_string(request):
return super(CommonMiddleware, self).process_request(request)
@ -194,6 +200,7 @@ class NonAtomicRequestsForSafeHttpMethodsMiddleware(MiddlewareMixin):
Middleware to make the view non-atomic if the HTTP method used is safe,
in order to avoid opening and closing a useless transaction.
"""
def process_view(self, request, view_func, view_args, view_kwargs):
# This uses undocumented django APIS:
# - transaction.get_connection() followed by in_atomic_block property,
@ -217,10 +224,12 @@ class ReadOnlyMiddleware(MiddlewareMixin):
Supports issuing `Retry-After` header.
"""
ERROR_MSG = _(
u'Some features are temporarily disabled while we '
u'perform website maintenance. We\'ll be back to '
u'full capacity shortly.')
u'full capacity shortly.'
)
def process_request(self, request):
if not settings.READ_ONLY:
@ -250,6 +259,7 @@ class SetRemoteAddrFromForwardedFor(MiddlewareMixin):
Our application servers should always be behind a load balancer that sets
this header correctly.
"""
def is_valid_ip(self, ip):
for af in (socket.AF_INET, socket.AF_INET6):
try:
@ -263,8 +273,7 @@ class SetRemoteAddrFromForwardedFor(MiddlewareMixin):
ips = []
if 'HTTP_X_FORWARDED_FOR' in request.META:
xff = [i.strip() for i in
request.META['HTTP_X_FORWARDED_FOR'].split(',')]
xff = [i.strip() for i in request.META['HTTP_X_FORWARDED_FOR'].split(',')]
ips = [ip for ip in xff if self.is_valid_ip(ip)]
else:
return

86
src/olympia/amo/models.py
Просмотреть файл

@ -39,7 +39,6 @@ def use_primary_db():
class BaseQuerySet(models.QuerySet):
def __init__(self, *args, **kwargs):
super(BaseQuerySet, self).__init__(*args, **kwargs)
self._transform_fns = []
@ -76,6 +75,7 @@ class BaseQuerySet(models.QuerySet):
def only_translations(self):
"""Remove all transforms except translations."""
from olympia.translations import transformer
# Add an extra select so these are cached separately.
qs = self.no_transforms()
if hasattr(self.model._meta, 'translated_fields'):
@ -108,6 +108,7 @@ class ManagerBase(models.Manager):
If a model has translated fields, they'll be attached through a transform
function.
"""
_queryset_class = BaseQuerySet
def get_queryset(self):
@ -116,6 +117,7 @@ class ManagerBase(models.Manager):
def _with_translations(self, qs):
from olympia.translations import transformer
# Since we're attaching translations to the object, we need to stick
# the locale in the query so objects aren't shared across locales.
if hasattr(self.model._meta, 'translated_fields'):
@ -130,8 +132,9 @@ class ManagerBase(models.Manager):
return self.all().transform(fn)
def raw(self, raw_query, params=None, *args, **kwargs):
return RawQuerySet(raw_query, self.model, params=params,
using=self._db, *args, **kwargs)
return RawQuerySet(
raw_query, self.model, params=params, using=self._db, *args, **kwargs
)
class _NoChangeInstance(object):
@ -228,8 +231,12 @@ class OnChangeMixin(object):
new_attr = old_attr.copy()
new_attr.update(new_attr_kw)
for cb in _on_change_callbacks[self.__class__]:
cb(old_attr=old_attr, new_attr=new_attr,
instance=_NoChangeInstance(self), sender=self.__class__)
cb(
old_attr=old_attr,
new_attr=new_attr,
instance=_NoChangeInstance(self),
sender=self.__class__,
)
def save(self, *args, **kw):
"""
@ -270,8 +277,12 @@ class SearchMixin(object):
def index(cls, document, id=None, refresh=False, index=None):
"""Wrapper around Elasticsearch.index."""
search.get_es().index(
body=document, index=index or cls._get_index(),
doc_type=cls.get_mapping_type(), id=id, refresh=refresh)
body=document,
index=index or cls._get_index(),
doc_type=cls.get_mapping_type(),
id=id,
refresh=refresh,
)
@classmethod
def unindex(cls, id, index=None):
@ -335,8 +346,7 @@ class SaveUpdateMixin(object):
objects = cls.get_unfiltered_manager()
objects.filter(pk=self.pk).update(**kw)
if signal:
models.signals.post_save.send(sender=cls, instance=self,
created=False)
models.signals.post_save.send(sender=cls, instance=self, created=False)
def save(self, **kwargs):
# Unfortunately we have to save our translations before we call `save`
@ -356,8 +366,7 @@ class ModelBase(SearchMixin, SaveUpdateMixin, models.Model):
* Fetches all translations in one subsequent query during initialization.
"""
created = models.DateTimeField(
default=timezone.now, editable=False, blank=True)
created = models.DateTimeField(default=timezone.now, editable=False, blank=True)
modified = models.DateTimeField(auto_now=True)
objects = ManagerBase()
@ -390,8 +399,7 @@ class ModelBase(SearchMixin, SaveUpdateMixin, models.Model):
"""
Return the relative URL pointing to the instance admin change page.
"""
urlname = 'admin:%s_%s_change' % (
self._meta.app_label, self._meta.model_name)
urlname = 'admin:%s_%s_change' % (self._meta.app_label, self._meta.model_name)
return reverse(urlname, args=(self.pk,))
def get_admin_absolute_url(self):
@ -415,9 +423,9 @@ def manual_order(qs, pks, pk_name='id'):
if not pks:
return qs.none()
return qs.filter(id__in=pks).extra(
select={'_manual': 'FIELD(%s, %s)' % (pk_name,
','.join(map(str, pks)))},
order_by=['_manual'])
select={'_manual': 'FIELD(%s, %s)' % (pk_name, ','.join(map(str, pks)))},
order_by=['_manual'],
)
class SlugField(models.SlugField):
@ -425,6 +433,7 @@ class SlugField(models.SlugField):
Django 1.6's SlugField rejects non-ASCII slugs. This field just
keeps the old behaviour of not checking contents.
"""
default_validators = []
@ -445,6 +454,7 @@ class BasePreview(object):
def _image_url(self, url_template):
from olympia.amo.templatetags.jinja_helpers import user_media_url
if self.modified is not None:
modified = int(time.mktime(self.modified.timetuple()))
else:
@ -454,6 +464,7 @@ class BasePreview(object):
def _image_path(self, url_template):
from olympia.amo.templatetags.jinja_helpers import user_media_path
args = [user_media_path(self.media_folder), self.id // 1000, self.id]
return url_template % tuple(args)
@ -488,28 +499,30 @@ class BasePreview(object):
@classmethod
def delete_preview_files(cls, sender, instance, **kw):
"""On delete of the Preview object from the database, unlink the image
and thumb on the file system """
and thumb on the file system"""
image_paths = [
instance.image_path, instance.thumbnail_path,
instance.original_path]
instance.image_path,
instance.thumbnail_path,
instance.original_path,
]
for filename in image_paths:
try:
log.info('Removing filename: %s for preview: %s'
% (filename, instance.pk))
log.info(
'Removing filename: %s for preview: %s' % (filename, instance.pk)
)
storage.delete(filename)
except Exception as e:
log.error(
'Error deleting preview file (%s): %s' % (filename, e))
log.error('Error deleting preview file (%s): %s' % (filename, e))
class LongNameIndex(models.Index):
"""Django's Index, but with a longer allowed name since we don't care about
compatibility with Oracle."""
max_name_length = 64 # Django default is 30, but MySQL can go up to 64.
class FilterableManyToManyDescriptor(ManyToManyDescriptor):
def __init__(self, *args, **kwargs):
self.q_filter = kwargs.pop('q_filter', None)
super().__init__(*args, **kwargs)
@ -518,13 +531,14 @@ class FilterableManyToManyDescriptor(ManyToManyDescriptor):
def _get_manager_with_default_filtering(cls, manager, q_filter):
"""This is wrapping the manager class so we can add an extra
filter to the queryset returned via get_queryset."""
class ManagerWithFiltering(manager):
def get_queryset(self):
# Check the queryset caching django uses during these lookups -
# we only want to add the q_filter the first time.
from_cache = (
self.prefetch_cache_name in
getattr(self.instance, '_prefetched_objects_cache', {}))
from_cache = self.prefetch_cache_name in getattr(
self.instance, '_prefetched_objects_cache', {}
)
qs = super().get_queryset()
if not from_cache and q_filter:
# Here is where we add the filter.
@ -570,9 +584,12 @@ class FilterableManyToManyField(models.fields.related.ManyToManyField):
super().contribute_to_class(cls, name, **kwargs)
# Add the descriptor for the m2m relation.
setattr(
cls, self.name,
cls,
self.name,
FilterableManyToManyDescriptor(
self.remote_field, reverse=False, q_filter=self.q_filter))
self.remote_field, reverse=False, q_filter=self.q_filter
),
)
def contribute_to_related_class(self, cls, related):
"""All we're doing here is overriding the `setattr` so it creates an
@ -580,10 +597,13 @@ class FilterableManyToManyField(models.fields.related.ManyToManyField):
ManyToManyDescriptor, and pass down the q_filter property."""
super().contribute_to_related_class(cls, related)
if (
not self.remote_field.is_hidden() and
not related.related_model._meta.swapped
not self.remote_field.is_hidden()
and not related.related_model._meta.swapped
):
setattr(
cls, related.get_accessor_name(),
cls,
related.get_accessor_name(),
FilterableManyToManyDescriptor(
self.remote_field, reverse=True, q_filter=self.q_filter))
self.remote_field, reverse=True, q_filter=self.q_filter
),
)

22
src/olympia/amo/monitors.py
Просмотреть файл

@ -48,8 +48,9 @@ def memcache():
memcache_results.append((ip, port, result))
if not using_twemproxy and len(memcache_results) < 2:
status = ('2+ memcache servers are required.'
'%s available') % len(memcache_results)
status = ('2+ memcache servers are required.' '%s available') % len(
memcache_results
)
monitor_log.warning(status)
if not memcache_results:
@ -70,8 +71,7 @@ def libraries():
msg = "Failed to create a jpeg image: %s" % e
libraries_results.append(('PIL+JPEG', False, msg))
missing_libs = [
lib for lib, success, _ in libraries_results if not success]
missing_libs = [lib for lib, success, _ in libraries_results if not success]
if missing_libs:
status = 'missing libs: %s' % ",".join(missing_libs)
return status, libraries_results
@ -101,10 +101,12 @@ def path():
user_media_path('guarded_addons'),
user_media_path('addon_icons'),
user_media_path('previews'),
user_media_path('userpics'),)
user_media_path('userpics'),
)
read_only = [os.path.join(settings.ROOT, 'locale')]
filepaths = [(path, os.R_OK | os.W_OK, 'We want read + write')
for path in read_and_write]
filepaths = [
(path, os.R_OK | os.W_OK, 'We want read + write') for path in read_and_write
]
filepaths += [(path, os.R_OK, 'We want read') for path in read_only]
filepath_results = []
filepath_status = True
@ -154,11 +156,13 @@ def signer():
try:
response = requests.get(
'{host}/__heartbeat__'.format(host=autograph_url),
timeout=settings.SIGNING_SERVER_MONITORING_TIMEOUT)
timeout=settings.SIGNING_SERVER_MONITORING_TIMEOUT,
)
if response.status_code != 200:
status = (
'Failed to chat with signing service. '
'Invalid HTTP response code.')
'Invalid HTTP response code.'
)
monitor_log.critical(status)
signer_results = False
else:

14
src/olympia/amo/pagination.py
Просмотреть файл

@ -2,7 +2,12 @@ from math import ceil
from django.conf import settings
from django.core.paginator import (
EmptyPage, InvalidPage, Page, PageNotAnInteger, Paginator)
EmptyPage,
InvalidPage,
Page,
PageNotAnInteger,
Paginator,
)
from django.utils.functional import cached_property
@ -38,9 +43,7 @@ class ESPaginator(Paginator):
return 0
# Make sure we never return a page beyond max_result_window
hits = min(
self.max_result_window,
max(1, self.count - self.orphans))
hits = min(self.max_result_window, max(1, self.count - self.orphans))
return int(ceil(hits / float(self.per_page)))
def validate_number(self, number):
@ -67,8 +70,7 @@ class ESPaginator(Paginator):
top = bottom + self.per_page
if top > self.max_result_window:
raise InvalidPage(
'That page number is too high for the current page size')
raise InvalidPage('That page number is too high for the current page size')
# Force the search to evaluate and then attach the count. We want to
# avoid an extra useless query even if there are no results, so we

18
src/olympia/amo/search.py
Просмотреть файл

@ -20,8 +20,11 @@ def get_es(hosts=None, timeout=None, **settings):
"""Create an ES object and return it."""
# Cheap way of de-None-ifying things
hosts = hosts or getattr(dj_settings, 'ES_HOSTS', DEFAULT_HOSTS)
timeout = (timeout if timeout is not None else
getattr(dj_settings, 'ES_TIMEOUT', DEFAULT_TIMEOUT))
timeout = (
timeout
if timeout is not None
else getattr(dj_settings, 'ES_TIMEOUT', DEFAULT_TIMEOUT)
)
if os.environ.get('RUNNING_IN_CI'):
settings['http_auth'] = ('elastic', 'changeme')
@ -30,7 +33,6 @@ def get_es(hosts=None, timeout=None, **settings):
class ES(object):
def __init__(self, type_, index):
self.type = type_
self.index = index
@ -151,8 +153,7 @@ class ES(object):
search = Search().query(query)
if query_string:
search = SearchQueryFilter().apply_search_query(
query_string, search)
search = SearchQueryFilter().apply_search_query(query_string, search)
if sort:
search = search.sort(*sort)
@ -191,7 +192,8 @@ class ES(object):
elif field_action == 'exists':
if val is not True:
raise NotImplementedError(
'<field>__exists only works with a "True" value.')
'<field>__exists only works with a "True" value.'
)
filters.append(Q('exists', **{'field': key}))
elif field_action == 'in':
filters.append(Q('terms', **{key: val}))
@ -269,7 +271,6 @@ class ES(object):
class SearchResults(object):
def __init__(self, type, results, source):
self.type = type
self.took = results['took']
@ -289,7 +290,6 @@ class SearchResults(object):
class DictSearchResults(SearchResults):
def set_objects(self, hits):
self.objects = [r['_source'] for r in hits]
@ -297,7 +297,6 @@ class DictSearchResults(SearchResults):
class ListSearchResults(SearchResults):
def set_objects(self, hits):
# When fields are specified in `values(...)` we return the fields.
objs = []
@ -308,7 +307,6 @@ class ListSearchResults(SearchResults):
class ObjectSearchResults(SearchResults):
def set_objects(self, hits):
self.ids = [int(r['_id']) for r in hits]
self.objects = self.type.objects.filter(id__in=self.ids)

18
src/olympia/amo/storage_utils.py
Просмотреть файл

@ -13,8 +13,9 @@ from django.utils.encoding import force_text
DEFAULT_CHUNK_SIZE = 64 * 2 ** 10 # 64kB
def walk_storage(path, topdown=True, onerror=None, followlinks=False,
storage=default_storage):
def walk_storage(
path, topdown=True, onerror=None, followlinks=False, storage=default_storage
):
"""
Generate the file names in a stored directory tree by walking the tree
top-down.
@ -45,8 +46,9 @@ def walk_storage(path, topdown=True, onerror=None, followlinks=False,
roots[:] = new_roots
def copy_stored_file(src_path, dest_path, storage=default_storage,
chunk_size=DEFAULT_CHUNK_SIZE):
def copy_stored_file(
src_path, dest_path, storage=default_storage, chunk_size=DEFAULT_CHUNK_SIZE
):
"""
Copy one storage path to another storage path.
@ -64,8 +66,9 @@ def copy_stored_file(src_path, dest_path, storage=default_storage,
break
def move_stored_file(src_path, dest_path, storage=default_storage,
chunk_size=DEFAULT_CHUNK_SIZE):
def move_stored_file(
src_path, dest_path, storage=default_storage, chunk_size=DEFAULT_CHUNK_SIZE
):
"""
Move a storage path to another storage path.
@ -73,8 +76,7 @@ def move_stored_file(src_path, dest_path, storage=default_storage,
This attempts to be compatible with a wide range of storage backends
rather than attempt to be optimized for each individual one.
"""
copy_stored_file(src_path, dest_path, storage=storage,
chunk_size=chunk_size)
copy_stored_file(src_path, dest_path, storage=storage, chunk_size=chunk_size)
storage.delete(src_path)

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше