diff --git a/apps/amo/helpers.py b/apps/amo/helpers.py index 4e28679341..d27cd660fd 100644 --- a/apps/amo/helpers.py +++ b/apps/amo/helpers.py @@ -351,3 +351,9 @@ def media(context, url): else: build = context['BUILD_ID_IMG'] return context['MEDIA_URL'] + utils.urlparams(url, b=build) + + +@register.function +@jinja2.evalcontextfunction +def attrs(ctx, *args, **kw): + return jinja2.filters.do_xmlattr(ctx, dict(*args, **kw)) diff --git a/apps/blocklist/admin.py b/apps/blocklist/admin.py index 0458a04c3b..17a74a747a 100644 --- a/apps/blocklist/admin.py +++ b/apps/blocklist/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from .models import BlocklistApp, BlocklistItem, BlocklistPlugin +from .models import BlocklistApp, BlocklistItem, BlocklistPlugin, BlocklistGfx admin.site.register(BlocklistApp) admin.site.register(BlocklistItem) admin.site.register(BlocklistPlugin) +admin.site.register(BlocklistGfx) diff --git a/apps/blocklist/models.py b/apps/blocklist/models.py index a49bee8f7b..24b9cd718a 100644 --- a/apps/blocklist/models.py +++ b/apps/blocklist/models.py @@ -1,10 +1,12 @@ from django.db import models +import redisutils + import amo.models class BlocklistApp(amo.models.ModelBase): - blitem = models.ForeignKey('BlocklistItem') + blitem = models.ForeignKey('BlocklistItem', related_name='app') guid = models.CharField(max_length=255, blank=True, db_index=True, null=True) min = models.CharField(max_length=255, blank=True, null=True) @@ -57,3 +59,25 @@ class BlocklistPlugin(amo.models.ModelBase): def flush_urls(self): return ['/blocklist*'] # no lang/app + + +class BlocklistGfx(amo.models.ModelBase): + guid = models.CharField(max_length=255, blank=True, null=True) + os = models.CharField(max_length=255, blank=True, null=True) + vendor = models.CharField(max_length=255, blank=True, null=True) + devices = models.CharField(max_length=255, blank=True, null=True) + feature = models.CharField(max_length=255, blank=True, null=True) + feature_status = models.CharField(max_length=255, blank=True, null=True) + driver_version = models.CharField(max_length=255, blank=True, null=True) + driver_version_comparator = models.CharField(max_length=255, blank=True, + null=True) + + class Meta: + db_table = 'blgfxdrivers' + + def __unicode__(self): + return '%s: %s : %s : %s' % (self.guid, self.os, self.vendor, + self.devices) + + def flush_urls(self): + return ['/blocklist*'] # no lang/app diff --git a/apps/blocklist/templates/blocklist/blocklist.xml b/apps/blocklist/templates/blocklist/blocklist.xml new file mode 100644 index 0000000000..ff91eb9a84 --- /dev/null +++ b/apps/blocklist/templates/blocklist/blocklist.xml @@ -0,0 +1,68 @@ + + +{% if items %} + + {% for guid, rows in items.items() %} + + {% for row in rows.rows %} + {% if row.min or row.max or row.severity or row.apps %} + + {% for app in row.apps %} + + {% if app.min and app.max %} + + {% endif %} + + {% endfor %} + + {% endif %} + {% endfor %} + + {% endfor %} + +{% endif %} + +{% if plugins %} + + {% for plugin in plugins %} + + {% if plugin.name %}{% endif %} + {% if plugin.description %}{% endif %} + {% if plugin.filename %}{% endif %} + {% if plugin.severity or plugin.min or plugin.max %} + + {% if apiver > 2 and plugin.min and plugin.max %} + + + + {% endif %} + + {% endif %} + + {% endfor %} + +{% endif %} + +{% if gfxs %} + + {% for gfx in gfxs %} + + {{ gfx.os }} + {{ gfx.vendor }} + + {% for device in gfx.devices.split(' ') %} + {{ device }} + {% endfor %} + + {{ gfx.feature }} + {{ gfx.feature_status }} + {{ gfx.driver_version }} + {{ gfx.driver_version_comparator }} + + {% endfor %} + +{% endif %} + + + diff --git a/apps/blocklist/tests.py b/apps/blocklist/tests.py new file mode 100644 index 0000000000..e870ef32e0 --- /dev/null +++ b/apps/blocklist/tests.py @@ -0,0 +1,308 @@ +from xml.dom import minidom + +from django.conf import settings +from django.core.cache import cache + +import redisutils +import test_utils +from nose.tools import eq_ + +import amo +from amo.urlresolvers import reverse +from .models import BlocklistItem, BlocklistApp, BlocklistPlugin, BlocklistGfx + + +base_xml = """ + + + +""" + + +class BlocklistTest(test_utils.TestCase): + + def setUp(self): + self.fx4_url = reverse('blocklist', args=[3, amo.FIREFOX.guid, '4.0']) + self.fx2_url = reverse('blocklist', args=[2, amo.FIREFOX.guid, '2.0']) + self.mobile_url = reverse('blocklist', args=[2, amo.MOBILE.guid, '.9']) + self._redis = redisutils.mock_redis() + + def tearDown(self): + redisutils.reset_redis(self._redis) + + def normalize(self, s): + return '\n'.join(x.strip() for x in s.split()) + + def eq_(self, x, y): + return eq_(self.normalize(x), self.normalize(y)) + + +class BlocklistItemTest(BlocklistTest): + + def setUp(self): + super(BlocklistItemTest, self).setUp() + self.item = BlocklistItem.objects.create(guid='guid@addon.com') + self.app = BlocklistApp.objects.create(blitem=self.item, + guid=amo.FIREFOX.guid) + + def test_no_items(self): + self.item.delete() + r = self.client.get(self.fx4_url) + self.eq_(r.content, base_xml) + + def test_new_cookie_per_user(self): + self.client.get(self.fx4_url) + assert settings.BLOCKLIST_COOKIE in self.client.cookies + c = self.client.cookies[settings.BLOCKLIST_COOKIE] + eq_(c['path'], '/blocklist/') + eq_(c['secure'], True) + + def test_existing_user_cookie(self): + self.client.cookies[settings.BLOCKLIST_COOKIE] = 'adfadf' + self.client.get(self.fx4_url) + eq_(self.client.cookies[settings.BLOCKLIST_COOKIE].value, 'adfadf') + + def test_url_params(self): + eq_(self.client.get(self.fx4_url).status_code, 200) + eq_(self.client.get(self.fx2_url).status_code, 200) + # We ignore trailing url parameters. + eq_(self.client.get(self.fx4_url + 'other/junk/').status_code, 200) + + def test_app_guid(self): + # There's one item for Firefox. + r = self.client.get(self.fx4_url) + eq_(r.status_code, 200) + eq_(len(r.context['items']), 1) + + # There are no items for mobile. + r = self.client.get(self.mobile_url) + eq_(r.status_code, 200) + eq_(len(r.context['items']), 0) + + # Without the app constraint we see the item. + self.app.delete() + r = self.client.get(self.mobile_url) + eq_(r.status_code, 200) + eq_(len(r.context['items']), 1) + + def dom(self, url): + r = self.client.get(self.fx4_url) + return minidom.parseString(r.content) + + def test_item_guid(self): + items = self.dom(self.fx4_url).getElementsByTagName('emItem') + eq_(len(items), 1) + eq_(items[0].getAttribute('id'), 'guid@addon.com') + + def test_item_os(self): + item = self.dom(self.fx4_url).getElementsByTagName('emItem')[0] + assert 'os' not in item.attributes.keys() + + self.item.update(os='win,mac') + item = self.dom(self.fx4_url).getElementsByTagName('emItem')[0] + eq_(item.getAttribute('os'), 'win,mac') + + def test_item_severity(self): + self.item.update(severity=2) + eq_(len(self.vr()), 1) + item = self.dom(self.fx4_url).getElementsByTagName('emItem')[0] + vrange = item.getElementsByTagName('versionRange') + eq_(vrange[0].getAttribute('severity'), '2') + + def vr(self): + item = self.dom(self.fx4_url).getElementsByTagName('emItem')[0] + return item.getElementsByTagName('versionRange') + + def test_item_version_range(self): + self.item.update(min='0.1') + eq_(len(self.vr()), 1) + eq_(self.vr()[0].attributes.keys(), ['minVersion']) + eq_(self.vr()[0].getAttribute('minVersion'), '0.1') + + self.item.update(max='0.2') + eq_(self.vr()[0].attributes.keys(), ['minVersion', 'maxVersion']) + eq_(self.vr()[0].getAttribute('minVersion'), '0.1') + eq_(self.vr()[0].getAttribute('maxVersion'), '0.2') + + def test_item_multiple_version_range(self): + # There should be two s under one . + self.item.update(min='0.1', max='0.2') + BlocklistItem.objects.create(guid=self.item.guid, severity=3) + + item = self.dom(self.fx4_url).getElementsByTagName('emItem') + eq_(len(item), 1) + vr = item[0].getElementsByTagName('versionRange') + eq_(len(vr), 2) + eq_(vr[0].getAttribute('minVersion'), '0.1') + eq_(vr[0].getAttribute('maxVersion'), '0.2') + eq_(vr[1].getAttribute('severity'), '3') + + def test_item_target_app(self): + app = self.app + self.app.delete() + self.item.update(severity=2) + version_range = self.vr()[0] + eq_(version_range.getElementsByTagName('targetApplication'), []) + + app.save() + version_range = self.vr()[0] + target_app = version_range.getElementsByTagName('targetApplication') + eq_(len(target_app), 1) + eq_(target_app[0].getAttribute('id'), amo.FIREFOX.guid) + + app.update(min='0.1', max='*') + version_range = self.vr()[0] + target_app = version_range.getElementsByTagName('targetApplication') + eq_(target_app[0].getAttribute('id'), amo.FIREFOX.guid) + tvr = target_app[0].getElementsByTagName('versionRange') + eq_(tvr[0].getAttribute('minVersion'), '0.1') + eq_(tvr[0].getAttribute('maxVersion'), '*') + + def test_item_multiple_apps(self): + # Make sure all s go under the same . + self.app.update(min='0.1', max='0.2') + BlocklistApp.objects.create(guid=amo.FIREFOX.guid, blitem=self.item, + min='3.0', max='3.1') + version_range = self.vr()[0] + apps = version_range.getElementsByTagName('targetApplication') + eq_(len(apps), 2) + eq_(apps[0].getAttribute('id'), amo.FIREFOX.guid) + vr = apps[0].getElementsByTagName('versionRange')[0] + eq_(vr.getAttribute('minVersion'), '0.1') + eq_(vr.getAttribute('maxVersion'), '0.2') + eq_(apps[1].getAttribute('id'), amo.FIREFOX.guid) + vr = apps[1].getElementsByTagName('versionRange')[0] + eq_(vr.getAttribute('minVersion'), '3.0') + eq_(vr.getAttribute('maxVersion'), '3.1') + + def test_item_empty_version_range(self): + # No version_range without an app, min, max, or severity. + self.app.delete() + self.item.update(min=None, max=None, severity=None) + eq_(len(self.vr()), 0) + + def test_item_empty_target_app(self): + # No empty . + self.item.update(severity=1) + self.app.delete() + eq_(self.dom(self.fx4_url).getElementsByTagName('targetApplication'), + []) + + def test_item_target_empty_version_range(self): + app = self.dom(self.fx4_url).getElementsByTagName('targetApplication') + eq_(app[0].getElementsByTagName('versionRange'), []) + + +class BlocklistPluginTest(BlocklistTest): + + def setUp(self): + super(BlocklistPluginTest, self).setUp() + self.plugin = BlocklistPlugin.objects.create(guid=amo.FIREFOX.guid) + + def test_no_plugins(self): + r = self.client.get(self.mobile_url) + self.eq_(r.content, base_xml) + + def dom(self, url=None): + url = url or self.fx4_url + r = self.client.get(url) + d = minidom.parseString(r.content) + return d.getElementsByTagName('pluginItem')[0] + + def test_plugin_empty(self): + eq_(self.dom().attributes.keys(), []) + eq_(self.dom().getElementsByTagName('match'), []) + eq_(self.dom().getElementsByTagName('versionRange'), []) + + def test_plugin_os(self): + self.plugin.update(os='win') + eq_(self.dom().attributes.keys(), ['os']) + eq_(self.dom().getAttribute('os'), 'win') + + def test_plugin_xpcomabi(self): + self.plugin.update(xpcomabi='win') + eq_(self.dom().attributes.keys(), ['xpcomabi']) + eq_(self.dom().getAttribute('xpcomabi'), 'win') + + def test_plugin_name(self): + self.plugin.update(name='flash') + match = self.dom().getElementsByTagName('match') + eq_(len(match), 1) + eq_(dict(match[0].attributes.items()), + {'name': 'name', 'exp': 'flash'}) + + def test_plugin_description(self): + self.plugin.update(description='flash') + match = self.dom().getElementsByTagName('match') + eq_(len(match), 1) + eq_(dict(match[0].attributes.items()), + {'name': 'description', 'exp': 'flash'}) + + def test_plugin_filename(self): + self.plugin.update(filename='flash') + match = self.dom().getElementsByTagName('match') + eq_(len(match), 1) + eq_(dict(match[0].attributes.items()), + {'name': 'filename', 'exp': 'flash'}) + + def test_plugin_severity(self): + self.plugin.update(severity=2) + v = self.dom().getElementsByTagName('versionRange')[0] + eq_(v.getAttribute('severity'), '2') + + def test_plugin_target_app(self): + self.plugin.update(min='1', max='2') + v = self.dom().getElementsByTagName('versionRange')[0] + app = v.getElementsByTagName('targetApplication')[0] + eq_(app.getAttribute('id'), amo.FIREFOX.guid) + vr = app.getElementsByTagName('versionRange')[0] + eq_(vr.getAttribute('minVersion'), '1') + eq_(vr.getAttribute('maxVersion'), '2') + + def test_plugin_apiver_lt_3(self): + self.plugin.update(severity='2') + # No min & max so the app matches. + e = self.dom(self.fx2_url).getElementsByTagName('versionRange')[0] + eq_(e.getAttribute('severity'), '2') + eq_(e.getElementsByTagName('targetApplication'), []) + + # The app version is not in range. + self.plugin.update(min='3.0', max='4.0') + self.assertRaises(IndexError, self.dom, self.fx2_url) + + # The app is back in range. + self.plugin.update(min='1.1') + e = self.dom(self.fx2_url).getElementsByTagName('versionRange')[0] + eq_(e.getAttribute('severity'), '2') + eq_(e.getElementsByTagName('targetApplication'), []) + + +class BlocklistGfxTest(BlocklistTest): + + def setUp(self): + super(BlocklistGfxTest, self).setUp() + self.gfx = BlocklistGfx.objects.create( + guid=amo.FIREFOX.guid, os='os', vendor='vendor', devices='x y z', + feature='feature', feature_status='status', + driver_version='version', driver_version_comparator='compare') + + def test_no_gfx(self): + r = self.client.get(self.mobile_url) + self.eq_(r.content, base_xml) + + def test_gfx(self): + r = self.client.get(self.fx4_url) + dom = minidom.parseString(r.content) + gfx = dom.getElementsByTagName('gfxBlacklistEntry')[0] + find = lambda e: gfx.getElementsByTagName(e)[0].childNodes[0].wholeText + eq_(find('os'), self.gfx.os) + eq_(find('feature'), self.gfx.feature) + eq_(find('vendor'), self.gfx.vendor) + eq_(find('featureStatus'), self.gfx.feature_status) + eq_(find('driverVersion'), self.gfx.driver_version) + eq_(find('driverVersionComparator'), + self.gfx.driver_version_comparator) + devices = gfx.getElementsByTagName('devices')[0] + for device, val in zip(devices.getElementsByTagName('device'), + self.gfx.devices.split(' ')): + eq_(device.childNodes[0].wholeText, val) diff --git a/apps/blocklist/views.py b/apps/blocklist/views.py new file mode 100644 index 0000000000..c173f448e5 --- /dev/null +++ b/apps/blocklist/views.py @@ -0,0 +1,91 @@ +import collections +from datetime import datetime, timedelta +import uuid + +from django.core.cache import cache +from django.conf import settings +from django.db.models import Q, signals as db_signals + +import jingo +import redisutils + +from amo.utils import sorted_groupby +from versions.compare import version_int +from .models import BlocklistItem, BlocklistPlugin, BlocklistGfx, BlocklistApp + + +App = collections.namedtuple('App', 'guid min max') + + +def blocklist(request, apiver, app, appver): + key = 'blocklist:%s:%s:%s'% (apiver, app, appver) + response = cache.get(key) + if response is None: + response = _blocklist(request, apiver, app, appver) + cache.set(key, response, 60 * 60) + # This gets cleared with the clear_blocklist signal handler. + redisutils.connections['master'].sadd('blocklist:keys', key) + if settings.BLOCKLIST_COOKIE not in request.COOKIES: + response.set_cookie(settings.BLOCKLIST_COOKIE, uuid.uuid4(), + expires=datetime.now() + timedelta(days=5 * 365), + path='/blocklist/', secure=True) + return response + + +def _blocklist(request, apiver, app, appver): + apiver = int(apiver) + items = get_items(apiver, app, appver) + plugins = get_plugins(apiver, app, appver) + gfxs = BlocklistGfx.objects.filter(Q(guid__isnull=True) | Q(guid=app)) + return jingo.render(request, 'blocklist/blocklist.xml', + dict(items=items, plugins=plugins, gfxs=gfxs, + apiver=apiver, appguid=app, appver=appver)) + + +def clear_blocklist(*args, **kw): + # Something in the blocklist changed; invalidate all responses. + keys = redisutils.connections['master'].smembers('blocklist:keys') + cache.delete_many(keys) + + +for m in BlocklistItem, BlocklistPlugin, BlocklistGfx, BlocklistApp: + db_signals.post_save.connect(clear_blocklist, sender=m, + dispatch_uid='save_%s' % m) + db_signals.post_delete.connect(clear_blocklist, sender=m, + dispatch_uid='delete_%s' % m) + + +def get_items(apiver, app, appver): + # Collapse multiple blocklist items (different version ranges) into one + # item and collapse each item's apps. + addons = (BlocklistItem.uncached + .filter(Q(app__guid__isnull=True) | Q(app__guid=app)) + .extra(select={'app_guid': 'blapps.guid', + 'app_min': 'blapps.min', + 'app_max': 'blapps.max'})) + items = {} + for guid, rows in sorted_groupby(addons, 'guid'): + rr = [] + for id, rs in sorted_groupby(list(rows), 'id'): + rs = list(rs) + rr.append(rs[0]) + rs[0].apps = [App(r.app_guid, r.app_min, r.app_max) + for r in rs if r.app_guid] + os = [r.os for r in rr if r.os] + items[guid] = {'rows': rr, 'os': os and os[0] or None} + return items + + +def get_plugins(apiver, app, appver): + # API versions < 3 ignore targetApplication entries for plugins so only + # block the plugin if the appver is within the block range. + plugins = BlocklistPlugin.uncached.filter( + Q(guid__isnull=True) | Q(guid=app)) + if apiver < 3: + def between(ver, min, max): + if not (min and max): + return True + return version_int(min) < ver < version_int(max) + app_version = version_int(appver) + plugins = [p for p in plugins if between(app_version, p.min, p.max)] + return plugins diff --git a/settings.py b/settings.py index 59835d7d34..4ea539b96f 100644 --- a/settings.py +++ b/settings.py @@ -165,12 +165,12 @@ ADMIN_MEDIA_PREFIX = '/admin-media/' # paths that don't require an app prefix SUPPORTED_NONAPPS = ('admin', 'developers', 'editors', 'img', 'jsi18n', 'localizers', 'media', 'robots.txt', - 'statistics', 'services', 'update') + 'statistics', 'services', 'update', 'blocklist') DEFAULT_APP = 'firefox' # paths that don't require a locale prefix SUPPORTED_NONLOCALES = ('img', 'media', 'robots.txt', 'services', 'downloads', - 'update') + 'update', 'blocklist') # Make this unique, and don't share it with anybody. SECRET_KEY = 'r#%9w^o_80)7f%!_ir5zx$tu3mupw9u%&s!)-_q%gy7i+fhx#)' @@ -798,3 +798,5 @@ DEFAULT_SUGGESTED_CONTRIBUTION = 5 # Path to `ps`. PS_BIN = '/bin/ps' + +BLOCKLIST_COOKIE = 'BLOCKLIST_v1' diff --git a/urls.py b/urls.py index 8e26908b19..0307d85a3e 100644 --- a/urls.py +++ b/urls.py @@ -5,6 +5,7 @@ from django.shortcuts import redirect from django.views.i18n import javascript_catalog from django.views.decorators.cache import cache_page +import blocklist.views import versions.urls admin.autodiscover() @@ -16,6 +17,11 @@ urlpatterns = patterns('', # Discovery pane is first for undetectable efficiency wins. ('^discovery/', include('discovery.urls')), + # There are many more params but we only care about these three. The end is + # not anchored on purpose! + url('^blocklist/(?P\d+)/(?P[^/]+)/(?P[^/]+)/', + blocklist.views.blocklist, name='blocklist'), + # Add-ons. ('', include('addons.urls')),