From a1635e241935de68ccb5f8a46f91e8f891cc134b Mon Sep 17 00:00:00 2001 From: Jeff Balogh Date: Fri, 17 Jun 2011 12:06:42 -0700 Subject: [PATCH] redoing compat reports with es (bug 656527) --- apps/compat/cron.py | 75 +++++++++++----------- apps/compat/models.py | 17 +++++ apps/compat/templates/compat/details.html | 11 ++++ apps/compat/templates/compat/index.html | 78 ++++++++++++++--------- apps/compat/urls.py | 1 - apps/compat/views.py | 75 ++++++++++++++-------- media/js/zamboni/compat.js | 29 --------- settings.py | 12 ++-- 8 files changed, 170 insertions(+), 128 deletions(-) create mode 100644 apps/compat/templates/compat/details.html delete mode 100644 media/js/zamboni/compat.js diff --git a/apps/compat/cron.py b/apps/compat/cron.py index b09f46403c..b402d24ecc 100644 --- a/apps/compat/cron.py +++ b/apps/compat/cron.py @@ -1,62 +1,63 @@ +import collections import logging from django.conf import settings from django.db.models import Sum, Max import cronjobs +import elasticutils import redisutils import amo +import amo.utils import versions.compare as vc from addons.models import Addon from stats.models import UpdateCount +from .models import AppCompat + log = logging.getLogger('z.compat') @cronjobs.register def compatibility_report(): redis = redisutils.connections['master'] + docs = collections.defaultdict(dict) - # for app in amo.APP_USAGE: - for compat in settings.COMPAT: - app = amo.APPS_ALL[compat['app']] - version = compat['version'] - log.info(u'Making compat report for %s %s.' % (app.pretty, version)) - versions = (('latest', version), ('beta', version + 'b'), - ('alpha', compat['alpha'])) - - rv = dict((k, 0) for k in dict(versions)) - rv['other'] = 0 - - ignore = (amo.STATUS_NULL, amo.STATUS_DISABLED) - qs = (Addon.objects.exclude(type=amo.ADDON_PERSONA, status__in=ignore) - .filter(appsupport__app=app.id, name__locale='en-us')) - + # Gather all the data for the index. + for app in amo.APP_USAGE: + log.info(u'Making compat report for %s.' % app.pretty) latest = UpdateCount.objects.aggregate(d=Max('date'))['d'] qs = UpdateCount.objects.filter(addon__appsupport__app=app.id, + addon__status__in=amo.VALID_STATUSES, + addon___current_version__isnull=False, date=latest) total = qs.aggregate(Sum('count'))['count__sum'] - addons = list(qs.values_list('addon', 'count', 'addon__appsupport__min', - 'addon__appsupport__max')) - - # Count up the top 95% of addons by ADU. + redis.hset('compat:%s' % app.id, 'total', total) adus = 0 - for addon, count, minver, maxver in addons: - # Don't count add-ons that weren't compatible with the previous - # release - if maxver < vc.version_int(compat['previous']): - continue - if adus < .95 * total: - adus += count - else: - break - for key, version in versions: - if minver <= vc.version_int(version) <= maxver: - rv[key] += 1 - break - else: - rv['other'] += 1 - log.info(u'Compat for %s %s: %s' % (app.pretty, version, rv)) - key = '%s:%s' % (app.id, version) - redis.hmset('compat:' + key, rv) + + updates = dict(qs.values_list('addon', 'count')) + for chunk in amo.utils.chunked(updates.items(), 50): + chunk = dict(chunk) + for addon in Addon.objects.filter(id__in=chunk): + doc = docs[addon.id] + doc.update(id=addon.id, slug=addon.slug, + name=unicode(addon.name)) + doc.setdefault('usage', {})[app.id] = updates[addon.id] + + if app not in addon.compatible_apps: + continue + compat = addon.compatible_apps[app] + d = {'min': compat.min.version_int, + 'max': compat.max.version_int} + doc.setdefault('support', {})[app.id] = d + doc.setdefault('max_version', {})[app.id] = compat.max.version + doc['top_95'] = adus > .95 * total + + adus += sum(chunk.values()) + + # Send it all to the index. + for chunk in amo.utils.chunked(docs.values(), 150): + for doc in chunk: + AppCompat.index(doc, id=doc['id'], bulk=True) + elasticutils.get_es().flush_bulk(forced=True) diff --git a/apps/compat/models.py b/apps/compat/models.py index 310781af86..1cd6c12748 100644 --- a/apps/compat/models.py +++ b/apps/compat/models.py @@ -19,3 +19,20 @@ class CompatReport(amo.models.ModelBase): class Meta: db_table = 'compatibility_reports' + + +class AppCompat(amo.models.ModelBase): + """ + Stub model for use with search. The schema: + + {id: addon.id, + name: addon.name, + slug: addon.slug, + max_version: {APP.id: version string}, + usage: {APP.id: addon.daily_usage}, + support: {APP.id: {max: version int, min: version int}, + } + """ + + class Meta: + abstract = True diff --git a/apps/compat/templates/compat/details.html b/apps/compat/templates/compat/details.html new file mode 100644 index 0000000000..b342466ea5 --- /dev/null +++ b/apps/compat/templates/compat/details.html @@ -0,0 +1,11 @@ +{% extends "base.html" %} + +{% set title = _('{app} {version} Add-on Compatibility Report')|f(app=request.APP.pretty, version=version) %} +{% block title %}{{ page_title(title) }}{% endblock %} +{% block bodyclass %}inverse{% endblock %} + +{% block content %} +

{{ title }}

+ + +{% endblock %} diff --git a/apps/compat/templates/compat/index.html b/apps/compat/templates/compat/index.html index e07f728afa..cdf553b007 100644 --- a/apps/compat/templates/compat/index.html +++ b/apps/compat/templates/compat/index.html @@ -1,44 +1,62 @@ {% extends "base.html" %} -{% block title %}{{ page_title(_('Compatibility Center')) }}{% endblock %} +{% set app = request.APP.pretty %} +{% set title = _('Add-on Compatibility Report for {app} {version}')|f(app=app, version=version) %} +{% block title %}{{ page_title(title) }}{% endblock %} {% block bodyclass %}inverse{% endblock %} -{% macro percent(x, y) %}{{ (x / y|float * 100)|int }}{% endmacro %} - -{% set app = request.APP.pretty %} -{% set num = percent(versions['latest'], total) %} - {% block content %} +
+

{{ title }}

+ {{ breadcrumbs([(None, _('{app} {version} Compatibility')|f(app=app, version=version))]) }} +
-

{{ _('{app} {version} Add-on Compatibility Report')|f(app=app, version=version) }}

-

{% trans %} -{{ num }}% of add-ons on addons.mozilla.org are compatible with {{ app }} {{ version }}. -{% endtrans %}

+{% set titles = { + 'prev': _('Top 95% compatible with previous version'), + 'top_95': _('Top 95% of all add-ons'), + 'all': _('All active add-ons'), +} %} -
-
- +
+ - - {# L10n: "#" means "number" #} - + + {% with key, (total, facets) = compat_levels[0] %} + {% for version, _ in facets %} + + {% endfor %} + {% endwith %} - {% for key, title in keys %} + + + {% for key, (total, facets) in compat_levels %} + {% if total %} - - + + {% for version, count in facets %} + + {% endfor %} + {% endif %} {% endfor %} -
{{ _('Version') }}{{ _('# of Add-ons') }}{{ version }}
{{ title }}{{ versions[key]|numberfmt }}{{ titles[key] }}{{ (100 * count / total)|round(2) }}% ({{ count }})
-

{{ _('Detailed Report') }}

-
+ + + + +

{{ _('Details for add-ons compatible with previous version:') }}

+ + + {% for addon in usage_addons.object_list %} + + + + + + + {% endfor %} + +
{{ addon.name }}{{ addon.max_version }}{{ addon.usage|numberfmt }} users ({{ (100 * addon.usage / usage_total)|round(2) }}%)
+{{ usage_addons|paginator }} {% endblock %} - -{% block js %} - - -{% endblock %} diff --git a/apps/compat/urls.py b/apps/compat/urls.py index 18251ce28b..0a54b387a7 100644 --- a/apps/compat/urls.py +++ b/apps/compat/urls.py @@ -10,5 +10,4 @@ urlpatterns = ( views.reporter_detail, name='compat.reporter_detail'), url('^(?P[.\w]+)?$', views.index, name='compat.index'), - url('^(?P[.\w]+)/details$', views.details, name='compat.details'), ) diff --git a/apps/compat/views.py b/apps/compat/views.py index c91fe3fe2c..c352526a04 100644 --- a/apps/compat/views.py +++ b/apps/compat/views.py @@ -4,47 +4,73 @@ import re from django import http from django.conf import settings from django.db.models import Count -from django.shortcuts import redirect, get_object_or_404 +from django.shortcuts import redirect from django.views.decorators.csrf import csrf_exempt import jingo import redisutils -from tower import ugettext_lazy as _lazy +from tower import ugettext as _ import amo.utils from amo.decorators import post_required from addons.models import Addon -from .models import CompatReport - -KEYS = ( - ('latest', _lazy('Latest')), - ('beta', _lazy('Beta')), - ('alpha', _lazy('Alpha')), - ('other', _lazy('Other')), -) +from versions.compare import version_int as vint +from .models import CompatReport, AppCompat def index(request, version=None): COMPAT = [v for v in settings.COMPAT if v['app'] == request.APP.id] - if version is None and COMPAT: - version = COMPAT[0]['version'] - - redis = redisutils.connections['master'] - compat = redis.hgetall('compat:%s:%s' % (request.APP.id, version)) - versions = dict((k, int(v)) for k, v in compat.items()) - - if version not in [v['version'] for v in COMPAT] or not versions: + compat_dict = dict((v['main'], v) for v in COMPAT) + if not COMPAT: raise http.Http404() + if version not in compat_dict: + return redirect('compat.index', COMPAT[0]['main']) - total = sum(versions.values()) - keys = [(k, unicode(v)) for k, v in KEYS] + compat, app = compat_dict[version], str(request.APP.id) + qs = AppCompat.search() + compat_queries = ( + ('prev', qs.query(top_95=True, **{ + 'support.%s.max__gte' % app: vint(compat['previous'])})), + ('top_95', qs.query(top_95=True)), + ('all', qs), + ) + compat_levels = [(key, version_compat(qs, compat, app)) + for key, qs in compat_queries] + usage_addons, usage_total = usage_stats(request, compat, app) return jingo.render(request, 'compat/index.html', - {'versions': versions, 'total': total, - 'version': version, 'keys': keys}) + {'version': version, + 'usage_addons': usage_addons, + 'usage_total': usage_total, + 'compat_levels': compat_levels}) -def details(request, version): - return http.HttpResponse('go away') +def version_compat(qs, compat, app): + facets = [] + for v, prev in zip(compat['versions'], (None,) + compat['versions']): + d = {'from': vint(v)} + if prev: + d['to'] = vint(prev) + facets.append(d) + qs = qs.facet(by_status={'range': {'support.%s.max' % app: facets}}) + result = qs[:0].raw() + total_addons = result['hits']['total'] + ranges = result['facets']['by_status']['ranges'] + faceted = [(v, r['count']) for v, r in zip(compat['versions'], ranges)] + other = total_addons - sum(r[1] for r in faceted) + return total_addons, faceted + [(_('Other'), other)] + + +def usage_stats(request, compat, app): + # Get the list of add-ons for usage stats. + redis = redisutils.connections['master'] + qs = (AppCompat.search().order_by('-usage.%s' % app).values_dict() + .filter(**{'support.%s.max__gte' % app: vint(compat['previous'])})) + addons = amo.utils.paginate(request, qs) + for obj in addons.object_list: + obj['usage'] = obj['usage'][app] + obj['max_version'] = obj['max_version'][app] + total = int(redis.hget('compat:%s' % app, 'total')) + return addons, total @csrf_exempt @@ -107,4 +133,3 @@ def reporter_detail(request, guid): return jingo.render(request, 'compat/reporter_detail.html', dict(reports=reports, works=works, name=name, guid=guid)) - diff --git a/media/js/zamboni/compat.js b/media/js/zamboni/compat.js deleted file mode 100644 index 0b9e70e67b..0000000000 --- a/media/js/zamboni/compat.js +++ /dev/null @@ -1,29 +0,0 @@ -$(function() { - var chart = $('#chart'), - data = JSON.parse(chart.attr('data-data')), - total = chart.attr('data-total'), - keys = JSON.parse(chart.attr('data-keys')), - series = _.map(data, function(value, key) { - return [keys[key], parseInt(value / total * 100)]; - }); -console.log(JSON.stringify(series)); - - var chart = new Highcharts.Chart({ - chart: { - renderTo: 'chart' - }, - title: '', - plotOptions: { - pie: { cursor: 'pointer' } - }, - tooltip: { - formatter: function() { - return ''+ this.point.name +': '+ this.y +' %'; - } - }, - series: [{ - type: 'pie', - data: series - }] - }); -}); diff --git a/settings.py b/settings.py index 48a2607de3..bbaa1c6241 100644 --- a/settings.py +++ b/settings.py @@ -965,13 +965,13 @@ MODIFIED_DELAY = 3 # This is a list of dictionaries that we should generate compat info for. # app: should match amo.FIREFOX.id. -# version: the app version we're generating compat info for. -# alpha: the first version that should be considered alpha for :version. -# previous: the major version before :version. +# main: the app version we're generating compat info for. +# versions: version numbers to show in comparisons. +# previous: the major version before :main. COMPAT = ( - dict(app=1, version='4.0', alpha='3.7a', previous='3.6'), - dict(app=1, version='5.0', alpha='5.0a', previous='4.0'), - dict(app=1, version='6.0', alpha='6.0a', previous='5.0'), + dict(app=1, main='6.0', versions=('6.0', '6.0a2', '6.0a1'), previous='5.0'), + dict(app=1, main='5.0', versions=('5.0', '5.0a2', '5.0a1'), previous='4.0'), + dict(app=1, main='4.0', versions=('4.0', '4.0a1', '3.7a'), previous='3.6'), ) # URL for reporting arecibo errors too. If not set, won't be sent.