redoing compat reports with es (bug 656527)
This commit is contained in:
Jeff Balogh
Родитель
8f6d83bb6c
Коммит
a1635e2419
|
|
@ -1,62 +1,63 @@
|
|||
import collections
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.db.models import Sum, Max
|
||||
|
||||
import cronjobs
|
||||
import elasticutils
|
||||
import redisutils
|
||||
|
||||
import amo
|
||||
import amo.utils
|
||||
import versions.compare as vc
|
||||
from addons.models import Addon
|
||||
from stats.models import UpdateCount
|
||||
|
||||
from .models import AppCompat
|
||||
|
||||
log = logging.getLogger('z.compat')
|
||||
|
||||
|
||||
@cronjobs.register
|
||||
def compatibility_report():
|
||||
redis = redisutils.connections['master']
|
||||
docs = collections.defaultdict(dict)
|
||||
|
||||
# for app in amo.APP_USAGE:
|
||||
for compat in settings.COMPAT:
|
||||
app = amo.APPS_ALL[compat['app']]
|
||||
version = compat['version']
|
||||
log.info(u'Making compat report for %s %s.' % (app.pretty, version))
|
||||
versions = (('latest', version), ('beta', version + 'b'),
|
||||
('alpha', compat['alpha']))
|
||||
|
||||
rv = dict((k, 0) for k in dict(versions))
|
||||
rv['other'] = 0
|
||||
|
||||
ignore = (amo.STATUS_NULL, amo.STATUS_DISABLED)
|
||||
qs = (Addon.objects.exclude(type=amo.ADDON_PERSONA, status__in=ignore)
|
||||
.filter(appsupport__app=app.id, name__locale='en-us'))
|
||||
|
||||
# Gather all the data for the index.
|
||||
for app in amo.APP_USAGE:
|
||||
log.info(u'Making compat report for %s.' % app.pretty)
|
||||
latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
|
||||
qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
|
||||
addon__status__in=amo.VALID_STATUSES,
|
||||
addon___current_version__isnull=False,
|
||||
date=latest)
|
||||
total = qs.aggregate(Sum('count'))['count__sum']
|
||||
addons = list(qs.values_list('addon', 'count', 'addon__appsupport__min',
|
||||
'addon__appsupport__max'))
|
||||
|
||||
# Count up the top 95% of addons by ADU.
|
||||
redis.hset('compat:%s' % app.id, 'total', total)
|
||||
adus = 0
|
||||
for addon, count, minver, maxver in addons:
|
||||
# Don't count add-ons that weren't compatible with the previous
|
||||
# release
|
||||
if maxver < vc.version_int(compat['previous']):
|
||||
continue
|
||||
if adus < .95 * total:
|
||||
adus += count
|
||||
else:
|
||||
break
|
||||
for key, version in versions:
|
||||
if minver <= vc.version_int(version) <= maxver:
|
||||
rv[key] += 1
|
||||
break
|
||||
else:
|
||||
rv['other'] += 1
|
||||
log.info(u'Compat for %s %s: %s' % (app.pretty, version, rv))
|
||||
key = '%s:%s' % (app.id, version)
|
||||
redis.hmset('compat:' + key, rv)
|
||||
|
||||
updates = dict(qs.values_list('addon', 'count'))
|
||||
for chunk in amo.utils.chunked(updates.items(), 50):
|
||||
chunk = dict(chunk)
|
||||
for addon in Addon.objects.filter(id__in=chunk):
|
||||
doc = docs[addon.id]
|
||||
doc.update(id=addon.id, slug=addon.slug,
|
||||
name=unicode(addon.name))
|
||||
doc.setdefault('usage', {})[app.id] = updates[addon.id]
|
||||
|
||||
if app not in addon.compatible_apps:
|
||||
continue
|
||||
compat = addon.compatible_apps[app]
|
||||
d = {'min': compat.min.version_int,
|
||||
'max': compat.max.version_int}
|
||||
doc.setdefault('support', {})[app.id] = d
|
||||
doc.setdefault('max_version', {})[app.id] = compat.max.version
|
||||
doc['top_95'] = adus > .95 * total
|
||||
|
||||
adus += sum(chunk.values())
|
||||
|
||||
# Send it all to the index.
|
||||
for chunk in amo.utils.chunked(docs.values(), 150):
|
||||
for doc in chunk:
|
||||
AppCompat.index(doc, id=doc['id'], bulk=True)
|
||||
elasticutils.get_es().flush_bulk(forced=True)
|
||||
|
|
|
|||
|
|
@ -19,3 +19,20 @@ class CompatReport(amo.models.ModelBase):
|
|||
|
||||
class Meta:
|
||||
db_table = 'compatibility_reports'
|
||||
|
||||
|
||||
class AppCompat(amo.models.ModelBase):
|
||||
"""
|
||||
Stub model for use with search. The schema:
|
||||
|
||||
{id: addon.id,
|
||||
name: addon.name,
|
||||
slug: addon.slug,
|
||||
max_version: {APP.id: version string},
|
||||
usage: {APP.id: addon.daily_usage},
|
||||
support: {APP.id: {max: version int, min: version int},
|
||||
}
|
||||
"""
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
{% extends "base.html" %}
|
||||
|
||||
{% set title = _('{app} {version} Add-on Compatibility Report')|f(app=request.APP.pretty, version=version) %}
|
||||
{% block title %}{{ page_title(title) }}{% endblock %}
|
||||
{% block bodyclass %}inverse{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>{{ title }}</h1>
|
||||
|
||||
|
||||
{% endblock %}
|
||||
|
|
@ -1,44 +1,62 @@
|
|||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}{{ page_title(_('Compatibility Center')) }}{% endblock %}
|
||||
{% set app = request.APP.pretty %}
|
||||
{% set title = _('Add-on Compatibility Report for {app} {version}')|f(app=app, version=version) %}
|
||||
{% block title %}{{ page_title(title) }}{% endblock %}
|
||||
{% block bodyclass %}inverse{% endblock %}
|
||||
|
||||
{% macro percent(x, y) %}{{ (x / y|float * 100)|int }}{% endmacro %}
|
||||
|
||||
{% set app = request.APP.pretty %}
|
||||
{% set num = percent(versions['latest'], total) %}
|
||||
|
||||
{% block content %}
|
||||
<header>
|
||||
<h2>{{ title }}</h2>
|
||||
{{ breadcrumbs([(None, _('{app} {version} Compatibility')|f(app=app, version=version))]) }}
|
||||
</header>
|
||||
|
||||
<h1>{{ _('{app} {version} Add-on Compatibility Report')|f(app=app, version=version) }}</h1>
|
||||
<p>{% trans %}
|
||||
<b>{{ num }}%</b> of add-ons on addons.mozilla.org are compatible with {{ app }} {{ version }}.
|
||||
{% endtrans %}</p>
|
||||
{% set titles = {
|
||||
'prev': _('Top 95% compatible with previous version'),
|
||||
'top_95': _('Top 95% of all add-ons'),
|
||||
'all': _('All active add-ons'),
|
||||
} %}
|
||||
|
||||
<div id="chart" class="primary"
|
||||
data-keys="{{ dict(keys)|json }}"
|
||||
data-data="{{ versions|json }}"
|
||||
data-total="{{ total }}"></div>
|
||||
<div class="secondary">
|
||||
<table id="compat">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>{{ _('Version') }}</th>
|
||||
{# L10n: "#" means "number" #}
|
||||
<th>{{ _('# of Add-ons') }}</th>
|
||||
<th></th>
|
||||
{% with key, (total, facets) = compat_levels[0] %}
|
||||
{% for version, _ in facets %}
|
||||
<th>{{ version }}</th>
|
||||
{% endfor %}
|
||||
{% endwith %}
|
||||
</tr>
|
||||
{% for key, title in keys %}
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for key, (total, facets) in compat_levels %}
|
||||
{% if total %}
|
||||
<tr>
|
||||
<td>{{ title }}</td>
|
||||
<td>{{ versions[key]|numberfmt }}</td>
|
||||
<th>{{ titles[key] }}</th>
|
||||
{% for version, count in facets %}
|
||||
<td>{{ (100 * count / total)|round(2) }}% ({{ count }})</td>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</table>
|
||||
<p><a href="{{ url('compat.details', version) }}">{{ _('Detailed Report') }}</a></p>
|
||||
</div>
|
||||
</tbody>
|
||||
</table>
|
||||
<ul>
|
||||
</ul>
|
||||
|
||||
<h3>{{ _('Details for add-ons compatible with previous version:') }}</h3>
|
||||
<table>
|
||||
<tbody>
|
||||
{% for addon in usage_addons.object_list %}
|
||||
<tr>
|
||||
<td><a href="{{ url('addons.detail', addon.slug) }}">{{ addon.name }}</a></td>
|
||||
<td>{{ addon.max_version }}</td>
|
||||
<td>{{ addon.usage|numberfmt }} users </td>
|
||||
<td>({{ (100 * addon.usage / usage_total)|round(2) }}%)</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{{ usage_addons|paginator }}
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block js %}
|
||||
<script src="{{ media('js/lib/highcharts.src.js') }}"></script>
|
||||
<script src="{{ media('js/zamboni/compat.js') }}"></script>
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -10,5 +10,4 @@ urlpatterns = (
|
|||
views.reporter_detail, name='compat.reporter_detail'),
|
||||
|
||||
url('^(?P<version>[.\w]+)?$', views.index, name='compat.index'),
|
||||
url('^(?P<version>[.\w]+)/details$', views.details, name='compat.details'),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,47 +4,73 @@ import re
|
|||
from django import http
|
||||
from django.conf import settings
|
||||
from django.db.models import Count
|
||||
from django.shortcuts import redirect, get_object_or_404
|
||||
from django.shortcuts import redirect
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
import jingo
|
||||
import redisutils
|
||||
from tower import ugettext_lazy as _lazy
|
||||
from tower import ugettext as _
|
||||
|
||||
import amo.utils
|
||||
from amo.decorators import post_required
|
||||
from addons.models import Addon
|
||||
from .models import CompatReport
|
||||
|
||||
KEYS = (
|
||||
('latest', _lazy('Latest')),
|
||||
('beta', _lazy('Beta')),
|
||||
('alpha', _lazy('Alpha')),
|
||||
('other', _lazy('Other')),
|
||||
)
|
||||
from versions.compare import version_int as vint
|
||||
from .models import CompatReport, AppCompat
|
||||
|
||||
|
||||
def index(request, version=None):
|
||||
COMPAT = [v for v in settings.COMPAT if v['app'] == request.APP.id]
|
||||
if version is None and COMPAT:
|
||||
version = COMPAT[0]['version']
|
||||
|
||||
redis = redisutils.connections['master']
|
||||
compat = redis.hgetall('compat:%s:%s' % (request.APP.id, version))
|
||||
versions = dict((k, int(v)) for k, v in compat.items())
|
||||
|
||||
if version not in [v['version'] for v in COMPAT] or not versions:
|
||||
compat_dict = dict((v['main'], v) for v in COMPAT)
|
||||
if not COMPAT:
|
||||
raise http.Http404()
|
||||
if version not in compat_dict:
|
||||
return redirect('compat.index', COMPAT[0]['main'])
|
||||
|
||||
total = sum(versions.values())
|
||||
keys = [(k, unicode(v)) for k, v in KEYS]
|
||||
compat, app = compat_dict[version], str(request.APP.id)
|
||||
qs = AppCompat.search()
|
||||
compat_queries = (
|
||||
('prev', qs.query(top_95=True, **{
|
||||
'support.%s.max__gte' % app: vint(compat['previous'])})),
|
||||
('top_95', qs.query(top_95=True)),
|
||||
('all', qs),
|
||||
)
|
||||
compat_levels = [(key, version_compat(qs, compat, app))
|
||||
for key, qs in compat_queries]
|
||||
usage_addons, usage_total = usage_stats(request, compat, app)
|
||||
return jingo.render(request, 'compat/index.html',
|
||||
{'versions': versions, 'total': total,
|
||||
'version': version, 'keys': keys})
|
||||
{'version': version,
|
||||
'usage_addons': usage_addons,
|
||||
'usage_total': usage_total,
|
||||
'compat_levels': compat_levels})
|
||||
|
||||
|
||||
def details(request, version):
|
||||
return http.HttpResponse('go away')
|
||||
def version_compat(qs, compat, app):
|
||||
facets = []
|
||||
for v, prev in zip(compat['versions'], (None,) + compat['versions']):
|
||||
d = {'from': vint(v)}
|
||||
if prev:
|
||||
d['to'] = vint(prev)
|
||||
facets.append(d)
|
||||
qs = qs.facet(by_status={'range': {'support.%s.max' % app: facets}})
|
||||
result = qs[:0].raw()
|
||||
total_addons = result['hits']['total']
|
||||
ranges = result['facets']['by_status']['ranges']
|
||||
faceted = [(v, r['count']) for v, r in zip(compat['versions'], ranges)]
|
||||
other = total_addons - sum(r[1] for r in faceted)
|
||||
return total_addons, faceted + [(_('Other'), other)]
|
||||
|
||||
|
||||
def usage_stats(request, compat, app):
|
||||
# Get the list of add-ons for usage stats.
|
||||
redis = redisutils.connections['master']
|
||||
qs = (AppCompat.search().order_by('-usage.%s' % app).values_dict()
|
||||
.filter(**{'support.%s.max__gte' % app: vint(compat['previous'])}))
|
||||
addons = amo.utils.paginate(request, qs)
|
||||
for obj in addons.object_list:
|
||||
obj['usage'] = obj['usage'][app]
|
||||
obj['max_version'] = obj['max_version'][app]
|
||||
total = int(redis.hget('compat:%s' % app, 'total'))
|
||||
return addons, total
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
|
|
@ -107,4 +133,3 @@ def reporter_detail(request, guid):
|
|||
return jingo.render(request, 'compat/reporter_detail.html',
|
||||
dict(reports=reports, works=works,
|
||||
name=name, guid=guid))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
$(function() {
|
||||
var chart = $('#chart'),
|
||||
data = JSON.parse(chart.attr('data-data')),
|
||||
total = chart.attr('data-total'),
|
||||
keys = JSON.parse(chart.attr('data-keys')),
|
||||
series = _.map(data, function(value, key) {
|
||||
return [keys[key], parseInt(value / total * 100)];
|
||||
});
|
||||
console.log(JSON.stringify(series));
|
||||
|
||||
var chart = new Highcharts.Chart({
|
||||
chart: {
|
||||
renderTo: 'chart'
|
||||
},
|
||||
title: '',
|
||||
plotOptions: {
|
||||
pie: { cursor: 'pointer' }
|
||||
},
|
||||
tooltip: {
|
||||
formatter: function() {
|
||||
return '<b>'+ this.point.name +'</b>: '+ this.y +' %';
|
||||
}
|
||||
},
|
||||
series: [{
|
||||
type: 'pie',
|
||||
data: series
|
||||
}]
|
||||
});
|
||||
});
|
||||
12
settings.py
12
settings.py
|
|
@ -965,13 +965,13 @@ MODIFIED_DELAY = 3
|
|||
|
||||
# This is a list of dictionaries that we should generate compat info for.
|
||||
# app: should match amo.FIREFOX.id.
|
||||
# version: the app version we're generating compat info for.
|
||||
# alpha: the first version that should be considered alpha for :version.
|
||||
# previous: the major version before :version.
|
||||
# main: the app version we're generating compat info for.
|
||||
# versions: version numbers to show in comparisons.
|
||||
# previous: the major version before :main.
|
||||
COMPAT = (
|
||||
dict(app=1, version='4.0', alpha='3.7a', previous='3.6'),
|
||||
dict(app=1, version='5.0', alpha='5.0a', previous='4.0'),
|
||||
dict(app=1, version='6.0', alpha='6.0a', previous='5.0'),
|
||||
dict(app=1, main='6.0', versions=('6.0', '6.0a2', '6.0a1'), previous='5.0'),
|
||||
dict(app=1, main='5.0', versions=('5.0', '5.0a2', '5.0a1'), previous='4.0'),
|
||||
dict(app=1, main='4.0', versions=('4.0', '4.0a1', '3.7a'), previous='3.6'),
|
||||
)
|
||||
|
||||
# URL for reporting arecibo errors too. If not set, won't be sent.
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче