redoing compat reports with es (bug 656527)

apps/compat/cron.py

@@ -1,62 +1,63 @@
+import collections
 import logging
 
 from django.conf import settings
 from django.db.models import Sum, Max
 
 import cronjobs
+import elasticutils
 import redisutils
 
 import amo
+import amo.utils
 import versions.compare as vc
 from addons.models import Addon
 from stats.models import UpdateCount
 
+from .models import AppCompat
+
 log = logging.getLogger('z.compat')
 
 
 @cronjobs.register
 def compatibility_report():
     redis = redisutils.connections['master']
+    docs = collections.defaultdict(dict)
 
-    # for app in amo.APP_USAGE:
+    # Gather all the data for the index.
-    for compat in settings.COMPAT:
+    for app in amo.APP_USAGE:
-        app = amo.APPS_ALL[compat['app']]
+        log.info(u'Making compat report for %s.' % app.pretty)
-        version = compat['version']
-        log.info(u'Making compat report for %s %s.' % (app.pretty, version))
-        versions = (('latest', version), ('beta', version + 'b'),
-                    ('alpha', compat['alpha']))
-
-        rv = dict((k, 0) for k in dict(versions))
-        rv['other'] = 0
-
-        ignore = (amo.STATUS_NULL, amo.STATUS_DISABLED)
-        qs = (Addon.objects.exclude(type=amo.ADDON_PERSONA, status__in=ignore)
-              .filter(appsupport__app=app.id, name__locale='en-us'))
-
         latest = UpdateCount.objects.aggregate(d=Max('date'))['d']
         qs = UpdateCount.objects.filter(addon__appsupport__app=app.id,
+                                        addon__status__in=amo.VALID_STATUSES,
+                                        addon___current_version__isnull=False,
                                         date=latest)
         total = qs.aggregate(Sum('count'))['count__sum']
-        addons = list(qs.values_list('addon', 'count', 'addon__appsupport__min',
+        redis.hset('compat:%s' % app.id, 'total', total)
-                                     'addon__appsupport__max'))
-
-        # Count up the top 95% of addons by ADU.
         adus = 0
-        for addon, count, minver, maxver in addons:
+
-            # Don't count add-ons that weren't compatible with the previous
+        updates = dict(qs.values_list('addon', 'count'))
-            # release
+        for chunk in amo.utils.chunked(updates.items(), 50):
-            if maxver < vc.version_int(compat['previous']):
+            chunk = dict(chunk)
-                continue
+            for addon in Addon.objects.filter(id__in=chunk):
-            if adus < .95 * total:
+                doc = docs[addon.id]
-                adus += count
+                doc.update(id=addon.id, slug=addon.slug,
-            else:
+                           name=unicode(addon.name))
-                break
+                doc.setdefault('usage', {})[app.id] = updates[addon.id]
-            for key, version in versions:
+
-                if minver <= vc.version_int(version) <= maxver:
+                if app not in addon.compatible_apps:
-                    rv[key] += 1
+                    continue
-                    break
+                compat = addon.compatible_apps[app]
-            else:
+                d = {'min': compat.min.version_int,
-                rv['other'] += 1
+                     'max': compat.max.version_int}
-        log.info(u'Compat for %s %s: %s' % (app.pretty, version, rv))
+                doc.setdefault('support', {})[app.id] = d
-        key = '%s:%s' % (app.id, version)
+                doc.setdefault('max_version', {})[app.id] = compat.max.version
-        redis.hmset('compat:' + key, rv)
+                doc['top_95'] = adus > .95 * total
+
+            adus += sum(chunk.values())
+
+    # Send it all to the index.
+    for chunk in amo.utils.chunked(docs.values(), 150):
+        for doc in chunk:
+            AppCompat.index(doc, id=doc['id'], bulk=True)
+        elasticutils.get_es().flush_bulk(forced=True)

apps/compat/models.py

@@ -19,3 +19,20 @@ class CompatReport(amo.models.ModelBase):
 
     class Meta:
         db_table = 'compatibility_reports'
+
+
+class AppCompat(amo.models.ModelBase):
+    """
+    Stub model for use with search. The schema:
+
+        {id: addon.id,
+         name: addon.name,
+         slug: addon.slug,
+         max_version: {APP.id: version string},
+         usage: {APP.id: addon.daily_usage},
+         support: {APP.id: {max: version int, min: version int},
+        }
+    """
+
+    class Meta:
+        abstract = True

apps/compat/templates/compat/details.html

@@ -0,0 +1,11 @@
+{% extends "base.html" %}
+
+{% set title = _('{app} {version} Add-on Compatibility Report')|f(app=request.APP.pretty, version=version) %}
+{% block title %}{{ page_title(title) }}{% endblock %}
+{% block bodyclass %}inverse{% endblock %}
+
+{% block content %}
+<h1>{{ title }}</h1>
+
+
+{% endblock %}

apps/compat/templates/compat/index.html

@@ -1,44 +1,62 @@
 {% extends "base.html" %}
 
-{% block title %}{{ page_title(_('Compatibility Center')) }}{% endblock %}
+{% set app = request.APP.pretty %}
+{% set title = _('Add-on Compatibility Report for {app} {version}')|f(app=app, version=version) %}
+{% block title %}{{ page_title(title) }}{% endblock %}
 {% block bodyclass %}inverse{% endblock %}
 
-{% macro percent(x, y) %}{{ (x / y|float * 100)|int }}{% endmacro %}
-
-{% set app = request.APP.pretty %}
-{% set num = percent(versions['latest'], total) %}
-
 {% block content %}
+<header>
+  <h2>{{ title }}</h2>
+  {{ breadcrumbs([(None, _('{app} {version} Compatibility')|f(app=app, version=version))]) }}
+</header>
 
-<h1>{{ _('{app} {version} Add-on Compatibility Report')|f(app=app, version=version) }}</h1>
+{% set titles = {
-<p>{% trans %}
+  'prev': _('Top 95% compatible with previous version'),
-<b>{{ num }}%</b> of add-ons on addons.mozilla.org are compatible with {{ app }} {{ version }}.
+  'top_95': _('Top 95% of all add-ons'),
-{% endtrans %}</p>
+  'all': _('All active add-ons'),
+} %}
 
-<div id="chart" class="primary"
+<table>
-  data-keys="{{ dict(keys)|json }}"
+  <thead>
-  data-data="{{ versions|json }}"
-  data-total="{{ total }}"></div>
-<div class="secondary">
-  <table id="compat">
     <tr>
-      <th>{{ _('Version') }}</th>
+      <th></th>
-      {# L10n: "#" means "number" #}
+      {% with key, (total, facets) = compat_levels[0] %}
-      <th>{{ _('# of Add-ons') }}</th>
+        {% for version, _ in facets %}
+          <th>{{ version }}</th>
+        {% endfor %}
+      {% endwith %}
     </tr>
-    {% for key, title in keys %}
+  </thead>
+  <tbody>
+    {% for key, (total, facets) in compat_levels %}
+      {% if total %}
       <tr>
-        <td>{{ title }}</td>
+        <th>{{ titles[key] }}</th>
-        <td>{{ versions[key]|numberfmt }}</td>
+        {% for version, count in facets %}
+          <td>{{ (100 * count / total)|round(2) }}% ({{ count }})</td>
+        {% endfor %}
       </tr>
+      {% endif %}
     {% endfor %}
-  </table>
+  </tbody>
-  <p><a href="{{ url('compat.details', version) }}">{{ _('Detailed Report') }}</a></p>
+</table>
-</div>
+<ul>
+</ul>
+
+<h3>{{ _('Details for add-ons compatible with previous version:') }}</h3>
+<table>
+  <tbody>
+    {% for addon in usage_addons.object_list %}
+    <tr>
+      <td><a href="{{ url('addons.detail', addon.slug) }}">{{ addon.name }}</a></td>
+      <td>{{ addon.max_version }}</td>
+      <td>{{ addon.usage|numberfmt }} users </td>
+      <td>({{ (100 * addon.usage / usage_total)|round(2) }}%)</td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+{{ usage_addons|paginator }}
 
 {% endblock %}
-
-{% block js %}
-<script src="{{ media('js/lib/highcharts.src.js') }}"></script>
-<script src="{{ media('js/zamboni/compat.js') }}"></script>
-{% endblock %}

apps/compat/urls.py

@@ -10,5 +10,4 @@ urlpatterns = (
         views.reporter_detail, name='compat.reporter_detail'),
 
     url('^(?P<version>[.\w]+)?$', views.index, name='compat.index'),
-    url('^(?P<version>[.\w]+)/details$', views.details, name='compat.details'),
 )

apps/compat/views.py

@@ -4,47 +4,73 @@ import re
 from django import http
 from django.conf import settings
 from django.db.models import Count
-from django.shortcuts import redirect, get_object_or_404
+from django.shortcuts import redirect
 from django.views.decorators.csrf import csrf_exempt
 
 import jingo
 import redisutils
-from tower import ugettext_lazy as _lazy
+from tower import ugettext as _
 
 import amo.utils
 from amo.decorators import post_required
 from addons.models import Addon
-from .models import CompatReport
+from versions.compare import version_int as vint
-
+from .models import CompatReport, AppCompat
-KEYS = (
-    ('latest', _lazy('Latest')),
-    ('beta', _lazy('Beta')),
-    ('alpha', _lazy('Alpha')),
-    ('other', _lazy('Other')),
-)
 
 
 def index(request, version=None):
     COMPAT = [v for v in settings.COMPAT if v['app'] == request.APP.id]
-    if version is None and COMPAT:
+    compat_dict = dict((v['main'], v) for v in COMPAT)
-        version = COMPAT[0]['version']
+    if not COMPAT:
-
-    redis = redisutils.connections['master']
-    compat = redis.hgetall('compat:%s:%s' % (request.APP.id, version))
-    versions = dict((k, int(v)) for k, v in compat.items())
-
-    if version not in [v['version'] for v in COMPAT] or not versions:
         raise http.Http404()
+    if version not in compat_dict:
+        return redirect('compat.index', COMPAT[0]['main'])
 
-    total = sum(versions.values())
+    compat, app = compat_dict[version], str(request.APP.id)
-    keys = [(k, unicode(v)) for k, v in KEYS]
+    qs = AppCompat.search()
+    compat_queries = (
+        ('prev', qs.query(top_95=True, **{
+            'support.%s.max__gte' % app: vint(compat['previous'])})),
+        ('top_95', qs.query(top_95=True)),
+        ('all', qs),
+    )
+    compat_levels = [(key, version_compat(qs, compat, app))
+                     for key, qs in compat_queries]
+    usage_addons, usage_total = usage_stats(request, compat, app)
     return jingo.render(request, 'compat/index.html',
-                        {'versions': versions, 'total': total,
+                        {'version': version,
-                         'version': version, 'keys': keys})
+                         'usage_addons': usage_addons,
+                         'usage_total': usage_total,
+                         'compat_levels': compat_levels})
 
 
-def details(request, version):
+def version_compat(qs, compat, app):
-    return http.HttpResponse('go away')
+    facets = []
+    for v, prev in zip(compat['versions'], (None,) + compat['versions']):
+        d = {'from': vint(v)}
+        if prev:
+            d['to'] = vint(prev)
+        facets.append(d)
+    qs = qs.facet(by_status={'range': {'support.%s.max' % app: facets}})
+    result = qs[:0].raw()
+    total_addons = result['hits']['total']
+    ranges = result['facets']['by_status']['ranges']
+    faceted = [(v, r['count']) for v, r in zip(compat['versions'], ranges)]
+    other = total_addons - sum(r[1] for r in faceted)
+    return total_addons, faceted + [(_('Other'), other)]
+
+
+def usage_stats(request, compat, app):
+    # Get the list of add-ons for usage stats.
+    redis = redisutils.connections['master']
+    qs = (AppCompat.search().order_by('-usage.%s' % app).values_dict()
+          .filter(**{'support.%s.max__gte' % app: vint(compat['previous'])}))
+    addons = amo.utils.paginate(request, qs)
+    for obj in addons.object_list:
+        obj['usage'] = obj['usage'][app]
+        obj['max_version'] = obj['max_version'][app]
+    total = int(redis.hget('compat:%s' % app, 'total'))
+    return addons, total
 
 
 @csrf_exempt
@@ -107,4 +133,3 @@ def reporter_detail(request, guid):
     return jingo.render(request, 'compat/reporter_detail.html',
                         dict(reports=reports, works=works,
                              name=name, guid=guid))
-

media/js/zamboni/compat.js

@@ -1,29 +0,0 @@
-$(function() {
-    var chart = $('#chart'),
-        data = JSON.parse(chart.attr('data-data')),
-        total = chart.attr('data-total'),
-        keys = JSON.parse(chart.attr('data-keys')),
-        series = _.map(data, function(value, key) {
-            return [keys[key], parseInt(value / total * 100)];
-        });
-console.log(JSON.stringify(series));
-
-    var chart = new Highcharts.Chart({
-        chart: {
-            renderTo: 'chart'
-        },
-        title: '',
-        plotOptions: {
-            pie: { cursor: 'pointer' }
-        },
-        tooltip: {
-            formatter: function() {
-                return '<b>'+ this.point.name +'</b>: '+ this.y +' %';
-            }
-        },
-        series: [{
-            type: 'pie',
-            data: series
-        }]
-    });
-});

settings.py

@@ -965,13 +965,13 @@ MODIFIED_DELAY = 3
 
 # This is a list of dictionaries that we should generate compat info for.
 # app: should match amo.FIREFOX.id.
-# version: the app version we're generating compat info for.
+# main: the app version we're generating compat info for.
-# alpha: the first version that should be considered alpha for :version.
+# versions: version numbers to show in comparisons.
-# previous: the major version before :version.
+# previous: the major version before :main.
 COMPAT = (
-    dict(app=1, version='4.0', alpha='3.7a', previous='3.6'),
+    dict(app=1, main='6.0', versions=('6.0', '6.0a2', '6.0a1'), previous='5.0'),
-    dict(app=1, version='5.0', alpha='5.0a', previous='4.0'),
+    dict(app=1, main='5.0', versions=('5.0', '5.0a2', '5.0a1'), previous='4.0'),
-    dict(app=1, version='6.0', alpha='6.0a', previous='5.0'),
+    dict(app=1, main='4.0', versions=('4.0', '4.0a1', '3.7a'), previous='3.6'),
 )
 
 # URL for reporting arecibo errors too. If not set, won't be sent.