calculate top 95% for app/version combos (bug 675810)

apps/compat/cron.py

@@ -1,7 +1,8 @@
-import collections
 import logging
+from collections import defaultdict
 
-from django.db.models import Sum, Max
+from django.conf import settings
+from django.db.models import Max
 
 import cronjobs
 import elasticutils
@@ -11,6 +12,7 @@ import amo
 import amo.utils
 from addons.models import Addon
 from stats.models import UpdateCount
+from versions.compare import version_int as vint
 
 from .models import AppCompat
 
@@ -20,7 +22,7 @@ log = logging.getLogger('z.compat')
 @cronjobs.register
 def compatibility_report():
     redis = redisutils.connections['master']
-    docs = collections.defaultdict(dict)
+    docs = defaultdict(dict)
 
     # Gather all the data for the index.
     for app in amo.APP_USAGE:
@@ -30,10 +32,7 @@ def compatibility_report():
                                         addon__disabled_by_user=False,
                                         addon__status__in=amo.VALID_STATUSES,
                                         addon___current_version__isnull=False,
-                                        date=latest).order_by('-count')
-        total = qs.aggregate(Sum('count'))['count__sum']
-        redis.hset('compat:%s' % app.id, 'total', total)
-        adus = 0
+                                        date=latest)
 
         updates = dict(qs.values_list('addon', 'count'))
         for chunk in amo.utils.chunked(updates.items(), 50):
@@ -42,6 +41,10 @@ def compatibility_report():
                 doc = docs[addon.id]
                 doc.update(id=addon.id, slug=addon.slug, binary=addon.binary,
                            name=unicode(addon.name), created=addon.created)
+                doc['count'] = chunk[addon.id]
+                doc.setdefault('top_95',
+                               defaultdict(lambda: defaultdict(dict)))
+                doc.setdefault('top_95_all', {})
                 doc.setdefault('usage', {})[app.id] = updates[addon.id]
 
                 if app not in addon.compatible_apps:
@@ -51,9 +54,35 @@ def compatibility_report():
                      'max': compat.max.version_int}
                 doc.setdefault('support', {})[app.id] = d
                 doc.setdefault('max_version', {})[app.id] = compat.max.version
-                doc['top_95'] = adus > .95 * total
 
-            adus += sum(chunk.values())
+        total = sum(updates.values())
+        # Remember the total so we can show % of usage later.
+        redis.hset('compat:%s' % app.id, 'total', total)
+
+        # Figure out which add-ons are in the top 95% for this app.
+        running_total = 0
+        for addon, count in sorted(updates.items(), key=lambda x: x[1]):
+            running_total += count
+            if 'top_95_all' not in docs[addon]:
+                print docs[addon]
+            docs[addon]['top_95_all'][app.id] = running_total < (.95 * total)
+
+    # Mark the top 95% of add-ons compatible with the previous version for each
+    # app + version combo.
+    for compat in settings.COMPAT:
+        app, ver = compat['app'], vint(compat['previous'])
+        # Find all the docs that have a max_version compatible with ver.
+        supported = [doc for doc in docs.values()
+                     if app in doc.get('support', {})
+                        and doc['support'][app]['max'] >= ver]
+        # Sort by count so we can get the top 95% most-used add-ons.
+        supported = sorted(supported, key=lambda d: d['count'])
+        total = sum(doc['count'] for doc in supported)
+        # Figure out which add-ons are in the top 95% for this app + version.
+        running_total = 0
+        for doc in supported:
+            running_total += doc['count']
+            doc['top_95'][app][ver] = running_total < (.95 * total)
 
     # Send it all to the index.
     for chunk in amo.utils.chunked(docs.values(), 150):

apps/compat/models.py

@@ -29,6 +29,9 @@ class AppCompat(amo.models.ModelBase):
          name: addon.name,
          slug: addon.slug,
          binary: addon.binary,
+         count: total # of update counts,
+         top_95_all: {APP.id: bool},
+         top_95: {APP.id: {version int: bool}},
          max_version: {APP.id: version string},
          usage: {APP.id: addon.daily_usage},
          support: {APP.id: {max: version int, min: version int},

apps/compat/views.py

@@ -31,7 +31,7 @@ def index(request, version=None):
     qs = AppCompat.search()
     binary = None
 
-    initial= {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'}
+    initial = {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'}
     initial.update(request.GET.items())
     form = CompatForm(initial)
     if request.GET and form.is_valid():
@@ -48,9 +48,10 @@ def index(request, version=None):
 
     compat, app = compat_dict[version], str(request.APP.id)
     compat_queries = (
-        ('prev', qs.query(top_95=True, **{
+        ('prev', qs.query(**{
+            'top_95.%s.%s' % (app, vint(compat['previous'])): True,
             'support.%s.max__gte' % app: vint(compat['previous'])})),
-        ('top_95', qs.query(top_95=True)),
+        ('top_95', qs.query(**{'top_95_all.%s' % app: True})),
         ('all', qs),
     )
     compat_levels = [(key, version_compat(qs, compat, app, binary))