Revert "move collection stats to es (bug 722556)"

This reverts commit a75edd3952.
2012-02-27 17:11:20 -08:00 · 2012-02-27 17:11:20 -08:00 · 5ae1928c60
--- a/apps/stats/management/commands/index_stats.py
+++ b/apps/stats/management/commands/index_stats.py
@ -8,9 +8,8 @@ from django.db.models import Max, Min
 from celery.task.sets import TaskSet

 from amo.utils import chunked
-from stats.models import CollectionCount, DownloadCount, UpdateCount
-from stats.tasks import (index_collection_counts, index_download_counts,
-                         index_update_counts)
+from stats.models import UpdateCount, DownloadCount
+from stats.tasks import index_update_counts, index_download_counts

 log = logging.getLogger('z.stats')

@ -55,11 +54,6 @@ class Command(BaseCommand):
        queries = [(UpdateCount.objects, index_update_counts),
                   (DownloadCount.objects, index_download_counts)]

-        if not addons:
-            # We can't filter this by addons, so if that is specified,
-            # we'll skip that.
-            queries.append((CollectionCount.objects, index_collection_counts))
-
        for qs, task in queries:
            qs = qs.order_by('-date').values_list('id', flat=True)
            if addons:
@ -78,10 +72,6 @@ class Command(BaseCommand):
                limits = (qs.model.objects.filter(date__isnull=False)
                          .extra(where=['date <> "0000-00-00"'])
                          .aggregate(min=Min('date'), max=Max('date')))
-                # If there isn't any data at all, skip over.
-                if not (limits['max'] or limits['min']):
-                    continue
-
                num_days = (limits['max'] - limits['min']).days
                today = date.today()
                for start in range(0, num_days, STEP):
--- a/apps/stats/models.py
+++ b/apps/stats/models.py
@ -31,7 +31,7 @@ class AddonCollectionCount(models.Model):
        db_table = 'stats_addons_collections_counts'


-class CollectionCount(SearchMixin, models.Model):
+class CollectionCount(models.Model):
    collection = models.ForeignKey('bandwagon.Collection')
    count = models.PositiveIntegerField()
    date = models.DateField()
--- a/apps/stats/search.py
+++ b/apps/stats/search.py
@ -81,20 +81,6 @@ def extract_download_count(dl):
            'id': dl.id}


-def extract_addon_collection(collection_count, addon_collections,
-                             collection_stats):
-    addon_collection_count = sum([c.count for c in addon_collections])
-    collection_stats = dict([[c.name, c.count] for c in collection_stats])
-    return {'date': collection_count.date,
-            'count': collection_count.count,
-            'data': {
-                'downloads': addon_collection_count,
-                'votes_up': collection_stats.get('new_votes_up', 0),
-                'votes_down': collection_stats.get('new_votes_down', 0),
-                'subscribers': collection_stats.get('new_subscribers', 0),
-            }}
-
-
 def get_all_app_versions():
    vals = AppVersion.objects.values_list('application', 'version')
    rv = collections.defaultdict(list)
--- a/apps/stats/tasks.py
+++ b/apps/stats/tasks.py
@ -14,9 +14,7 @@ from stats.models import Contribution
 from reviews.models import Review
 from users.models import UserProfile
 from versions.models import Version
-from .models import (AddonCollectionCount, CollectionCount, CollectionStats,
-                     DownloadCount, UpdateCount)
-
+from .models import UpdateCount, DownloadCount, AddonCollectionCount
 from . import search

 log = commonware.log.getLogger('z.task')
@ -222,25 +220,3 @@ def index_download_counts(ids, **kw):
    except Exception, exc:
        index_download_counts.retry(args=[ids], exc=exc)
        raise
-
-
-def index_collection_counts(ids, **kw):
-    es = elasticutils.get_es()
-    qs = CollectionCount.objects.filter(id__in=ids)
-    if qs:
-        log.info('Indexing %s addon collection counts: %s'
-                 % (len(qs), qs[0].date))
-    try:
-        for collection_count in qs:
-            collection = collection_count.collection_id
-            key = '%s-%s' % (collection, collection_count.date)
-            filters = dict(collection=collection,
-                           date=collection_count.date)
-            data = search.extract_addon_collection(collection_count,
-                            AddonCollectionCount.objects.filter(**filters),
-                            CollectionStats.objects.filter(**filters))
-            CollectionCount.index(data, bulk=True, id=key)
-        es.flush_bulk(forced=True)
-    except Exception, exc:
-        index_collection_counts.retry(args=[ids], exc=exc)
-        raise
--- a/apps/stats/tests/test_cron.py
+++ b/apps/stats/tests/test_cron.py
@ -7,8 +7,7 @@ from nose.tools import eq_

 import amo.tests
 from addons.models import Addon
-from stats.models import (CollectionCount, DownloadCount, UpdateCount,
-                          GlobalStat, Contribution)
+from stats.models import DownloadCount, UpdateCount, GlobalStat, Contribution
 from stats import tasks, cron


@ -64,25 +63,13 @@ class TestIndexStats(amo.tests.TestCase):
    def test_by_date(self, tasks_mock):
        call_command('index_stats', addons=None, date='2009-06-01')
        qs = self.downloads.filter(date='2009-06-01')
-        download = tasks_mock.call_args_list[1][0]
-        eq_(download[0], tasks.index_download_counts)
-        eq_(download[1], list(qs))
-
-    def test_called_three(self, tasks_mock):
-        call_command('index_stats', addons=None, date='2009-06-01')
-        eq_(tasks_mock.call_count, 3)
-
-    def test_called_two(self, tasks_mock):
-        call_command('index_stats', addons='5', date='2009-06-01')
-        eq_(tasks_mock.call_count, 2)
+        tasks_mock.assert_called_with(tasks.index_download_counts, list(qs))

    def test_by_date_range(self, tasks_mock):
        call_command('index_stats', addons=None,
                     date='2009-06-01:2009-06-07')
        qs = self.downloads.filter(date__range=('2009-06-01', '2009-06-07'))
-        download = tasks_mock.call_args_list[1][0]
-        eq_(download[0], tasks.index_download_counts)
-        eq_(download[1], list(qs))
+        tasks_mock.assert_called_with(tasks.index_download_counts, list(qs))

    def test_by_addon(self, tasks_mock):
        call_command('index_stats', addons='5', date=None)
--- a/apps/stats/tests/test_views.py
+++ b/apps/stats/tests/test_views.py
@ -8,11 +8,8 @@ from nose.tools import eq_
 import amo.tests
 from amo.urlresolvers import reverse
 from access.models import Group, GroupUser
-from bandwagon.models import Collection
 from stats import views, tasks
-from stats.search import es_dict
-from stats.models import (CollectionCount, DownloadCount, GlobalStat,
-                          UpdateCount)
+from stats.models import DownloadCount, GlobalStat, UpdateCount
 from users.models import UserProfile


@ -721,56 +718,3 @@ class TestSite(amo.tests.TestCase):
        content = json.loads(res.content)
        eq_(len(content), 14)
        eq_(content[0]['addons_created'], 14)
-
-
-class TestCollections(amo.tests.ESTestCase):
-    fixtures = ['bandwagon/test_models', 'base/users',
-                'base/addon_3615', 'base/addon_5369']
-
-    def setUp(self):
-        super(TestCollections, self).setUp()
-        self.today = datetime.date.today()
-        self.collection = Collection.objects.get(pk=512)
-        self.url = reverse('stats.collection',
-                           args=[self.collection.uuid, 'json'])
-
-        for x in xrange(1, 4):
-            data = {'date': self.today - datetime.timedelta(days=x - 1),
-                    'id': int(self.collection.pk), 'count': x,
-                    'data': es_dict({'subscribers': x, 'votes_up': x,
-                                     'votes_down': x, 'downloads': x})}
-            CollectionCount.index(data, id='%s-%s' % (x, self.collection.pk))
-
-        self.refresh('stats_collections_counts')
-
-    def tests_collection_anon(self):
-        res = self.client.get(self.url)
-        eq_(res.status_code, 403)
-
-    def tests_collection_user(self):
-        self.client.login(username='admin@mozilla.com', password='password')
-        res = self.client.get(self.url)
-        eq_(res.status_code, 200)
-
-    def tests_collection_admin(self):
-        self.client.login(username='admin@mozilla.com', password='password')
-        self.collection.update(author=None)
-        res = self.client.get(self.url)
-        eq_(res.status_code, 200)
-
-    def test_collection_json(self):
-        self.client.login(username='admin@mozilla.com', password='password')
-        res = self.client.get(self.url)
-        content = json.loads(res.content)
-        eq_(len(content), 3)
-        eq_(content[0]['count'], 1)
-        eq_(content[0]['data']['votes_down'], 1)
-        eq_(content[0]['data']['downloads'], 1)
-
-    def test_collection_csv(self):
-        self.client.login(username='admin@mozilla.com', password='password')
-        self.url = reverse('stats.collection',
-                           args=[self.collection.uuid, 'csv'])
-        res = self.client.get(self.url)
-        date = (self.today.strftime('%Y-%m-%d'))
-        assert '%s,1,1,1,1,1' % date in res.content
--- a/apps/stats/urls.py
+++ b/apps/stats/urls.py
@ -13,9 +13,7 @@ series = dict((type, '%s-%s' % (type, series_re)) for type in views.SERIES)
 urlpatterns = patterns('',
    url('^site%s/%s$' % (format_re, group_date_re),
        views.site, name='stats.site'),
-    url('^site-%s' % series_re, views.site, name='stats.site.new'),
-    url('^collection/(?P<uuid>[\w-]+).%s$' % (format_re),
-        views.collection, name='stats.collection')
+    url('^site-%s' % series_re, views.site, name='stats.site.new')
 )

 # Addon specific stats.
--- a/apps/stats/views.py
+++ b/apps/stats/views.py
@ -13,7 +13,6 @@ from django.utils.cache import add_never_cache_headers, patch_cache_control
 from django.utils.datastructures import SortedDict
 from django.core.serializers.json import DjangoJSONEncoder
 from django.core.exceptions import PermissionDenied
-from django.shortcuts import get_object_or_404

 import jingo
 from product_details import product_details
@ -21,7 +20,6 @@ from product_details import product_details
 from access import acl
 from addons.decorators import addon_view, addon_view_factory
 from addons.models import Addon
-from bandwagon.models import Collection
 from zadmin.models import SiteEvent

 import amo
@ -30,7 +28,8 @@ from amo.urlresolvers import reverse
 from amo.utils import memoize

 from .decorators import allow_cross_site_request
-from .models import CollectionCount, Contribution, DownloadCount, UpdateCount
+from .models import DownloadCount, UpdateCount, Contribution
+

 SERIES_GROUPS = ('day', 'week', 'month')
 SERIES_GROUPS_DATE = ('date', 'week', 'month')  # Backwards compat.
@ -449,29 +448,6 @@ def site(request, format, group, start=None, end=None):
    return render_json(request, None, series)


-def collection(request, uuid, format):
-    """
-    Collection data taken from the stats_collections and the
-    stats_addons_collections_counts table.
-    """
-    collection = get_object_or_404(Collection, uuid=uuid)
-    if (not acl.action_allowed(request, 'CollectionStats', 'View') and
-        not (request.amo_user and collection.author and
-             collection.author.id == request.amo_user.pk)):
-        return http.HttpResponseForbidden()
-
-    start = date.today() - timedelta(days=365)
-    end = date.today()
-    series = get_series(CollectionCount, id=int(collection.pk),
-                        date__range=(start, end), extra_field='data')
-
-    if format == 'csv':
-        series, fields = csv_fields(series)
-        return render_csv(request, collection, series,
-                          ['date', 'count'] + list(fields))
-    return render_json(request, collection, series)
-
-
 def fudge_headers(response, stats):
    """Alter cache headers. Don't cache content where data could be missing."""
    if not stats:
--- a/lib/settings_base.py
+++ b/lib/settings_base.py
@ -1283,8 +1283,7 @@ BUILDER_VERSIONS_URL = ('https://builder.addons.mozilla.org/repackage/' +
 ES_HOSTS = ['127.0.0.1:9200']
 ES_INDEXES = {'default': 'amo',
              'update_counts': 'amo_stats',
-              'download_counts': 'amo_stats',
-              'stats_collections_counts': 'amo_stats'}
+              'download_counts': 'amo_stats'}
 ES_TIMEOUT = 5

 # Default AMO user id to use for tasks.