add in some apps stats

2012-10-09 12:24:29 -07:00 · 2012-10-09 12:24:29 -07:00 · 919f74db6e
--- a/apps/stats/tasks.py
+++ b/apps/stats/tasks.py
@ -94,7 +94,8 @@ def _get_daily_jobs(date=None):
        date = datetime.date.today()

    extra = dict(where=['DATE(created)=%s'], params=[date])
-
+    # Where we need to specify the extra on some of the joins.
+    addon_extra = dict(where=['DATE(addons.created)=%s'], params=[date])
    # If you're editing these, note that you are returning a function!  This
    # cheesy hackery was done so that we could pass the queries to celery
    # lazily and not hammer the db with a ton of these all at once.
@ -132,6 +133,17 @@ def _get_daily_jobs(date=None):
        'collection_addon_downloads': (lambda:
            AddonCollectionCount.objects.filter(date__lte=date).aggregate(
                sum=Sum('count'))['sum']),
+
+        # Marketplace stats
+        'apps_count_new': (Addon.objects.extra(**addon_extra)
+                .filter(type=amo.ADDON_WEBAPP).count),
+        'apps_count_installed': (Installed.objects.extra(**addon_extra)
+                .filter(addon__type=amo.ADDON_WEBAPP).count),
+
+        # Marketplace reviews
+        'apps_review_count_new': Review.objects.extra(**addon_extra)
+                .filter(editorreview=0, addon__type=amo.ADDON_WEBAPP).count,
+
    }

    # If we're processing today's stats, we'll do some extras.  We don't do
--- a/apps/stats/tests/test_cron.py
+++ b/apps/stats/tests/test_cron.py
@ -7,9 +7,12 @@ from nose.tools import eq_

 import amo.tests
 from addons.models import Addon
+from mkt.webapps.models import Installed
+from reviews.models import Review
 from stats.models import (Contribution, DownloadCount, GlobalStat,
                          UpdateCount)
 from stats import cron, tasks
+from users.models import UserProfile


 class TestGlobalStats(amo.tests.TestCase):
@ -26,6 +29,28 @@ class TestGlobalStats(amo.tests.TestCase):
        eq_(len(GlobalStat.objects.no_cache().filter(date=date,
                                                 name=job)), 1)

+    def test_marketplace_stats(self):
+        res = tasks._get_daily_jobs()
+        for k in ['apps_count_new', 'apps_count_installed',
+                  'apps_review_count_new']:
+            assert k in res, 'Job %s missing from _get_daily_jobs' % k
+
+    def test_app_new(self):
+        Addon.objects.create(type=amo.ADDON_WEBAPP)
+        eq_(tasks._get_daily_jobs()['apps_count_new'](), 1)
+
+    def test_apps_installed(self):
+        addon = Addon.objects.create(type=amo.ADDON_WEBAPP)
+        user = UserProfile.objects.create(username='foo')
+        Installed.objects.create(addon=addon, user=user)
+        eq_(tasks._get_daily_jobs()['apps_count_installed'](), 1)
+
+    def test_app_reviews(self):
+        addon = Addon.objects.create(type=amo.ADDON_WEBAPP)
+        user = UserProfile.objects.create(username='foo')
+        Review.objects.create(addon=addon, user=user)
+        eq_(tasks._get_daily_jobs()['apps_review_count_new'](), 1)
+

 class TestTotalContributions(amo.tests.TestCase):
    fixtures = ['base/apps', 'base/appversion', 'base/users',
--- a/apps/stats/urls.py
+++ b/apps/stats/urls.py
@ -14,6 +14,7 @@ global_series = dict((type, '%s-%s' % (type, series_re))
 collection_series = dict((type, '%s-%s' % (type, series_re))
                         for type in views.COLLECTION_SERIES)

+
 urlpatterns = patterns('',
    url('^$', lambda r: redirect('stats.addons_in_use', permanent=False),
        name='stats.dashboard'),
@ -23,40 +24,29 @@ urlpatterns = patterns('',
    url('^fake-%s' % series_re, views.fake_collection_stats),
    url('^collection/(?P<uuid>[\w-]+).%s$' % (format_re),
        views.collection, name='stats.collection'),
-
-    # global series urls.
-    url(global_series['addons_in_use'], views.site_series,
-        kwargs={'field': 'addons_in_use'}),
-    url(global_series['addons_updated'], views.site_series,
-        kwargs={'field': 'addons_updated'}),
-    url(global_series['addons_downloaded'], views.site_series,
-        kwargs={'field': 'addons_downloaded'}),
-    url(global_series['addons_created'], views.site_series,
-        kwargs={'field': 'addons_created'}),
-    url(global_series['reviews_created'], views.site_series,
-        kwargs={'field': 'reviews_created'}),
-    url(global_series['collections_created'], views.site_series,
-        kwargs={'field': 'collections_created'}),
-    url(global_series['users_created'], views.site_series,
-        kwargs={'field': 'users_created'}),
-
-    # global series urls.
-    url('^addons_in_use/$', views.site_stats_report,
-        kwargs={'report': 'addons_in_use'}, name='stats.addons_in_use'),
-    url('^addons_updated/$', views.site_stats_report,
-        kwargs={'report': 'addons_updated'}, name='stats.addons_updated'),
-    url('^addons_downloaded/$', views.site_stats_report,
-        kwargs={'report': 'addons_downloaded'}, name='stats.addons_downloaded'),
-    url('^addons_created/$', views.site_stats_report,
-        kwargs={'report': 'addons_created'}, name='stats.addons_created'),
-    url('^reviews_created/$', views.site_stats_report,
-        kwargs={'report': 'reviews_created'}, name='stats.reviews_created'),
-    url('^collections_created/$', views.site_stats_report,
-        kwargs={'report': 'collections_created'}, name='stats.collections_created'),
-    url('^users_created/$', views.site_stats_report,
-        kwargs={'report': 'users_created'}, name='stats.users_created'),
 )

+# These are the front end pages, so that when you click the links on the
+# navigation page, you end up on the correct stats page for AMO. There are
+# different keys for marketplace, looking in mkt for that.
+keys = ['addons_in_use', 'addons_updated', 'addons_downloaded',
+        'addons_created', 'collections_created',
+        'reviews_created', 'users_created']
+urls = []
+for key in keys:
+    urls.append(url('^%s/$' % key, views.site_stats_report,
+                name='stats.%s' % key, kwargs={'report': key}))
+
+# These are the URLs that return JSON back to the front end and actually
+# do the SQL query. These means that Marketplace is using the same backend as
+# AMO to actually produce the statistics.
+keys += ['apps_count_new', 'apps_count_installed', 'apps_review_count_new']
+for key in keys:
+    urls.append(url(global_series[key], views.site_series,
+                    kwargs={'field': key}))
+
+urlpatterns += patterns('', *urls)
+
 collection_stats_urls = patterns('',
    url(collection_series['subscribers'], views.collection_series,
        kwargs={'field': 'subscribers'}),
--- a/apps/stats/views.py
+++ b/apps/stats/views.py
@ -40,7 +40,9 @@ SERIES = ('downloads', 'usage', 'contributions', 'overview',
          'sources', 'os', 'locales', 'statuses', 'versions', 'apps')
 COLLECTION_SERIES = ('downloads', 'subscribers', 'ratings')
 GLOBAL_SERIES = ('addons_in_use', 'addons_updated', 'addons_downloaded',
-                 'collections_created', 'reviews_created', 'addons_created',
+                 'apps_count_installed', 'apps_count_new',
+                 'apps_review_count_new', 'collections_created',
+                 'reviews_created', 'addons_created',
                 'users_created')


@ -466,6 +468,9 @@ def _site_query(period, start, end):
        'addon_downloads_new': 'addons_downloaded',
        'addon_total_updatepings': 'addons_in_use',
        'addon_count_new': 'addons_created',
+        'apps_count_new': 'apps_count_new',
+        'apps_count_installed': 'apps_count_installed',
+        'apps_review_count_new': 'apps_review_count_new',
        'version_count_new': 'addons_updated',
        'user_count_new': 'users_created',
        'review_count_new': 'reviews_created',
@ -549,7 +554,7 @@ def site_series(request, format, group, start, end, field):
        series, fields = csv_fields(series)
        return render_csv(request, None, series,
                          ['date', 'count'] + list(fields),
-                          title='addons.mozilla.org week Site Statistics',
+                          title='%s week Site Statistics' % settings.DOMAIN,
                          show_disclaimer=True)
    return render_json(request, None, series)

--- a/media/js/mkt/stats/csv_keys.js
+++ b/media/js/mkt/stats/csv_keys.js
@ -63,7 +63,8 @@ var csv_keys = {
        'category'              : gettext('Category Pages'),
        'collection'            : gettext('Collections'),
        'cb-hc-featured'        : gettext('Category Landing Featured Carousel'),
-        'cb-dl-featured'        : gettext('Category Landing Featured Carousel'),
+        // Duplicate of line 75.
+        //'cb-dl-featured'        : gettext('Category Landing Featured Carousel'),
        'cb-hc-toprated'        : gettext('Category Landing Top Rated'),
        'cb-dl-toprated'        : gettext('Category Landing Top Rated'),
        'cb-hc-mostpopular'     : gettext('Category Landing Most Popular'),
@ -155,6 +156,18 @@ var csv_keys = {
        'source_refunds' : [
            gettext('Total Units Refunded by Source'),
            gettext('Total Units Refunded by Source')
+        ],
+        'apps_count_new': [
+            gettext('Apps added'),
+            gettext('Apps added')
+        ],
+        'apps_count_installed': [
+            gettext('Apps installed'),
+            gettext('Apps installed')
+        ],
+        'apps_review_count_new': [
+            gettext('Reviews'),
+            gettext('Reviews')
        ]
    },
    aggregateLabel: {
--- a/mkt/detail/urls.py
+++ b/mkt/detail/urls.py
@ -3,6 +3,7 @@ from django.conf.urls import include, patterns, url
 from mkt.purchase.urls import app_purchase_patterns
 from mkt.ratings.urls import review_patterns
 from mkt.receipts.urls import app_receipt_patterns
+from mkt.stats.urls import app_stats_patterns
 from . import views


@ -19,7 +20,7 @@ urlpatterns = patterns('',
    ('^purchase/', include(app_receipt_patterns)),

    # Statistics.
-    ('^statistics/', include('mkt.stats.urls')),
+    ('^statistics/', include(app_stats_patterns)),

    # Ratings.
    ('^reviews/', include(review_patterns)),
--- a/mkt/stats/templates/appstats/stats.html
+++ b/mkt/stats/templates/appstats/stats.html
@ -51,51 +51,7 @@
      {% endif %}
    </hgroup>

-    <div class="criteria island">
-      <ul>
-        <li>
-          <a id="chart-zoomout" class="inactive" href="#">
-             {{ _('reset zoom') }}
-          </a>
-        </li>
-      </ul>
-    </div>
-
-    <div class="criteria range island">
-      <ul>
-        <li>{{ _('For last:') }}</li>
-        <li data-range="7 days"
-          {% if range == '7' %}class="selected"{% endif %}>
-          <a class="days-7" href="#">{{ _('7 days') }}</a></li>
-        <li data-range="30 days"
-          {% if range == '30' %}class="selected"{% endif %}>
-          <a class="days-30" href="#">{{ _('30 days') }}</a></li>
-        <li data-range="90 days"
-          {% if range == '90' %}class="selected"{% endif %}>
-          <a href="#">{{ _('90 days') }}</a></li>
-        <li data-range="365 days"
-          {% if range == '365' %}class="selected"{% endif %}>
-          <a href="#">{{ _('365 days') }}</a></li>
-        <li data-range="custom"
-          {% if range == 'custom' %}class="selected"{% endif %}>
-          <a id="custom-date-range" href="#">{{ _('Custom Range') }}</a></li>
-      </ul>
-    </div>
-
-    <div class="criteria group island">
-      <ul>
-        <li>{{ _('Group by:') }}</li>
-        <li data-group="day">
-          <a class="group-day" href="#">{{ _('Day') }}</a>
-        </li>
-        <li data-group="week">
-          <a class="group-week" href="#">{{ _('Week') }}</a>
-        </li>
-        <li data-group="month">
-          <a class="group-month" href="#">{{ _('Month') }}</a>
-        </li>
-      </ul>
-    </div>
+    {% include 'includes/criteria.html' %}

  </header>

--- a/mkt/stats/templates/includes/criteria.html
+++ b/mkt/stats/templates/includes/criteria.html
@ -0,0 +1,46 @@
+<div class="criteria island">
+  <ul>
+    <li>
+      <a id="chart-zoomout" class="inactive" href="#">
+         {{ _('reset zoom') }}
+      </a>
+    </li>
+  </ul>
+</div>
+
+
+<div class="criteria range island">
+  <ul>
+    <li>{{ _('For last:') }}</li>
+    <li data-range="7 days"
+      {% if range == '7' %}class="selected"{% endif %}>
+      <a class="days-7" href="#">{{ _('7 days') }}</a></li>
+    <li data-range="30 days"
+      {% if range == '30' %}class="selected"{% endif %}>
+      <a class="days-30" href="#">{{ _('30 days') }}</a></li>
+    <li data-range="90 days"
+      {% if range == '90' %}class="selected"{% endif %}>
+      <a href="#">{{ _('90 days') }}</a></li>
+    <li data-range="365 days"
+      {% if range == '365' %}class="selected"{% endif %}>
+      <a href="#">{{ _('365 days') }}</a></li>
+    <li data-range="custom"
+      {% if range == 'custom' %}class="selected"{% endif %}>
+      <a id="custom-date-range" href="#">{{ _('Custom Range') }}</a></li>
+  </ul>
+</div>
+
+<div class="criteria group island">
+  <ul>
+    <li>{{ _('Group by:') }}</li>
+    <li data-group="day">
+      <a class="group-day" href="#">{{ _('Day') }}</a>
+    </li>
+    <li data-group="week">
+      <a class="group-week" href="#">{{ _('Week') }}</a>
+    </li>
+    <li data-group="month">
+      <a class="group-month" href="#">{{ _('Month') }}</a>
+    </li>
+  </ul>
+</div>
--- a/mkt/stats/templates/sitestats/includes/stats_nav.html
+++ b/mkt/stats/templates/sitestats/includes/stats_nav.html
@ -0,0 +1,16 @@
+{% set urls = [
+  (url('mkt.stats.apps_count_new'), _('Apps added')),
+  (url('mkt.stats.apps_count_installed'), _('Apps installed')),
+  (url('mkt.stats.apps_review_count_new'), _('Reviews')),
+] %}
+
+<section class="secondary">
+  <div class="report-menu" id="side-nav">
+    <ul>
+      {% for link, title in urls %}
+        <li{% if url_quote(request.path) == link %} class="selected active"{% endif %}>
+          <a href="{{ link }}">{{ title }}</a></li>
+      {% endfor %}
+    </ul
+  </div>
+</section>
--- a/mkt/stats/templates/sitestats/stats.html
+++ b/mkt/stats/templates/sitestats/stats.html
@ -0,0 +1,111 @@
+{% extends 'developers/base_impala.html' %}
+
+{% set range = view.range %}
+
+{% block bodyclass %}developer-hub statistics{% endblock %}
+
+{% block extrahead %}
+  {{ css('mkt/stats') }}
+  <link rel="stylesheet"
+        href="{{ media('css/zamboni/jquery-ui/custom-1.7.2.css') }}">
+{% endblock %}
+
+{% block paypal_js %}{% endblock %}
+
+{% block title %}
+  {# L10n: {0} is the app name #}
+  {{ _('Marketplace Statistics Dashboard') }}
+{% endblock %}
+
+{% block content %}
+<section id="stats">
+  <header class="c">
+    <hgroup class="c">
+      <h1>
+        {{ _('Marketplace Statistics Dashboard') }}
+      </h1>
+    </hgroup>
+
+    {% include 'includes/criteria.html' %}
+
+  </header>
+
+  {% include 'sitestats/includes/stats_nav.html' %}
+
+  <div id="lm" class="loadmessage">
+    <span>{{ _('Loading the latest data&hellip;') }}</span>
+  </div>
+  <div class="primary statistics"
+    data-report="{{ report }}"
+    {% if view.last %}
+    data-range="{{ view.last }}"
+    {% endif %}
+    {% if view.start and view.end %}
+    data-range="custom"
+    data-start_date="{{ view.start }}"
+    data-end_date="{{ view.end }}"
+    {% endif %}
+    data-base_url="{{ stats_base_url }}">
+    <div class="island chart">
+      <div id="head-chart">
+      </div>
+      <div class="no-data-overlay">
+        <p>{{ _('No data available.') }}</p>
+      </div>
+    </div>
+    {% block stats %}
+    {% endblock %}
+    {% block csvtable %}
+      <div class="island">
+        {% block csvtitle %}{% endblock %}
+        <div class="tabular csv-table">
+          <div class="table-box">
+            <table>
+              <thead>
+              </thead>
+            </table>
+          </div>
+          <footer>
+            <nav class="paginator c" role="navigation">
+              <p class="range">
+              </p>
+              <p class="rel">
+                <a href="#"
+                   class="button prev disabled">
+                  &#x25C2; {{ _('Previous') }}</a>
+                <a href="#"
+                   class="button next">
+                  {{ _('Next') }} &#x25B8;</a>
+              </p>
+            </nav>
+          </footer>
+        </div>
+      </div>
+    {% endblock %}
+    <div class="hidden">
+      <div id="fieldMenuPopup" class="popup">
+        <form id="fieldMenu">
+          <ul id="fieldList">
+          </ul>
+        </form>
+      </div>
+    </div>
+  </div>
+  <div id="popup-container">
+    <div class="modal" id="stats-note">
+      <a class="close">{{ _('close') }}</a>
+      {% block stats_note %}{% endblock %}
+    </div>
+    {% include 'stats/popup.html' %}
+  </div>
+</section>
+{% endblock %}
+
+{% block js %}
+  <!--[if IE]>
+    <script
+      src="{{ media('js/lib/excanvas.compiled.js') }}">
+    </script>
+  <![endif]-->
+  {{ js('mkt/stats') }}
+{% endblock %}
--- a/mkt/stats/tests/test_views.py
+++ b/mkt/stats/tests/test_views.py
@ -10,6 +10,7 @@ import amo
 import amo.tests
 from amo.urlresolvers import reverse
 from addons.models import Addon, AddonUser
+from apps.stats.models import GlobalStat
 from market.models import Price
 from mkt.inapp_pay.models import InappConfig, InappPayment
 from mkt.webapps.models import Installed
@ -379,3 +380,28 @@ class TestPadMissingStats(amo.tests.ESTestCase):
        days = [dummy['date'].date() for dummy in dummies]
        for day in expected_days:
            eq_(day in days, True)
+
+
+class TestOverall(amo.tests.TestCase):
+
+    def setUp(self):
+        self.keys = ['apps_count_new', 'apps_count_installed',
+                     'apps_review_count_new']
+
+    def test_url(self):
+        eq_(self.client.get(reverse('mkt.stats.overall')).status_code, 200)
+
+    def get_url(self, name):
+        return (reverse('mkt.stats.%s' % name) +
+                '/%s-day-20090601-20090630.json' % name)
+
+    def test_stats(self):
+        for stat in self.keys:
+            GlobalStat.objects.create(name=stat, count=1,
+                                      date=datetime.date(2009, 06, 12))
+
+        for stat in self.keys:
+            res = self.client.get(self.get_url(stat))
+            content = json.loads(res.content)
+            eq_(content[0]['date'], '2009-06-12')
+            eq_(content[0]['count'], 1)
--- a/mkt/stats/urls.py
+++ b/mkt/stats/urls.py
@ -90,7 +90,7 @@ def sales_series_urls(category='', inapp_flag=False):
    return url_patterns


-urlpatterns = patterns('',
+app_stats_patterns = patterns('',
    # Overview (not implemented).
    url('^$', views.stats_report, name='mkt.stats.overview',
        kwargs={'report': 'installs'}),
@ -109,17 +109,33 @@ urlpatterns = patterns('',
        name='mkt.stats.usage_series'),
 )

-urlpatterns += sales_stats_report_urls(category='currency', inapp_flag=True)
-urlpatterns += sales_series_urls(category='currency', inapp_flag=True)
-urlpatterns += sales_stats_report_urls(category='source', inapp_flag=True)
-urlpatterns += sales_series_urls(category='source', inapp_flag=True)
-urlpatterns += sales_stats_report_urls(inapp_flag=True)
-urlpatterns += sales_series_urls(inapp_flag=True)
+app_stats_patterns += sales_stats_report_urls(category='currency',
+                                           inapp_flag=True)
+app_stats_patterns += sales_series_urls(category='currency', inapp_flag=True)
+app_stats_patterns += sales_stats_report_urls(category='source',
+                                              inapp_flag=True)
+app_stats_patterns += sales_series_urls(category='source', inapp_flag=True)
+app_stats_patterns += sales_stats_report_urls(inapp_flag=True)
+app_stats_patterns += sales_series_urls(inapp_flag=True)

-urlpatterns += sales_stats_report_urls(category='currency')
-urlpatterns += sales_series_urls(category='currency')
-urlpatterns += sales_stats_report_urls(category='source')
-urlpatterns += sales_series_urls(category='source')
+app_stats_patterns += sales_stats_report_urls(category='currency')
+app_stats_patterns += sales_series_urls(category='currency')
+app_stats_patterns += sales_stats_report_urls(category='source')
+app_stats_patterns += sales_series_urls(category='source')

-urlpatterns += sales_stats_report_urls()
-urlpatterns += sales_series_urls()
+app_stats_patterns += sales_stats_report_urls()
+app_stats_patterns += sales_series_urls()
+
+# Overall site statistics.
+app_site_patterns = patterns('',
+    url('^$', views.overall, name='mkt.stats.overall',
+        kwargs={'report': 'overview'}),
+)
+
+keys = ['apps_count_new', 'apps_count_installed', 'apps_review_count_new']
+urls = []
+for key in keys:
+    urls.append(url('^%s/$' % key, views.overall,
+                name='mkt.stats.%s' % key, kwargs={'report': key}))
+
+app_site_patterns += patterns('', *urls)
--- a/mkt/stats/views.py
+++ b/mkt/stats/views.py
@ -411,3 +411,9 @@ def fake_app_stats(request, addon, group, start, end, format):
        }})
        val += .01
    return faked
+
+
+def overall(request, report):
+    view = get_report_view(request)
+    return jingo.render(request, 'sitestats/stats.html', {'report': report,
+                                                          'view': view})
--- a/mkt/urls.py
+++ b/mkt/urls.py
@ -14,6 +14,7 @@ from mkt.account.urls import (purchases_patterns, settings_patterns,
 from mkt.developers.views import login
 from mkt.purchase.urls import bluevia_services_patterns
 from mkt.ratings.urls import theme_review_patterns
+from mkt.stats.urls import app_site_patterns
 from mkt.themes.urls import theme_patterns


@ -60,13 +61,15 @@ urlpatterns = patterns('',
    url('^statistics/events-(?P<start>\d{8})-(?P<end>\d{8}).json$',
        'stats.views.site_events', name='amo.site_events'),

-    # Disable currently not working statistics.
-    # Site statistics that we are going to catch, the rest will fall through.
-    #url('^statistics/', include('stats.urls')),
+    # Catch marketplace specific statistics urls.
+    url('^statistics/', include(app_site_patterns)),
+
+    # Let the rest of the URLs fall through.
+    url('^statistics/', include('stats.urls')),

    # Disable currently not working statistics.
    # Fall through for any URLs not matched above stats dashboard.
-    #url('^statistics/', lambda r: redirect('/'), name='statistics.dashboard'),
+    url('^statistics/', lambda r: redirect('/'), name='statistics.dashboard'),

    # Support (e.g., refunds, FAQs).
    ('^support/', include('mkt.support.urls')),