Bug 1116511: Add view for serving JSON contributor data.

bedrock/mozorg/models.py

@@ -6,6 +6,19 @@ from picklefield import PickledObjectField
 from django_extensions.db.fields import ModificationDateTimeField
 
 
+CONTRIBUTOR_SOURCE_NAMES = {
+    'all': None,
+    'sumo': 'team',
+    'reps': 'team',
+    'qa': 'team',
+    'firefox': 'team',
+    'firefoxos': 'team',
+    'firefoxforandroid': 'team',
+    'bugzilla': 'source',
+    'github': 'source',
+}
+
+
 class TwitterCacheManager(models.Manager):
     def get_tweets_for(self, account):
         cache_key = 'tweets-for-' + str(account)
@@ -33,6 +46,22 @@ class TwitterCache(models.Model):
         return u'Tweets from @' + self.account
 
 
+class ContributorActivityManager(models.Manager):
+    def group_by_date_and_source(self, source):
+        try:
+            source_type = CONTRIBUTOR_SOURCE_NAMES[source]
+        except KeyError:
+            raise ContributorActivity.DoesNotExist
+
+        qs = self.values('date')
+        if source_type is not None:
+            field_name = source_type + '_name'
+            qs = qs.filter(**{field_name: source})
+
+        # dates are grouped in weeks. 52 results gives us a year.
+        return qs.annotate(models.Sum('total'), models.Sum('new'))[:52]
+
+
 class ContributorActivity(models.Model):
     date = models.DateField()
     source_name = models.CharField(max_length=100)
@@ -40,6 +69,8 @@ class ContributorActivity(models.Model):
     total = models.IntegerField()
     new = models.IntegerField()
 
+    objects = ContributorActivityManager()
+
     class Meta:
         unique_together = ('date', 'source_name', 'team_name')
         get_latest_by = 'date'

bedrock/mozorg/tests/test_views.py

@@ -2,11 +2,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from datetime import date
+import json
 
 from django.conf import settings
 from django.core import mail
 from django.core.cache import cache
 from django.db.utils import DatabaseError
+from django.http.response import Http404
 from django.test.client import RequestFactory
 from django.test.utils import override_settings
 from django.utils import simplejson
@@ -21,6 +24,7 @@ from nose.tools import assert_false, eq_, ok_
 from bedrock.mozorg.tests import TestCase
 from bedrock.mozorg import views
 from lib import l10n_utils
+from scripts import update_tableau_data
 
 
 _ALL = settings.STUB_INSTALLER_ALL
@@ -743,6 +747,52 @@ class TestProcessPartnershipForm(TestCase):
             return mock.call_args[0][1]['lead_source']
 
         eq_(_req(None),
-           'www.mozilla.org/about/partnerships/')
+            'www.mozilla.org/about/partnerships/')
         eq_(_req({'lead_source': 'www.mozilla.org/firefox/partners/'}),
-           'www.mozilla.org/firefox/partners/')
+            'www.mozilla.org/firefox/partners/')
+
+
+class TestMozIDDataView(TestCase):
+    def setUp(self):
+        with patch.object(update_tableau_data, 'get_external_data') as ged:
+            ged.return_value = (
+                (date(2015, 2, 2), 'Firefox', 'bugzilla', 100, 10),
+                (date(2015, 2, 2), 'Firefox OS', 'bugzilla', 100, 10),
+                (date(2015, 2, 9), 'Sumo', 'sumo', 100, 10),
+                (date(2015, 2, 9), 'Firefox OS', 'sumo', 100, 10),
+                (date(2015, 2, 9), 'QA', 'reps', 100, 10),
+            )
+            update_tableau_data.run()
+
+    def _get_json(self, source):
+        cache.clear()
+        req = RequestFactory().get('/')
+        resp = views.mozid_data_view(req, source)
+        eq_(resp['content-type'], 'application/json')
+        eq_(resp['access-control-allow-origin'], '*')
+        return json.loads(resp.content)
+
+    def test_all(self):
+        eq_(self._get_json('all'), [
+            {'wkcommencing': '2015-02-09', 'totalactive': 300, 'new': 30},
+            {'wkcommencing': '2015-02-02', 'totalactive': 200, 'new': 20},
+        ])
+
+    def test_team(self):
+        """When acting on a team, should just return sums for that team."""
+        eq_(self._get_json('firefoxos'), [
+            {'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
+            {'wkcommencing': '2015-02-02', 'totalactive': 100, 'new': 10},
+        ])
+
+    def test_source(self):
+        """When acting on a source, should just return sums for that source."""
+        eq_(self._get_json('sumo'), [
+            {'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
+        ])
+
+    @patch('bedrock.mozorg.models.CONTRIBUTOR_SOURCE_NAMES', {})
+    def test_unknown(self):
+        """An unknown source should raise a 404."""
+        with self.assertRaises(Http404):
+            self._get_json('does-not-exist')

bedrock/mozorg/urls.py

@@ -203,4 +203,6 @@ urlpatterns = patterns('',
         views.plugincheck,
         name='mozorg.plugincheck'),
     url(r'^robots.txt$', views.Robots.as_view(), name='robots.txt'),
+    url(r'^contributor-data/(?P<source_name>[a-z]{2,20})\.json$', views.mozid_data_view,
+        name='mozorg.contributor-data'),
 )

bedrock/mozorg/util.py

@@ -26,11 +26,14 @@ log = commonware.log.getLogger('mozorg.util')
 
 
 class HttpResponseJSON(HttpResponse):
-    def __init__(self, data, status=None):
+    def __init__(self, data, status=None, cors=False):
         super(HttpResponseJSON, self).__init__(content=json.dumps(data),
                                                content_type='application/json',
                                                status=status)
 
+        if cors:
+            self['Access-Control-Allow-Origin'] = '*'
+
 
 def page(name, tmpl, decorators=None, url_name=None, **kwargs):
     """

bedrock/mozorg/views.py

@@ -8,7 +8,8 @@ import re
 from django.conf import settings
 from django.contrib.staticfiles.finders import find as find_static
 from django.core.context_processors import csrf
-from django.http import HttpResponseRedirect
+from django.http import HttpResponseRedirect, Http404
+from django.views.decorators.cache import cache_page
 from django.views.decorators.csrf import csrf_exempt, csrf_protect
 from django.views.decorators.http import last_modified, require_safe
 from django.views.generic import FormView, TemplateView
@@ -29,9 +30,8 @@ from bedrock.mozorg.forms import (ContributeForm,
                                   ContributeStudentAmbassadorForm,
                                   WebToLeadForm, ContributeSignupForm)
 from bedrock.mozorg.forums import ForumsFile
-from bedrock.mozorg.models import TwitterCache
-from bedrock.mozorg.util import hide_contrib_form
-from bedrock.mozorg.util import HttpResponseJSON
+from bedrock.mozorg.models import ContributorActivity, TwitterCache
+from bedrock.mozorg.util import hide_contrib_form, HttpResponseJSON
 from bedrock.newsletter.forms import NewsletterFooterForm
 
 
@@ -51,6 +51,21 @@ def hacks_newsletter(request):
                              'mozorg/newsletter/hacks.mozilla.org.html')
 
 
+@cache_page(60 * 60 * 24 * 7)  # one week
+def mozid_data_view(request, source_name):
+    try:
+        qs = ContributorActivity.objects.group_by_date_and_source(source_name)
+    except ContributorActivity.DoesNotExist:
+        # not a valid source_name
+        raise Http404
+
+    data = [{'wkcommencing': activity['date'].isoformat(),
+             'totalactive': activity['total__sum'],
+             'new': activity['new__sum']} for activity in qs]
+
+    return HttpResponseJSON(data, cors=True)
+
+
 class ContributeSignup(l10n_utils.LangFilesMixin, FormView):
     template_name = 'mozorg/contribute/signup.html'
     form_class = ContributeSignupForm

bedrock/settings/base.py

@@ -76,6 +76,7 @@ SUPPORTED_NONLOCALES += [
     'robots.txt',
     'telemetry',
     'webmaker',
+    'contributor-data',
 ]
 
 ALLOWED_HOSTS = [

etc/httpd/global.conf

@@ -887,3 +887,6 @@ RewriteRule ^/seamonkey-transition\.html$ http://www-archive.mozilla.org/seamonk
 
 # bug 1121082
 RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?hello/?$ /$1firefox/hello/ [L,R=301]
+
+# bug 1116511
+RewriteRule ^/(contributor-data/[a-z]+.json)$ /b/$1 [L,PT]

scripts/update_tableau_data.py

@@ -13,8 +13,13 @@ QUERY = ('SELECT c_date, team_name, source_name, count(*) AS total, IFNULL(SUM(i
          'FROM contributor_active {where} GROUP BY c_date, team_name, source_name')
 
 
-def run():
-    """Get contributor activity data from Tableau and insert it into bedrock DB."""
+def process_name_fields(team_name):
+    """Lowercase and remove spaces"""
+    return team_name.replace(' ', '').lower()
+
+
+def get_external_data():
+    """Get the data and return it as a tuple of tuples."""
     if not settings.TABLEAU_DB_URL:
         print 'Must set TABLEAU_DB_URL.'
         sys.exit(1)
@@ -44,20 +49,7 @@ def run():
         con = MySQLdb.connect(**con_data)
         cur = con.cursor()
         cur.execute(QUERY.format(where=where_clause))
-        rows = cur.fetchall()
-        activities = []
-        for row in rows:
-            activities.append(ContributorActivity(
-                date=row[0],
-                team_name=row[1],
-                source_name=row[2],
-                total=row[3],
-                new=row[4],
-            ))
-
-        ContributorActivity.objects.bulk_create(activities)
-        print 'Created {0} contributor activity rows'.format(len(rows))
-
+        return cur.fetchall()
     except MySQLdb.Error as e:
         sys.stderr.write('Error %d: %s' % (e.args[0], e.args[1]))
         sys.exit(1)
@@ -65,3 +57,18 @@ def run():
     finally:
         if con:
             con.close()
+
+
+def run():
+    """Get contributor activity data from Tableau and insert it into bedrock DB."""
+    activities = []
+    for row in get_external_data():
+        activities.append(ContributorActivity(
+            date=row[0],
+            team_name=process_name_fields(row[1]),
+            source_name=process_name_fields(row[2]),
+            total=row[3],
+            new=row[4],
+        ))
+
+    ContributorActivity.objects.bulk_create(activities)