Merge pull request #2717 from pmclanahan/automate-mozid-data-tableau-1116511

Bug 1116511: Add tableau data sync and JSON view for MozID.

bedrock/mozorg/migrations/0002_auto__add_contributoractivity__add_unique_contributoractivity_date_sou.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'ContributorActivity'
+        db.create_table(u'mozorg_contributoractivity', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('date', self.gf('django.db.models.fields.DateField')()),
+            ('source_name', self.gf('django.db.models.fields.CharField')(max_length=100)),
+            ('team_name', self.gf('django.db.models.fields.CharField')(max_length=100)),
+            ('total', self.gf('django.db.models.fields.IntegerField')()),
+            ('new', self.gf('django.db.models.fields.IntegerField')()),
+        ))
+        db.send_create_signal(u'mozorg', ['ContributorActivity'])
+
+        # Adding unique constraint on 'ContributorActivity', fields ['date', 'source_name', 'team_name']
+        db.create_unique(u'mozorg_contributoractivity', ['date', 'source_name', 'team_name'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'ContributorActivity', fields ['date', 'source_name', 'team_name']
+        db.delete_unique(u'mozorg_contributoractivity', ['date', 'source_name', 'team_name'])
+
+        # Deleting model 'ContributorActivity'
+        db.delete_table(u'mozorg_contributoractivity')
+
+
+    models = {
+        u'mozorg.contributoractivity': {
+            'Meta': {'ordering': "['-date']", 'unique_together': "(('date', 'source_name', 'team_name'),)", 'object_name': 'ContributorActivity'},
+            'date': ('django.db.models.fields.DateField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'new': ('django.db.models.fields.IntegerField', [], {}),
+            'source_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'team_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'total': ('django.db.models.fields.IntegerField', [], {})
+        },
+        u'mozorg.twittercache': {
+            'Meta': {'object_name': 'TwitterCache'},
+            'account': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'tweets': ('picklefield.fields.PickledObjectField', [], {'default': '[]'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['mozorg']

bedrock/mozorg/models.py

@@ -6,6 +6,19 @@ from picklefield import PickledObjectField
 from django_extensions.db.fields import ModificationDateTimeField
 
 
+CONTRIBUTOR_SOURCE_NAMES = {
+    'all': None,
+    'sumo': 'team',
+    'reps': 'team',
+    'qa': 'team',
+    'firefox': 'team',
+    'firefoxos': 'team',
+    'firefoxforandroid': 'team',
+    'bugzilla': 'source',
+    'github': 'source',
+}
+
+
 class TwitterCacheManager(models.Manager):
     def get_tweets_for(self, account):
         cache_key = 'tweets-for-' + str(account)
@@ -31,3 +44,34 @@ class TwitterCache(models.Model):
 
     def __unicode__(self):
         return u'Tweets from @' + self.account
+
+
+class ContributorActivityManager(models.Manager):
+    def group_by_date_and_source(self, source):
+        try:
+            source_type = CONTRIBUTOR_SOURCE_NAMES[source]
+        except KeyError:
+            raise ContributorActivity.DoesNotExist
+
+        qs = self.values('date')
+        if source_type is not None:
+            field_name = source_type + '_name'
+            qs = qs.filter(**{field_name: source})
+
+        # dates are grouped in weeks. 52 results gives us a year.
+        return qs.annotate(models.Sum('total'), models.Sum('new'))[:52]
+
+
+class ContributorActivity(models.Model):
+    date = models.DateField()
+    source_name = models.CharField(max_length=100)
+    team_name = models.CharField(max_length=100)
+    total = models.IntegerField()
+    new = models.IntegerField()
+
+    objects = ContributorActivityManager()
+
+    class Meta:
+        unique_together = ('date', 'source_name', 'team_name')
+        get_latest_by = 'date'
+        ordering = ['-date']

bedrock/mozorg/tests/test_views.py

@@ -2,11 +2,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from datetime import date
+import json
 
 from django.conf import settings
 from django.core import mail
 from django.core.cache import cache
 from django.db.utils import DatabaseError
+from django.http.response import Http404
 from django.test.client import RequestFactory
 from django.test.utils import override_settings
 from django.utils import simplejson
@@ -21,6 +24,7 @@ from nose.tools import assert_false, eq_, ok_
 from bedrock.mozorg.tests import TestCase
 from bedrock.mozorg import views
 from lib import l10n_utils
+from scripts import update_tableau_data
 
 
 _ALL = settings.STUB_INSTALLER_ALL
@@ -743,6 +747,52 @@ class TestProcessPartnershipForm(TestCase):
             return mock.call_args[0][1]['lead_source']
 
         eq_(_req(None),
-           'www.mozilla.org/about/partnerships/')
+            'www.mozilla.org/about/partnerships/')
         eq_(_req({'lead_source': 'www.mozilla.org/firefox/partners/'}),
-           'www.mozilla.org/firefox/partners/')
+            'www.mozilla.org/firefox/partners/')
+
+
+class TestMozIDDataView(TestCase):
+    def setUp(self):
+        with patch.object(update_tableau_data, 'get_external_data') as ged:
+            ged.return_value = (
+                (date(2015, 2, 2), 'Firefox', 'bugzilla', 100, 10),
+                (date(2015, 2, 2), 'Firefox OS', 'bugzilla', 100, 10),
+                (date(2015, 2, 9), 'Sumo', 'sumo', 100, 10),
+                (date(2015, 2, 9), 'Firefox OS', 'sumo', 100, 10),
+                (date(2015, 2, 9), 'QA', 'reps', 100, 10),
+            )
+            update_tableau_data.run()
+
+    def _get_json(self, source):
+        cache.clear()
+        req = RequestFactory().get('/')
+        resp = views.mozid_data_view(req, source)
+        eq_(resp['content-type'], 'application/json')
+        eq_(resp['access-control-allow-origin'], '*')
+        return json.loads(resp.content)
+
+    def test_all(self):
+        eq_(self._get_json('all'), [
+            {'wkcommencing': '2015-02-09', 'totalactive': 300, 'new': 30},
+            {'wkcommencing': '2015-02-02', 'totalactive': 200, 'new': 20},
+        ])
+
+    def test_team(self):
+        """When acting on a team, should just return sums for that team."""
+        eq_(self._get_json('firefoxos'), [
+            {'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
+            {'wkcommencing': '2015-02-02', 'totalactive': 100, 'new': 10},
+        ])
+
+    def test_source(self):
+        """When acting on a source, should just return sums for that source."""
+        eq_(self._get_json('sumo'), [
+            {'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
+        ])
+
+    @patch('bedrock.mozorg.models.CONTRIBUTOR_SOURCE_NAMES', {})
+    def test_unknown(self):
+        """An unknown source should raise a 404."""
+        with self.assertRaises(Http404):
+            self._get_json('does-not-exist')

bedrock/mozorg/urls.py

@@ -202,4 +202,6 @@ urlpatterns = patterns('',
         views.plugincheck,
         name='mozorg.plugincheck'),
     url(r'^robots.txt$', views.Robots.as_view(), name='robots.txt'),
+    url(r'^contributor-data/(?P<source_name>[a-z]{2,20})\.json$', views.mozid_data_view,
+        name='mozorg.contributor-data'),
 )

bedrock/mozorg/util.py

@@ -26,11 +26,14 @@ log = commonware.log.getLogger('mozorg.util')
 
 
 class HttpResponseJSON(HttpResponse):
-    def __init__(self, data, status=None):
+    def __init__(self, data, status=None, cors=False):
         super(HttpResponseJSON, self).__init__(content=json.dumps(data),
                                                content_type='application/json',
                                                status=status)
 
+        if cors:
+            self['Access-Control-Allow-Origin'] = '*'
+
 
 def page(name, tmpl, decorators=None, url_name=None, **kwargs):
     """

bedrock/mozorg/views.py

@@ -8,7 +8,8 @@ import re
 from django.conf import settings
 from django.contrib.staticfiles.finders import find as find_static
 from django.core.context_processors import csrf
-from django.http import HttpResponseRedirect
+from django.http import HttpResponseRedirect, Http404
+from django.views.decorators.cache import cache_page
 from django.views.decorators.csrf import csrf_exempt, csrf_protect
 from django.views.decorators.http import last_modified, require_safe
 from django.views.generic import FormView, TemplateView
@@ -29,9 +30,8 @@ from bedrock.mozorg.forms import (ContributeForm,
                                   ContributeStudentAmbassadorForm,
                                   WebToLeadForm, ContributeSignupForm)
 from bedrock.mozorg.forums import ForumsFile
-from bedrock.mozorg.models import TwitterCache
-from bedrock.mozorg.util import hide_contrib_form
-from bedrock.mozorg.util import HttpResponseJSON
+from bedrock.mozorg.models import ContributorActivity, TwitterCache
+from bedrock.mozorg.util import hide_contrib_form, HttpResponseJSON
 from bedrock.newsletter.forms import NewsletterFooterForm
 
 
@@ -51,6 +51,21 @@ def hacks_newsletter(request):
                              'mozorg/newsletter/hacks.mozilla.org.html')
 
 
+@cache_page(60 * 60 * 24 * 7)  # one week
+def mozid_data_view(request, source_name):
+    try:
+        qs = ContributorActivity.objects.group_by_date_and_source(source_name)
+    except ContributorActivity.DoesNotExist:
+        # not a valid source_name
+        raise Http404
+
+    data = [{'wkcommencing': activity['date'].isoformat(),
+             'totalactive': activity['total__sum'],
+             'new': activity['new__sum']} for activity in qs]
+
+    return HttpResponseJSON(data, cors=True)
+
+
 class ContributeSignup(l10n_utils.LangFilesMixin, FormView):
     template_name = 'mozorg/contribute/signup.html'
     form_class = ContributeSignupForm

bedrock/settings/base.py

@@ -76,6 +76,7 @@ SUPPORTED_NONLOCALES += [
     'robots.txt',
     'telemetry',
     'webmaker',
+    'contributor-data',
 ]
 
 ALLOWED_HOSTS = [
@@ -2273,3 +2274,5 @@ FIREFOX_OS_FEEDS = (
     ('pt-BR', 'https://blog.mozilla.org/press-br/category/firefox-os/feed/'),
 )
 FIREFOX_OS_FEED_LOCALES = [feed[0] for feed in FIREFOX_OS_FEEDS]
+
+TABLEAU_DB_URL = None

etc/cron.d/bedrock-prod.tmpl

@@ -26,3 +26,7 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 # bug 1128587
 38 * * * * {{ django_manage }} runscript update_firefox_os_feeds > /dev/null 2>&1
 
+# bug 1116511
+# every tuesday midnight
+0 0 * * 2 {{ django_manage }} runscript update_tableau_data
+

etc/cron.d/bedrock-stage.tmpl

@@ -29,3 +29,7 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 # bug 1128587
 38 * * * * {{ django_manage }} runscript update_firefox_os_feeds > /dev/null 2>&1
 
+# bug 1116511
+# every tuesday midnight
+0 0 * * 2 {{ django_manage }} runscript update_tableau_data
+

etc/httpd/global.conf

@@ -889,3 +889,6 @@ RewriteRule ^/seamonkey-transition\.html$ http://www-archive.mozilla.org/seamonk
 
 # bug 1121082
 RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?hello/?$ /$1firefox/hello/ [L,R=301]
+
+# bug 1116511
+RewriteRule ^/(contributor-data/[a-z]+.json)$ /b/$1 [L,PT]

scripts/update_tableau_data.py

@@ -0,0 +1,74 @@
+import urlparse
+import sys
+
+from django.conf import settings
+
+import MySQLdb
+
+from bedrock.mozorg.models import ContributorActivity
+
+
+urlparse.uses_netloc.append('mysql')
+QUERY = ('SELECT c_date, team_name, source_name, count(*) AS total, IFNULL(SUM(is_new), 0) AS new '
+         'FROM contributor_active {where} GROUP BY c_date, team_name, source_name')
+
+
+def process_name_fields(team_name):
+    """Lowercase and remove spaces"""
+    return team_name.replace(' ', '').lower()
+
+
+def get_external_data():
+    """Get the data and return it as a tuple of tuples."""
+    if not settings.TABLEAU_DB_URL:
+        print 'Must set TABLEAU_DB_URL.'
+        sys.exit(1)
+
+    url = urlparse.urlparse(settings.TABLEAU_DB_URL)
+    if not url.path:
+        # bad db url
+        print 'TABLEAU_DB_URL not parseable.'
+        sys.exit(1)
+
+    con_data = {
+        # remove slash
+        'db': url.path[1:],
+        'user': url.username,
+        'passwd': url.password,
+        'host': url.hostname,
+    }
+    con = None
+
+    try:
+        latest_date = ContributorActivity.objects.only('date').latest().date
+        where_clause = 'WHERE c_date > "{0}"'.format(latest_date.isoformat())
+    except ContributorActivity.DoesNotExist:
+        where_clause = ''
+
+    try:
+        con = MySQLdb.connect(**con_data)
+        cur = con.cursor()
+        cur.execute(QUERY.format(where=where_clause))
+        return cur.fetchall()
+    except MySQLdb.Error as e:
+        sys.stderr.write('Error %d: %s' % (e.args[0], e.args[1]))
+        sys.exit(1)
+
+    finally:
+        if con:
+            con.close()
+
+
+def run():
+    """Get contributor activity data from Tableau and insert it into bedrock DB."""
+    activities = []
+    for row in get_external_data():
+        activities.append(ContributorActivity(
+            date=row[0],
+            team_name=process_name_fields(row[1]),
+            source_name=process_name_fields(row[2]),
+            total=row[3],
+            new=row[4],
+        ))
+
+    ContributorActivity.objects.bulk_create(activities)