зеркало из https://github.com/mozilla/bedrock.git
Родитель
7a2d812ef3
Коммит
21e76ddf5b
|
@ -8,19 +8,6 @@ from picklefield import PickledObjectField
|
|||
from django_extensions.db.fields import ModificationDateTimeField
|
||||
|
||||
|
||||
CONTRIBUTOR_SOURCE_NAMES = {
|
||||
'all': None,
|
||||
'sumo': 'team',
|
||||
'reps': 'team',
|
||||
'qa': 'team',
|
||||
'firefox': 'team',
|
||||
'firefoxos': 'team',
|
||||
'firefoxforandroid': 'team',
|
||||
'bugzilla': 'source',
|
||||
'github': 'source',
|
||||
}
|
||||
|
||||
|
||||
def twitter_cache_key(account):
|
||||
return 'tweets-for-' + str(account)
|
||||
|
||||
|
@ -58,34 +45,3 @@ class TwitterCache(models.Model):
|
|||
def twitter_cache_post_save(sender, **kwargs):
|
||||
instance = kwargs['instance']
|
||||
cache_tweets(instance.account, instance.tweets)
|
||||
|
||||
|
||||
class ContributorActivityManager(models.Manager):
|
||||
def group_by_date_and_source(self, source):
|
||||
try:
|
||||
source_type = CONTRIBUTOR_SOURCE_NAMES[source]
|
||||
except KeyError:
|
||||
raise ContributorActivity.DoesNotExist
|
||||
|
||||
qs = self.values('date')
|
||||
if source_type is not None:
|
||||
field_name = source_type + '_name'
|
||||
qs = qs.filter(**{field_name: source})
|
||||
|
||||
# dates are grouped in weeks. 52 results gives us a year.
|
||||
return qs.annotate(models.Sum('total'), models.Sum('new'))[:52]
|
||||
|
||||
|
||||
class ContributorActivity(models.Model):
|
||||
date = models.DateField()
|
||||
source_name = models.CharField(max_length=100)
|
||||
team_name = models.CharField(max_length=100)
|
||||
total = models.IntegerField()
|
||||
new = models.IntegerField()
|
||||
|
||||
objects = ContributorActivityManager()
|
||||
|
||||
class Meta:
|
||||
unique_together = ('date', 'source_name', 'team_name')
|
||||
get_latest_by = 'date'
|
||||
ordering = ['-date']
|
||||
|
|
|
@ -3,11 +3,7 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
import os
|
||||
from datetime import date
|
||||
import json
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.http.response import Http404
|
||||
from django.test import override_settings
|
||||
from django.test.client import RequestFactory
|
||||
|
||||
|
@ -16,7 +12,6 @@ from mock import ANY, patch
|
|||
|
||||
from bedrock.mozorg.tests import TestCase
|
||||
from bedrock.mozorg import views
|
||||
from scripts import update_tableau_data
|
||||
|
||||
|
||||
class TestViews(TestCase):
|
||||
|
@ -59,52 +54,6 @@ class TestRobots(TestCase):
|
|||
self.assertEqual(response.get('Content-Type'), 'text/plain')
|
||||
|
||||
|
||||
class TestMozIDDataView(TestCase):
|
||||
def setUp(self):
|
||||
with patch.object(update_tableau_data, 'get_external_data') as ged:
|
||||
ged.return_value = (
|
||||
(date(2015, 2, 2), 'Firefox', 'bugzilla', 100, 10),
|
||||
(date(2015, 2, 2), 'Firefox OS', 'bugzilla', 100, 10),
|
||||
(date(2015, 2, 9), 'Sumo', 'sumo', 100, 10),
|
||||
(date(2015, 2, 9), 'Firefox OS', 'sumo', 100, 10),
|
||||
(date(2015, 2, 9), 'QA', 'reps', 100, 10),
|
||||
)
|
||||
update_tableau_data.run()
|
||||
|
||||
def _get_json(self, source):
|
||||
cache.clear()
|
||||
req = RequestFactory().get('/')
|
||||
resp = views.mozid_data_view(req, source)
|
||||
assert resp['content-type'] == 'application/json'
|
||||
assert resp['access-control-allow-origin'] == '*'
|
||||
return json.loads(resp.content)
|
||||
|
||||
def test_all(self):
|
||||
assert self._get_json('all') == [
|
||||
{'wkcommencing': '2015-02-09', 'totalactive': 300, 'new': 30},
|
||||
{'wkcommencing': '2015-02-02', 'totalactive': 200, 'new': 20},
|
||||
]
|
||||
|
||||
def test_team(self):
|
||||
"""When acting on a team, should just return sums for that team."""
|
||||
assert self._get_json('firefoxos') == [
|
||||
{'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
|
||||
{'wkcommencing': '2015-02-02', 'totalactive': 100, 'new': 10},
|
||||
]
|
||||
|
||||
def test_source(self):
|
||||
"""When acting on a source, should just return sums for that source."""
|
||||
assert self._get_json('sumo') == [
|
||||
{'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
|
||||
]
|
||||
|
||||
@patch('bedrock.mozorg.models.CONTRIBUTOR_SOURCE_NAMES', {})
|
||||
def test_unknown(self):
|
||||
"""An unknown source should raise a 404."""
|
||||
with self.assertRaises(Http404):
|
||||
self._get_json('does-not-exist')
|
||||
|
||||
|
||||
@patch('bedrock.mozorg.views.l10n_utils.render')
|
||||
class TestTechnology(TestCase):
|
||||
def setUp(self):
|
||||
|
|
|
@ -273,8 +273,6 @@ urlpatterns = (
|
|||
page('contribute/stories/michael', 'mozorg/contribute/story-michael.html'),
|
||||
page('contribute/stories/ruben', 'mozorg/contribute/story-ruben.html'),
|
||||
page('contribute/stories/shreyas', 'mozorg/contribute/story-shreyas.html'),
|
||||
url(r'^contributor-data/(?P<source_name>[a-z]{2,20})\.json$', views.mozid_data_view,
|
||||
name='mozorg.contributor-data'),
|
||||
url('^internet-health/$', views.IHView.as_view(), name='mozorg.internet-health'),
|
||||
page('internet-health/privacy-security', 'mozorg/internet-health/privacy-security.html'),
|
||||
page('internet-health/digital-inclusion', 'mozorg/internet-health/digital-inclusion.html'),
|
||||
|
|
|
@ -6,10 +6,10 @@ import re
|
|||
|
||||
from commonware.decorators import xframe_allow
|
||||
from django.conf import settings
|
||||
from django.http import Http404, HttpResponseRedirect, JsonResponse
|
||||
from django.http import HttpResponseRedirect
|
||||
from django.shortcuts import render as django_render
|
||||
from django.urls import reverse
|
||||
from django.views.decorators.cache import cache_page, never_cache
|
||||
from django.views.decorators.cache import never_cache
|
||||
from django.views.decorators.http import require_safe
|
||||
from django.views.generic import TemplateView
|
||||
from lib import l10n_utils
|
||||
|
@ -18,7 +18,6 @@ from bedrock.base.waffle import switch
|
|||
from bedrock.contentcards.models import get_page_content_cards
|
||||
from bedrock.mozorg.credits import CreditsFile
|
||||
from bedrock.mozorg.forums import ForumsFile
|
||||
from bedrock.mozorg.models import ContributorActivity
|
||||
from bedrock.mozorg.util import (
|
||||
fxa_concert_rsvp,
|
||||
get_fxa_oauth_token,
|
||||
|
@ -45,23 +44,6 @@ def hacks_newsletter(request):
|
|||
'mozorg/newsletter/hacks.mozilla.org.html')
|
||||
|
||||
|
||||
@cache_page(60 * 60 * 24 * 7) # one week
|
||||
def mozid_data_view(request, source_name):
|
||||
try:
|
||||
qs = ContributorActivity.objects.group_by_date_and_source(source_name)
|
||||
except ContributorActivity.DoesNotExist:
|
||||
# not a valid source_name
|
||||
raise Http404
|
||||
|
||||
data = [{'wkcommencing': activity['date'].isoformat(),
|
||||
'totalactive': activity['total__sum'],
|
||||
'new': activity['new__sum']} for activity in qs]
|
||||
|
||||
response = JsonResponse(data, safe=False)
|
||||
response['Access-Control-Allow-Origin'] = '*'
|
||||
return response
|
||||
|
||||
|
||||
@require_safe
|
||||
def credits_view(request):
|
||||
"""Display the names of our contributors."""
|
||||
|
|
|
@ -1402,9 +1402,6 @@ LOGGING = {
|
|||
}
|
||||
|
||||
PASSWORD_HASHERS = ['django.contrib.auth.hashers.PBKDF2PasswordHasher']
|
||||
|
||||
TABLEAU_DB_URL = config('TABLEAU_DB_URL', default='')
|
||||
|
||||
ADMINS = MANAGERS = config('ADMINS', parser=json.loads,
|
||||
default='[]')
|
||||
|
||||
|
|
|
@ -1,79 +0,0 @@
|
|||
import urllib.parse
|
||||
import sys
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
try:
|
||||
import MySQLdb
|
||||
except ImportError:
|
||||
# not everyone needs the mysql lib, but this function needs it.
|
||||
# we'll pass here and fail later so tests can work.
|
||||
MySQLdb = None
|
||||
|
||||
from bedrock.mozorg.models import ContributorActivity
|
||||
|
||||
|
||||
urllib.parse.uses_netloc.append('mysql')
|
||||
QUERY = ('SELECT c_date, team_name, source_name, count(*) AS total, IFNULL(SUM(is_new), 0) AS new '
|
||||
'FROM contributor_active {where} GROUP BY c_date, team_name, source_name')
|
||||
|
||||
|
||||
def process_name_fields(team_name):
|
||||
"""Lowercase and remove spaces"""
|
||||
return team_name.replace(' ', '').lower()
|
||||
|
||||
|
||||
def get_external_data():
|
||||
"""Get the data and return it as a tuple of tuples."""
|
||||
if not settings.TABLEAU_DB_URL:
|
||||
print('Must set TABLEAU_DB_URL.')
|
||||
sys.exit(1)
|
||||
|
||||
url = urllib.parse.urlparse(settings.TABLEAU_DB_URL)
|
||||
if not url.path:
|
||||
# bad db url
|
||||
print('TABLEAU_DB_URL not parseable.')
|
||||
sys.exit(1)
|
||||
|
||||
con_data = {
|
||||
# remove slash
|
||||
'db': url.path[1:],
|
||||
'user': url.username,
|
||||
'passwd': url.password,
|
||||
'host': url.hostname,
|
||||
}
|
||||
con = None
|
||||
|
||||
try:
|
||||
latest_date = ContributorActivity.objects.only('date').latest().date
|
||||
where_clause = 'WHERE c_date > "{0}"'.format(latest_date.isoformat())
|
||||
except ContributorActivity.DoesNotExist:
|
||||
where_clause = ''
|
||||
|
||||
try:
|
||||
con = MySQLdb.connect(**con_data)
|
||||
cur = con.cursor()
|
||||
cur.execute(QUERY.format(where=where_clause))
|
||||
return cur.fetchall()
|
||||
except MySQLdb.Error as e:
|
||||
sys.stderr.write('Error %d: %s' % (e.args[0], e.args[1]))
|
||||
sys.exit(1)
|
||||
|
||||
finally:
|
||||
if con:
|
||||
con.close()
|
||||
|
||||
|
||||
def run():
|
||||
"""Get contributor activity data from Tableau and insert it into bedrock DB."""
|
||||
activities = []
|
||||
for row in get_external_data():
|
||||
activities.append(ContributorActivity(
|
||||
date=row[0],
|
||||
team_name=process_name_fields(row[1]),
|
||||
source_name=process_name_fields(row[2]),
|
||||
total=row[3],
|
||||
new=row[4],
|
||||
))
|
||||
|
||||
ContributorActivity.objects.bulk_create(activities)
|
Загрузка…
Ссылка в новой задаче