Remove all contributor data machinery

Fix #8471
This commit is contained in:
Paul McLanahan 2020-01-22 12:24:06 -05:00 коммит произвёл Alex Gibson
Родитель 7a2d812ef3
Коммит 21e76ddf5b
6 изменённых файлов: 2 добавлений и 199 удалений

Просмотреть файл

@ -8,19 +8,6 @@ from picklefield import PickledObjectField
from django_extensions.db.fields import ModificationDateTimeField
CONTRIBUTOR_SOURCE_NAMES = {
'all': None,
'sumo': 'team',
'reps': 'team',
'qa': 'team',
'firefox': 'team',
'firefoxos': 'team',
'firefoxforandroid': 'team',
'bugzilla': 'source',
'github': 'source',
}
def twitter_cache_key(account):
return 'tweets-for-' + str(account)
@ -58,34 +45,3 @@ class TwitterCache(models.Model):
def twitter_cache_post_save(sender, **kwargs):
instance = kwargs['instance']
cache_tweets(instance.account, instance.tweets)
class ContributorActivityManager(models.Manager):
def group_by_date_and_source(self, source):
try:
source_type = CONTRIBUTOR_SOURCE_NAMES[source]
except KeyError:
raise ContributorActivity.DoesNotExist
qs = self.values('date')
if source_type is not None:
field_name = source_type + '_name'
qs = qs.filter(**{field_name: source})
# dates are grouped in weeks. 52 results gives us a year.
return qs.annotate(models.Sum('total'), models.Sum('new'))[:52]
class ContributorActivity(models.Model):
date = models.DateField()
source_name = models.CharField(max_length=100)
team_name = models.CharField(max_length=100)
total = models.IntegerField()
new = models.IntegerField()
objects = ContributorActivityManager()
class Meta:
unique_together = ('date', 'source_name', 'team_name')
get_latest_by = 'date'
ordering = ['-date']

Просмотреть файл

@ -3,11 +3,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from datetime import date
import json
from django.core.cache import cache
from django.http.response import Http404
from django.test import override_settings
from django.test.client import RequestFactory
@ -16,7 +12,6 @@ from mock import ANY, patch
from bedrock.mozorg.tests import TestCase
from bedrock.mozorg import views
from scripts import update_tableau_data
class TestViews(TestCase):
@ -59,52 +54,6 @@ class TestRobots(TestCase):
self.assertEqual(response.get('Content-Type'), 'text/plain')
class TestMozIDDataView(TestCase):
def setUp(self):
with patch.object(update_tableau_data, 'get_external_data') as ged:
ged.return_value = (
(date(2015, 2, 2), 'Firefox', 'bugzilla', 100, 10),
(date(2015, 2, 2), 'Firefox OS', 'bugzilla', 100, 10),
(date(2015, 2, 9), 'Sumo', 'sumo', 100, 10),
(date(2015, 2, 9), 'Firefox OS', 'sumo', 100, 10),
(date(2015, 2, 9), 'QA', 'reps', 100, 10),
)
update_tableau_data.run()
def _get_json(self, source):
cache.clear()
req = RequestFactory().get('/')
resp = views.mozid_data_view(req, source)
assert resp['content-type'] == 'application/json'
assert resp['access-control-allow-origin'] == '*'
return json.loads(resp.content)
def test_all(self):
assert self._get_json('all') == [
{'wkcommencing': '2015-02-09', 'totalactive': 300, 'new': 30},
{'wkcommencing': '2015-02-02', 'totalactive': 200, 'new': 20},
]
def test_team(self):
"""When acting on a team, should just return sums for that team."""
assert self._get_json('firefoxos') == [
{'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
{'wkcommencing': '2015-02-02', 'totalactive': 100, 'new': 10},
]
def test_source(self):
"""When acting on a source, should just return sums for that source."""
assert self._get_json('sumo') == [
{'wkcommencing': '2015-02-09', 'totalactive': 100, 'new': 10},
]
@patch('bedrock.mozorg.models.CONTRIBUTOR_SOURCE_NAMES', {})
def test_unknown(self):
"""An unknown source should raise a 404."""
with self.assertRaises(Http404):
self._get_json('does-not-exist')
@patch('bedrock.mozorg.views.l10n_utils.render')
class TestTechnology(TestCase):
def setUp(self):

Просмотреть файл

@ -273,8 +273,6 @@ urlpatterns = (
page('contribute/stories/michael', 'mozorg/contribute/story-michael.html'),
page('contribute/stories/ruben', 'mozorg/contribute/story-ruben.html'),
page('contribute/stories/shreyas', 'mozorg/contribute/story-shreyas.html'),
url(r'^contributor-data/(?P<source_name>[a-z]{2,20})\.json$', views.mozid_data_view,
name='mozorg.contributor-data'),
url('^internet-health/$', views.IHView.as_view(), name='mozorg.internet-health'),
page('internet-health/privacy-security', 'mozorg/internet-health/privacy-security.html'),
page('internet-health/digital-inclusion', 'mozorg/internet-health/digital-inclusion.html'),

Просмотреть файл

@ -6,10 +6,10 @@ import re
from commonware.decorators import xframe_allow
from django.conf import settings
from django.http import Http404, HttpResponseRedirect, JsonResponse
from django.http import HttpResponseRedirect
from django.shortcuts import render as django_render
from django.urls import reverse
from django.views.decorators.cache import cache_page, never_cache
from django.views.decorators.cache import never_cache
from django.views.decorators.http import require_safe
from django.views.generic import TemplateView
from lib import l10n_utils
@ -18,7 +18,6 @@ from bedrock.base.waffle import switch
from bedrock.contentcards.models import get_page_content_cards
from bedrock.mozorg.credits import CreditsFile
from bedrock.mozorg.forums import ForumsFile
from bedrock.mozorg.models import ContributorActivity
from bedrock.mozorg.util import (
fxa_concert_rsvp,
get_fxa_oauth_token,
@ -45,23 +44,6 @@ def hacks_newsletter(request):
'mozorg/newsletter/hacks.mozilla.org.html')
@cache_page(60 * 60 * 24 * 7) # one week
def mozid_data_view(request, source_name):
try:
qs = ContributorActivity.objects.group_by_date_and_source(source_name)
except ContributorActivity.DoesNotExist:
# not a valid source_name
raise Http404
data = [{'wkcommencing': activity['date'].isoformat(),
'totalactive': activity['total__sum'],
'new': activity['new__sum']} for activity in qs]
response = JsonResponse(data, safe=False)
response['Access-Control-Allow-Origin'] = '*'
return response
@require_safe
def credits_view(request):
"""Display the names of our contributors."""

Просмотреть файл

@ -1402,9 +1402,6 @@ LOGGING = {
}
PASSWORD_HASHERS = ['django.contrib.auth.hashers.PBKDF2PasswordHasher']
TABLEAU_DB_URL = config('TABLEAU_DB_URL', default='')
ADMINS = MANAGERS = config('ADMINS', parser=json.loads,
default='[]')

Просмотреть файл

@ -1,79 +0,0 @@
import urllib.parse
import sys
from django.conf import settings
try:
import MySQLdb
except ImportError:
# not everyone needs the mysql lib, but this function needs it.
# we'll pass here and fail later so tests can work.
MySQLdb = None
from bedrock.mozorg.models import ContributorActivity
urllib.parse.uses_netloc.append('mysql')
QUERY = ('SELECT c_date, team_name, source_name, count(*) AS total, IFNULL(SUM(is_new), 0) AS new '
'FROM contributor_active {where} GROUP BY c_date, team_name, source_name')
def process_name_fields(team_name):
"""Lowercase and remove spaces"""
return team_name.replace(' ', '').lower()
def get_external_data():
"""Get the data and return it as a tuple of tuples."""
if not settings.TABLEAU_DB_URL:
print('Must set TABLEAU_DB_URL.')
sys.exit(1)
url = urllib.parse.urlparse(settings.TABLEAU_DB_URL)
if not url.path:
# bad db url
print('TABLEAU_DB_URL not parseable.')
sys.exit(1)
con_data = {
# remove slash
'db': url.path[1:],
'user': url.username,
'passwd': url.password,
'host': url.hostname,
}
con = None
try:
latest_date = ContributorActivity.objects.only('date').latest().date
where_clause = 'WHERE c_date > "{0}"'.format(latest_date.isoformat())
except ContributorActivity.DoesNotExist:
where_clause = ''
try:
con = MySQLdb.connect(**con_data)
cur = con.cursor()
cur.execute(QUERY.format(where=where_clause))
return cur.fetchall()
except MySQLdb.Error as e:
sys.stderr.write('Error %d: %s' % (e.args[0], e.args[1]))
sys.exit(1)
finally:
if con:
con.close()
def run():
"""Get contributor activity data from Tableau and insert it into bedrock DB."""
activities = []
for row in get_external_data():
activities.append(ContributorActivity(
date=row[0],
team_name=process_name_fields(row[1]),
source_name=process_name_fields(row[2]),
total=row[3],
new=row[4],
))
ContributorActivity.objects.bulk_create(activities)