drop global stats and dashboard (#11047)
This commit is contained in:
Andrew Williamson
GitHub
Родитель
78c16bd645
Коммит
a7a6d50be7
|
|
@ -35,7 +35,6 @@ HOME=/tmp
|
|||
# Once per day after metrics import is done
|
||||
30 12 * * * %(z_cron)s update_addon_download_totals
|
||||
35 12 * * * %(z_cron)s weekly_downloads
|
||||
25 13 * * * %(z_cron)s update_global_totals
|
||||
30 13 * * * %(z_cron)s update_addon_average_daily_users
|
||||
00 14 * * * %(z_cron)s index_latest_stats
|
||||
|
||||
|
|
|
|||
|
|
@ -127,7 +127,6 @@ class TestHomepageFeatures(TestCase):
|
|||
'base/users',
|
||||
'base/addon_3615',
|
||||
'base/collections',
|
||||
'base/global-stats',
|
||||
'base/featured',
|
||||
'addons/featured',
|
||||
'bandwagon/featured_collections']
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
[
|
||||
{
|
||||
"pk": 216632,
|
||||
"model": "stats.globalstat",
|
||||
"fields": {
|
||||
"count": 1757933362,
|
||||
"date": "2010-01-17",
|
||||
"name": "addon_total_downloads"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pk": 216653,
|
||||
"model": "stats.globalstat",
|
||||
"fields": {
|
||||
"count": 160702913,
|
||||
"date": "2010-01-17",
|
||||
"name": "addon_total_updatepings"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -8,7 +8,7 @@ from olympia.amo.tests import TestCase
|
|||
|
||||
|
||||
class TestRedirects(TestCase):
|
||||
fixtures = ['ratings/test_models', 'addons/persona', 'base/global-stats']
|
||||
fixtures = ['ratings/test_models', 'addons/persona']
|
||||
|
||||
def test_persona_category(self):
|
||||
"""`/personas/film and tv/` should go to /themes/film-and-tv/"""
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ class Test404(TestCase):
|
|||
|
||||
|
||||
class TestCommon(TestCase):
|
||||
fixtures = ('base/users', 'base/global-stats', 'base/addon_3615')
|
||||
fixtures = ('base/users', 'base/addon_3615')
|
||||
|
||||
def setUp(self):
|
||||
super(TestCommon, self).setUp()
|
||||
|
|
|
|||
|
|
@ -1179,7 +1179,6 @@ CELERY_TASK_ROUTES = {
|
|||
'olympia.stats.tasks.index_download_counts': {'queue': 'stats'},
|
||||
'olympia.stats.tasks.index_theme_user_counts': {'queue': 'stats'},
|
||||
'olympia.stats.tasks.index_update_counts': {'queue': 'stats'},
|
||||
'olympia.stats.tasks.update_global_totals': {'queue': 'stats'},
|
||||
|
||||
# Tags
|
||||
'olympia.tags.tasks.update_all_tag_stats': {'queue': 'tags'},
|
||||
|
|
@ -1777,7 +1776,6 @@ CRON_JOBS = {
|
|||
'cleanup_extracted_file': 'olympia.files.cron',
|
||||
'cleanup_validation_results': 'olympia.files.cron',
|
||||
|
||||
'update_global_totals': 'olympia.stats.cron',
|
||||
'index_latest_stats': 'olympia.stats.cron',
|
||||
|
||||
'update_user_ratings': 'olympia.users.cron',
|
||||
|
|
|
|||
|
|
@ -1,17 +1,13 @@
|
|||
import datetime
|
||||
|
||||
from django.core.management import call_command
|
||||
from django.db.models import Max
|
||||
|
||||
import waffle
|
||||
|
||||
from celery import group
|
||||
|
||||
import olympia.core.logger
|
||||
|
||||
from olympia.lib.es.utils import raise_if_reindex_in_progress
|
||||
|
||||
from . import tasks
|
||||
from .models import UpdateCount
|
||||
|
||||
|
||||
|
|
@ -19,26 +15,6 @@ task_log = olympia.core.logger.getLogger('z.task')
|
|||
cron_log = olympia.core.logger.getLogger('z.cron')
|
||||
|
||||
|
||||
def update_global_totals(date=None):
|
||||
"""Update global statistics totals."""
|
||||
raise_if_reindex_in_progress('amo')
|
||||
|
||||
if date:
|
||||
date = datetime.datetime.strptime(date, '%Y-%m-%d').date()
|
||||
# Assume that we want to populate yesterday's stats by default.
|
||||
today = date or datetime.date.today() - datetime.timedelta(days=1)
|
||||
today_jobs = [{'job': job, 'date': today} for job in
|
||||
tasks._get_daily_jobs(date)]
|
||||
|
||||
max_update = date or UpdateCount.objects.aggregate(max=Max('date'))['max']
|
||||
metrics_jobs = [{'job': job, 'date': max_update} for job in
|
||||
tasks._get_metrics_jobs(date)]
|
||||
|
||||
ts = [tasks.update_global_totals.subtask(kwargs=kw)
|
||||
for kw in today_jobs + metrics_jobs]
|
||||
group(ts).apply_async()
|
||||
|
||||
|
||||
def index_latest_stats(index=None):
|
||||
if not waffle.switch_is_active('local-statistics-processing'):
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -105,18 +105,6 @@ class ThemeUpdateCountBulk(models.Model):
|
|||
db_table = 'theme_update_counts_bulk'
|
||||
|
||||
|
||||
class GlobalStat(models.Model):
|
||||
id = PositiveAutoField(primary_key=True)
|
||||
name = models.CharField(max_length=255)
|
||||
count = models.IntegerField()
|
||||
date = models.DateField()
|
||||
|
||||
class Meta:
|
||||
db_table = 'global_stats'
|
||||
unique_together = ('name', 'date')
|
||||
get_latest_by = 'date'
|
||||
|
||||
|
||||
class ThemeUserCount(StatsSearchMixin, models.Model):
|
||||
"""Theme popularity (weekly average of users).
|
||||
|
||||
|
|
|
|||
|
|
@ -1,23 +1,9 @@
|
|||
import datetime
|
||||
|
||||
from django.db import connection
|
||||
from django.db.models import Max, Sum
|
||||
|
||||
import six
|
||||
|
||||
from dateutil.parser import parse as dateutil_parser
|
||||
from elasticsearch.helpers import bulk as bulk_index
|
||||
|
||||
import olympia.core.logger
|
||||
|
||||
from olympia import amo
|
||||
from olympia.addons.models import Addon
|
||||
from olympia.amo import search as amo_search
|
||||
from olympia.amo.celery import task
|
||||
from olympia.bandwagon.models import Collection
|
||||
from olympia.ratings.models import Rating
|
||||
from olympia.users.models import UserProfile
|
||||
from olympia.versions.models import Version
|
||||
|
||||
from . import search
|
||||
from .models import DownloadCount, ThemeUserCount, UpdateCount
|
||||
|
|
@ -26,140 +12,6 @@ from .models import DownloadCount, ThemeUserCount, UpdateCount
|
|||
log = olympia.core.logger.getLogger('z.task')
|
||||
|
||||
|
||||
@task
|
||||
def update_global_totals(job, date, **kw):
|
||||
log.info('Updating global statistics totals (%s) for (%s)' % (job, date))
|
||||
|
||||
if isinstance(date, six.string_types):
|
||||
# Because of celery serialization, date is not date object, it has been
|
||||
# transformed into a string, we need the date object back.
|
||||
date = dateutil_parser(date).date()
|
||||
|
||||
jobs = _get_daily_jobs(date)
|
||||
jobs.update(_get_metrics_jobs(date))
|
||||
|
||||
num = jobs[job]()
|
||||
|
||||
q = """REPLACE INTO global_stats (`name`, `count`, `date`)
|
||||
VALUES (%s, %s, %s)"""
|
||||
p = [job, num or 0, date]
|
||||
|
||||
try:
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(q, p)
|
||||
except Exception as e:
|
||||
log.critical('Failed to update global stats: (%s): %s' % (p, e))
|
||||
else:
|
||||
log.debug('Committed global stats details: (%s) has (%s) for (%s)'
|
||||
% tuple(p))
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
|
||||
def _get_daily_jobs(date=None):
|
||||
"""Return a dictionary of statistics queries.
|
||||
|
||||
If a date is specified and applies to the job it will be used. Otherwise
|
||||
the date will default to the previous day.
|
||||
"""
|
||||
if not date:
|
||||
date = datetime.date.today() - datetime.timedelta(days=1)
|
||||
|
||||
# Passing through a datetime would not generate an error,
|
||||
# but would pass and give incorrect values.
|
||||
if isinstance(date, datetime.datetime):
|
||||
raise ValueError('This requires a valid date object, not a datetime')
|
||||
|
||||
# Testing on lte created date doesn't get you todays date, you need to do
|
||||
# less than next date. That's because 2012-1-1 becomes 2012-1-1 00:00
|
||||
next_date = date + datetime.timedelta(days=1)
|
||||
|
||||
date_str = date.strftime('%Y-%m-%d')
|
||||
extra = {'where': ['DATE(created)=%s'], 'params': [date_str]}
|
||||
|
||||
# If you're editing these, note that you are returning a function! This
|
||||
# cheesy hackery was done so that we could pass the queries to celery
|
||||
# lazily and not hammer the db with a ton of these all at once.
|
||||
stats = {
|
||||
# Add-on Downloads
|
||||
'addon_total_downloads': lambda: DownloadCount.objects.filter(
|
||||
date__lt=next_date).aggregate(sum=Sum('count'))['sum'],
|
||||
'addon_downloads_new': lambda: DownloadCount.objects.filter(
|
||||
date=date).aggregate(sum=Sum('count'))['sum'],
|
||||
|
||||
# Listed Add-on counts
|
||||
'addon_count_new': Addon.objects.valid().extra(**extra).count,
|
||||
|
||||
# Listed Version counts
|
||||
'version_count_new': Version.objects.filter(
|
||||
channel=amo.RELEASE_CHANNEL_LISTED).extra(**extra).count,
|
||||
|
||||
# User counts
|
||||
'user_count_total': UserProfile.objects.filter(
|
||||
created__lt=next_date).count,
|
||||
'user_count_new': UserProfile.objects.extra(**extra).count,
|
||||
|
||||
# Rating counts
|
||||
'review_count_total': Rating.objects.filter(created__lte=date,
|
||||
editorreview=0).count,
|
||||
# We can't use "**extra" here, because this query joins on reviews
|
||||
# itself, and thus raises the following error:
|
||||
# "Column 'created' in where clause is ambiguous".
|
||||
'review_count_new': Rating.objects.filter(editorreview=0).extra(
|
||||
where=['DATE(reviews.created)=%s'], params=[date_str]).count,
|
||||
|
||||
# Collection counts
|
||||
'collection_count_total': Collection.objects.filter(
|
||||
created__lt=next_date).count,
|
||||
'collection_count_new': Collection.objects.extra(**extra).count,
|
||||
}
|
||||
|
||||
# If we're processing today's stats, we'll do some extras. We don't do
|
||||
# these for re-processed stats because they change over time (eg. add-ons
|
||||
# move from sandbox -> public
|
||||
if date == (datetime.date.today() - datetime.timedelta(days=1)):
|
||||
stats.update({
|
||||
'addon_count_nominated': Addon.objects.filter(
|
||||
created__lte=date, status=amo.STATUS_NOMINATED,
|
||||
disabled_by_user=0).count,
|
||||
'addon_count_public': Addon.objects.filter(
|
||||
created__lte=date, status=amo.STATUS_PUBLIC,
|
||||
disabled_by_user=0).count,
|
||||
'addon_count_pending': Version.objects.filter(
|
||||
created__lte=date, files__status=amo.STATUS_PENDING).count,
|
||||
|
||||
'collection_count_private': Collection.objects.filter(
|
||||
created__lte=date, listed=0).count,
|
||||
'collection_count_public': Collection.objects.filter(
|
||||
created__lte=date, listed=1).count,
|
||||
'collection_count_editorspicks': Collection.objects.filter(
|
||||
created__lte=date, type=amo.COLLECTION_FEATURED).count,
|
||||
'collection_count_normal': Collection.objects.filter(
|
||||
created__lte=date, type=amo.COLLECTION_NORMAL).count,
|
||||
})
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def _get_metrics_jobs(date=None):
|
||||
"""Return a dictionary of statistics queries.
|
||||
|
||||
If a date is specified and applies to the job it will be used. Otherwise
|
||||
the date will default to the last date metrics put something in the db.
|
||||
"""
|
||||
|
||||
if not date:
|
||||
date = UpdateCount.objects.aggregate(max=Max('date'))['max']
|
||||
|
||||
# If you're editing these, note that you are returning a function!
|
||||
stats = {
|
||||
'addon_total_updatepings': lambda: UpdateCount.objects.filter(
|
||||
date=date).aggregate(sum=Sum('count'))['sum'],
|
||||
}
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
@task
|
||||
def index_update_counts(ids, index=None, **kw):
|
||||
index = index or search.get_alias()
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
{% extends "stats/stats.html" %}
|
||||
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
{# L10n: This is an email. Whitespace matters! #}
|
||||
{{ thankyou_note }}
|
||||
|
||||
{% trans %}Learn more about the future of {{ addon_name }} by clicking here:{% endtrans %}
|
||||
{{ learn_url }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Add-on Statistics') %}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
<nav id="side-nav">
|
||||
<ul>
|
||||
{# Set li class="type" from 'metricTypes' in chart.js for active toggle #}
|
||||
<li class="addons_in_use">
|
||||
<a href="{{ url('stats.addons_in_use') }}">
|
||||
{{ _('Add-ons in Use') }}</a>
|
||||
</li>
|
||||
<li class="addons_created">
|
||||
<a href="{{ url('stats.addons_created') }}">
|
||||
{{ _('Add-ons Created') }}</a>
|
||||
</li>
|
||||
<li class="addons_downloaded">
|
||||
<a href="{{ url('stats.addons_downloaded') }}">
|
||||
{{ _('Add-ons Downloaded') }}</a>
|
||||
</li>
|
||||
<li class="addons_updated">
|
||||
<a href="{{ url('stats.addons_updated') }}">
|
||||
{{ _('Add-ons Updated') }}</a>
|
||||
</li>
|
||||
<li class="collections_created">
|
||||
<a href="{{ url('stats.collections_created') }}">
|
||||
{{ _('Collections Created') }}</a>
|
||||
</li>
|
||||
<li class="reviews_created">
|
||||
<a href="{{ url('stats.reviews_created') }}">
|
||||
{{ _('Reviews Written') }}</a>
|
||||
</li>
|
||||
<li class="users_created">
|
||||
<a href="{{ url('stats.users_created') }}">
|
||||
{{ _('User Signups') }}</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="https://addons-server.readthedocs.io/en/latest/topics/api/stats.html#archiving">
|
||||
{{ _('Archived Data') }}</a>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Add-ons Created by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Add-ons Downloaded by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Add-ons in Use by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Add-ons Updated by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Collections Created by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Ratings by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Reviews Created by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('Subscribers by Date') %}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{% extends 'stats/report.html' %}
|
||||
|
||||
{% set title = _('User Signups by Date') %}
|
||||
|
|
@ -12,27 +12,23 @@ from olympia.bandwagon.models import Collection
|
|||
|
||||
@library.global_function
|
||||
@jinja2.contextfunction
|
||||
def report_menu(context, request, report, obj=None):
|
||||
def report_menu(context, request, report, obj):
|
||||
"""Reports Menu. navigation for the various statistic reports."""
|
||||
if obj:
|
||||
if isinstance(obj, Addon):
|
||||
has_privs = False
|
||||
if (request.user.is_authenticated and (
|
||||
acl.action_allowed(request, amo.permissions.STATS_VIEW) or
|
||||
obj.has_author(request.user))):
|
||||
has_privs = True
|
||||
t = loader.get_template('stats/addon_report_menu.html')
|
||||
c = {
|
||||
'addon': obj,
|
||||
'has_privs': has_privs
|
||||
}
|
||||
return jinja2.Markup(t.render(c))
|
||||
if isinstance(obj, Collection):
|
||||
t = loader.get_template('stats/collection_report_menu.html')
|
||||
c = {
|
||||
'collection': obj,
|
||||
}
|
||||
return jinja2.Markup(t.render(c))
|
||||
|
||||
t = loader.get_template('stats/global_report_menu.html')
|
||||
return jinja2.Markup(t.render())
|
||||
if isinstance(obj, Addon):
|
||||
has_privs = False
|
||||
if (request.user.is_authenticated and (
|
||||
acl.action_allowed(request, amo.permissions.STATS_VIEW) or
|
||||
obj.has_author(request.user))):
|
||||
has_privs = True
|
||||
t = loader.get_template('stats/addon_report_menu.html')
|
||||
c = {
|
||||
'addon': obj,
|
||||
'has_privs': has_privs
|
||||
}
|
||||
return jinja2.Markup(t.render(c))
|
||||
if isinstance(obj, Collection):
|
||||
t = loader.get_template('stats/collection_report_menu.html')
|
||||
c = {
|
||||
'collection': obj,
|
||||
}
|
||||
return jinja2.Markup(t.render(c))
|
||||
|
|
|
|||
|
|
@ -3,92 +3,12 @@ import datetime
|
|||
from django.core.management import call_command
|
||||
|
||||
import mock
|
||||
from freezegun import freeze_time
|
||||
|
||||
from olympia import amo
|
||||
from olympia.amo.tests import TestCase, addon_factory, version_factory
|
||||
from olympia.stats import cron, tasks
|
||||
from olympia.amo.tests import TestCase
|
||||
from olympia.stats import cron
|
||||
from olympia.stats.models import (
|
||||
DownloadCount, GlobalStat, ThemeUserCount, UpdateCount)
|
||||
|
||||
|
||||
class TestGlobalStats(TestCase):
|
||||
fixtures = ['stats/test_models']
|
||||
|
||||
def test_stats_for_date(self):
|
||||
date = datetime.date(2009, 6, 1)
|
||||
job = 'addon_total_downloads'
|
||||
|
||||
assert GlobalStat.objects.filter(
|
||||
date=date, name=job).count() == 0
|
||||
tasks.update_global_totals(job, date)
|
||||
assert len(GlobalStat.objects.filter(
|
||||
date=date, name=job)) == 1
|
||||
|
||||
def test_count_stats_for_date(self):
|
||||
# Add a listed add-on, it should show up in "addon_count_new".
|
||||
listed_addon = addon_factory(created=datetime.datetime.now())
|
||||
|
||||
# Add an unlisted version to that add-on, it should *not* increase the
|
||||
# "version_count_new" count.
|
||||
version_factory(
|
||||
addon=listed_addon, channel=amo.RELEASE_CHANNEL_UNLISTED)
|
||||
|
||||
# Add an unlisted add-on, it should not show up in either
|
||||
# "addon_count_new" or "version_count_new".
|
||||
addon_factory(version_kw={
|
||||
'channel': amo.RELEASE_CHANNEL_UNLISTED
|
||||
})
|
||||
|
||||
date = datetime.date.today()
|
||||
job = 'addon_count_new'
|
||||
tasks.update_global_totals(job, date)
|
||||
global_stat = GlobalStat.objects.get(date=date, name=job)
|
||||
assert global_stat.count == 1
|
||||
|
||||
# Should still work if the date is passed as a datetime string (what
|
||||
# celery serialization does).
|
||||
job = 'version_count_new'
|
||||
tasks.update_global_totals(job, datetime.datetime.now().isoformat())
|
||||
global_stat = GlobalStat.objects.get(date=date, name=job)
|
||||
assert global_stat.count == 1
|
||||
|
||||
def test_through_cron(self):
|
||||
# Yesterday, create some stuff.
|
||||
with freeze_time(datetime.datetime.now() - datetime.timedelta(days=1)):
|
||||
yesterday = datetime.date.today()
|
||||
|
||||
# Add a listed add-on, it should show up in "addon_count_new".
|
||||
listed_addon = addon_factory(created=datetime.datetime.now())
|
||||
|
||||
# Add an unlisted version to that add-on, it should *not* increase
|
||||
# the "version_count_new" count.
|
||||
version_factory(
|
||||
addon=listed_addon, channel=amo.RELEASE_CHANNEL_UNLISTED)
|
||||
|
||||
# Add an unlisted add-on, it should not show up in either
|
||||
# "addon_count_new" or "version_count_new".
|
||||
addon_factory(version_kw={
|
||||
'channel': amo.RELEASE_CHANNEL_UNLISTED
|
||||
})
|
||||
|
||||
# Launch the cron.
|
||||
cron.update_global_totals()
|
||||
|
||||
job = 'addon_count_new'
|
||||
global_stat = GlobalStat.objects.get(date=yesterday, name=job)
|
||||
assert global_stat.count == 1
|
||||
|
||||
job = 'version_count_new'
|
||||
global_stat = GlobalStat.objects.get(date=yesterday, name=job)
|
||||
assert global_stat.count == 1
|
||||
|
||||
def test_input(self):
|
||||
for x in ['2009-1-1',
|
||||
datetime.datetime(2009, 1, 1),
|
||||
datetime.datetime(2009, 1, 1, 11, 0)]:
|
||||
with self.assertRaises((TypeError, ValueError)):
|
||||
tasks._get_daily_jobs(x)
|
||||
DownloadCount, ThemeUserCount, UpdateCount)
|
||||
|
||||
|
||||
@mock.patch('olympia.stats.management.commands.index_stats.group')
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
|
||||
from django.http import Http404
|
||||
from django.test.client import RequestFactory
|
||||
from django.utils.encoding import force_text
|
||||
|
||||
import mock
|
||||
import six
|
||||
|
||||
from pyquery import PyQuery as pq
|
||||
|
|
@ -18,8 +16,7 @@ from olympia.addons.models import Addon, AddonUser
|
|||
from olympia.amo.tests import TestCase, version_factory
|
||||
from olympia.amo.urlresolvers import reverse
|
||||
from olympia.stats import tasks, views
|
||||
from olympia.stats.models import (
|
||||
DownloadCount, GlobalStat, ThemeUserCount, UpdateCount)
|
||||
from olympia.stats.models import DownloadCount, ThemeUserCount, UpdateCount
|
||||
from olympia.users.models import UserProfile
|
||||
|
||||
|
||||
|
|
@ -720,82 +717,6 @@ class TestResponses(ESStatsTest):
|
|||
2009-06-01,10,2,3""")
|
||||
|
||||
|
||||
# Test the SQL query by using known dates, for weeks and months etc.
|
||||
class TestSiteQuery(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSiteQuery, self).setUp()
|
||||
self.start = datetime.date(2012, 1, 1)
|
||||
self.end = datetime.date(2012, 1, 31)
|
||||
for k in range(0, 15):
|
||||
for name in ['addon_count_new', 'version_count_new']:
|
||||
date_ = self.start + datetime.timedelta(days=k)
|
||||
GlobalStat.objects.create(date=date_, name=name, count=k)
|
||||
|
||||
def test_day_grouping(self):
|
||||
res = views._site_query('date', self.start, self.end)[0]
|
||||
assert len(res) == 14
|
||||
assert res[0]['data']['addons_created'] == 14
|
||||
# Make sure we are returning counts as integers, otherwise
|
||||
# DjangoJSONSerializer will map them to strings.
|
||||
assert type(res[0]['data']['addons_created']) == int
|
||||
assert res[0]['date'] == '2012-01-15'
|
||||
|
||||
def test_week_grouping(self):
|
||||
res = views._site_query('week', self.start, self.end)[0]
|
||||
assert len(res) == 3
|
||||
assert res[1]['data']['addons_created'] == 70
|
||||
assert res[1]['date'] == '2012-01-08'
|
||||
|
||||
def test_month_grouping(self):
|
||||
res = views._site_query('month', self.start, self.end)[0]
|
||||
assert len(res) == 1
|
||||
assert res[0]['data']['addons_created'] == (14 * (14 + 1)) / 2
|
||||
assert res[0]['date'] == '2012-01-02'
|
||||
|
||||
def test_period(self):
|
||||
self.assertRaises(AssertionError, views._site_query, 'not_period',
|
||||
self.start, self.end)
|
||||
|
||||
|
||||
@mock.patch('olympia.stats.views._site_query')
|
||||
class TestSite(TestCase):
|
||||
|
||||
def tests_period(self, _site_query):
|
||||
_site_query.return_value = ['.', '.']
|
||||
for period in ['date', 'week', 'month']:
|
||||
self.client.get(reverse('stats.site', args=['json', period]))
|
||||
assert _site_query.call_args[0][0] == period
|
||||
|
||||
def tests_period_day(self, _site_query):
|
||||
_site_query.return_value = ['.', '.']
|
||||
start = (datetime.date.today() - datetime.timedelta(days=3))
|
||||
end = datetime.date.today()
|
||||
self.client.get(reverse('stats.site.new',
|
||||
args=['day', start.strftime('%Y%m%d'),
|
||||
end.strftime('%Y%m%d'), 'json']))
|
||||
assert _site_query.call_args[0][0] == 'date'
|
||||
assert _site_query.call_args[0][1] == start
|
||||
assert _site_query.call_args[0][2] == end
|
||||
|
||||
def test_csv(self, _site_query):
|
||||
_site_query.return_value = [[], []]
|
||||
res = self.client.get(reverse('stats.site', args=['csv', 'date']))
|
||||
assert res._headers['content-type'][1].startswith('text/csv')
|
||||
|
||||
def test_json(self, _site_query):
|
||||
_site_query.return_value = [[], []]
|
||||
res = self.client.get(reverse('stats.site', args=['json', 'date']))
|
||||
assert res._headers['content-type'][1].startswith('application/json')
|
||||
|
||||
def tests_no_date(self, _site_query):
|
||||
_site_query.return_value = ['.', '.']
|
||||
self.client.get(reverse('stats.site', args=['json', 'date']))
|
||||
assert _site_query.call_args[0][1] == (
|
||||
datetime.date.today() - datetime.timedelta(days=365))
|
||||
assert _site_query.call_args[0][2] == datetime.date.today()
|
||||
|
||||
|
||||
class TestXss(amo.tests.TestXss):
|
||||
|
||||
def test_stats_page(self):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from django.conf.urls import url
|
||||
from django.shortcuts import redirect
|
||||
|
||||
from . import views
|
||||
|
||||
|
|
@ -10,31 +9,6 @@ range_re = r'(?P<start>\d{8})-(?P<end>\d{8})'
|
|||
format_re = r'(?P<format>' + '|'.join(views.SERIES_FORMATS) + ')'
|
||||
series_re = r'%s-%s\.%s$' % (group_re, range_re, format_re)
|
||||
series = dict((type, r'%s-%s' % (type, series_re)) for type in views.SERIES)
|
||||
global_series = dict((type, r'%s-%s' % (type, series_re))
|
||||
for type in views.GLOBAL_SERIES)
|
||||
|
||||
|
||||
urlpatterns = [
|
||||
url(r'^$', lambda r: redirect('stats.addons_in_use', permanent=False),
|
||||
name='stats.dashboard'),
|
||||
url(r'^site%s/%s$' % (format_re, group_date_re),
|
||||
views.site, name='stats.site'),
|
||||
url(r'^site-%s' % series_re, views.site, name='stats.site.new'),
|
||||
]
|
||||
|
||||
# These are the front end pages, so that when you click the links on the
|
||||
# navigation page, you end up on the correct stats page for AMO.
|
||||
keys = ['addons_in_use', 'addons_updated', 'addons_downloaded',
|
||||
'addons_created', 'collections_created',
|
||||
'reviews_created', 'users_created']
|
||||
|
||||
for key in keys:
|
||||
urlpatterns.append(url(
|
||||
r'^%s/$' % key, views.site_stats_report,
|
||||
name='stats.%s' % key, kwargs={'report': key}))
|
||||
urlpatterns.append(url(
|
||||
global_series[key], views.site_series,
|
||||
kwargs={'field': key}))
|
||||
|
||||
|
||||
# Addon specific stats.
|
||||
|
|
|
|||
|
|
@ -2,14 +2,11 @@ import csv
|
|||
import json
|
||||
import time
|
||||
|
||||
from collections import OrderedDict
|
||||
from datetime import date, timedelta
|
||||
from datetime import timedelta
|
||||
|
||||
from django import http
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import PermissionDenied
|
||||
from django.core.files.storage import get_storage_class
|
||||
from django.db import connection
|
||||
from django.db.transaction import non_atomic_requests
|
||||
from django.utils.cache import add_never_cache_headers, patch_cache_control
|
||||
from django.utils.encoding import force_text
|
||||
|
|
@ -27,7 +24,6 @@ from olympia.amo.decorators import allow_cross_site_request
|
|||
from olympia.amo.urlresolvers import reverse
|
||||
from olympia.amo.utils import AMOJSONEncoder, render
|
||||
from olympia.core.languages import ALL_LANGUAGES
|
||||
from olympia.lib.cache import memoize
|
||||
from olympia.stats.decorators import addon_view_stats
|
||||
from olympia.stats.forms import DateForm
|
||||
|
||||
|
|
@ -42,23 +38,11 @@ SERIES_GROUPS_DATE = ('date', 'week', 'month') # Backwards compat.
|
|||
SERIES_FORMATS = ('json', 'csv')
|
||||
SERIES = ('downloads', 'usage', 'overview', 'sources', 'os',
|
||||
'locales', 'statuses', 'versions', 'apps')
|
||||
GLOBAL_SERIES = ('addons_in_use', 'addons_updated', 'addons_downloaded',
|
||||
'collections_created', 'reviews_created', 'addons_created',
|
||||
'users_created', 'my_apps')
|
||||
|
||||
|
||||
storage = get_storage_class()()
|
||||
|
||||
|
||||
@non_atomic_requests
|
||||
def dashboard(request):
|
||||
stats_base_url = reverse('stats.dashboard')
|
||||
view = get_report_view(request)
|
||||
return render(request, 'stats/dashboard.html',
|
||||
{'report': 'site', 'view': view,
|
||||
'stats_base_url': stats_base_url})
|
||||
|
||||
|
||||
def get_series(model, extra_field=None, source=None, **filters):
|
||||
"""
|
||||
Get a generator of dicts for the stats model given by the filters.
|
||||
|
|
@ -341,15 +325,6 @@ def stats_report(request, addon, report):
|
|||
'stats_base_url': stats_base_url})
|
||||
|
||||
|
||||
@non_atomic_requests
|
||||
def site_stats_report(request, report):
|
||||
stats_base_url = reverse('stats.dashboard')
|
||||
view = get_report_view(request)
|
||||
return render(request, 'stats/reports/%s.html' % report,
|
||||
{'report': report, 'view': view,
|
||||
'stats_base_url': stats_base_url})
|
||||
|
||||
|
||||
def get_report_view(request):
|
||||
"""Parse and validate a pair of YYYMMDD date strings."""
|
||||
dates = DateForm(data=request.GET)
|
||||
|
|
@ -387,101 +362,6 @@ def get_daterange_or_404(start, end):
|
|||
)
|
||||
|
||||
|
||||
def daterange(start_date, end_date):
|
||||
for n in range((end_date - start_date).days):
|
||||
yield start_date + timedelta(n)
|
||||
|
||||
|
||||
# Cached lookup of the keys and the SQL.
|
||||
# Taken from remora, a mapping of the old values.
|
||||
_KEYS = {
|
||||
'addon_downloads_new': 'addons_downloaded',
|
||||
'addon_total_updatepings': 'addons_in_use',
|
||||
'addon_count_new': 'addons_created',
|
||||
'version_count_new': 'addons_updated',
|
||||
'user_count_new': 'users_created',
|
||||
'review_count_new': 'reviews_created',
|
||||
'collection_count_new': 'collections_created',
|
||||
}
|
||||
_ALL_KEYS = list(_KEYS)
|
||||
|
||||
_CACHED_KEYS = sorted(_KEYS.values())
|
||||
|
||||
|
||||
@memoize(prefix='global_stats', timeout=60 * 60)
|
||||
def _site_query(period, start, end, field=None, request=None):
|
||||
with connection.cursor() as cursor:
|
||||
# Let MySQL make this fast. Make sure we prevent SQL injection with the
|
||||
# assert.
|
||||
if period not in SERIES_GROUPS_DATE:
|
||||
raise AssertionError('%s period is not valid.' % period)
|
||||
|
||||
sql = ("SELECT name, MIN(date), SUM(count) "
|
||||
"FROM global_stats "
|
||||
"WHERE date > %%s AND date <= %%s "
|
||||
"AND name IN (%s) "
|
||||
"GROUP BY %s(date), name "
|
||||
"ORDER BY %s(date) DESC;"
|
||||
% (', '.join(['%s' for key in _ALL_KEYS]), period, period))
|
||||
cursor.execute(sql, [start, end] + _ALL_KEYS)
|
||||
|
||||
# Process the results into a format that is friendly for render_*.
|
||||
default = {k: 0 for k in _CACHED_KEYS}
|
||||
result = OrderedDict()
|
||||
for name, date_, count in cursor.fetchall():
|
||||
date_ = date_.strftime('%Y-%m-%d')
|
||||
if date_ not in result:
|
||||
result[date_] = default.copy()
|
||||
result[date_]['date'] = date_
|
||||
result[date_]['data'] = {}
|
||||
result[date_]['data'][_KEYS[name]] = int(count)
|
||||
|
||||
return list(result.values()), _CACHED_KEYS
|
||||
|
||||
|
||||
@non_atomic_requests
|
||||
def site(request, format, group, start=None, end=None):
|
||||
"""Site data from the global_stats table."""
|
||||
if not start and not end:
|
||||
start = (date.today() - timedelta(days=365)).strftime('%Y%m%d')
|
||||
end = date.today().strftime('%Y%m%d')
|
||||
|
||||
group = 'date' if group == 'day' else group
|
||||
start, end = get_daterange_or_404(start, end)
|
||||
series, keys = _site_query(group, start, end, request)
|
||||
|
||||
if format == 'csv':
|
||||
return render_csv(request, None, series, ['date'] + keys,
|
||||
title='addons.mozilla.org week Site Statistics',
|
||||
show_disclaimer=True)
|
||||
|
||||
return render_json(request, None, series)
|
||||
|
||||
|
||||
@non_atomic_requests
|
||||
def site_series(request, format, group, start, end, field):
|
||||
"""Pull a single field from the site_query data"""
|
||||
start, end = get_daterange_or_404(start, end)
|
||||
group = 'date' if group == 'day' else group
|
||||
series = []
|
||||
full_series, keys = _site_query(group, start, end, field, request)
|
||||
for row in full_series:
|
||||
if field in row['data']:
|
||||
series.append({
|
||||
'date': row['date'],
|
||||
'count': row['data'][field],
|
||||
'data': {},
|
||||
})
|
||||
# TODO: (dspasovski) check whether this is the CSV data we really want
|
||||
if format == 'csv':
|
||||
series, fields = csv_fields(series)
|
||||
return render_csv(request, None, series,
|
||||
['date', 'count'] + list(fields),
|
||||
title='%s week Site Statistics' % settings.DOMAIN,
|
||||
show_disclaimer=True)
|
||||
return render_json(request, None, series)
|
||||
|
||||
|
||||
def fudge_headers(response, stats):
|
||||
"""Alter cache headers. Don't cache content where data could be missing."""
|
||||
if not stats:
|
||||
|
|
|
|||
|
|
@ -84,10 +84,7 @@ urlpatterns = [
|
|||
# API v3+.
|
||||
url(r'^api/', include('olympia.api.urls')),
|
||||
|
||||
# Site statistics that we are going to catch, the rest will fall through.
|
||||
url(r'^statistics/', include('olympia.stats.urls')),
|
||||
|
||||
# Fall through for any URLs not matched above stats dashboard.
|
||||
# Redirect for all global stats URLs.
|
||||
url(r'^statistics/', lambda r: redirect('/'), name='statistics.dashboard'),
|
||||
|
||||
# Redirect patterns.
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче