Merge pull request from tarekziade/808505-check-indexer-flag

Block indexing cron while a reindexing is going on (Bug 808505)
This commit is contained in:
Tarek Ziade 2012-11-06 01:36:05 -08:00
Родитель 975262dcaa ab251db27f
Коммит 1694b826b2
9 изменённых файлов: 101 добавлений и 11 удалений
apps
lib/es
management/commands
utils.py
mkt

Просмотреть файл

@ -24,6 +24,7 @@ from amo.utils import chunked
from addons import search from addons import search
from addons.models import Addon, FrozenAddon, AppSupport from addons.models import Addon, FrozenAddon, AppSupport
from files.models import File from files.models import File
from lib.es.utils import raise_if_reindex_in_progress
from stats.models import UpdateCount from stats.models import UpdateCount
log = logging.getLogger('z.cron') log = logging.getLogger('z.cron')
@ -94,6 +95,7 @@ def _update_addons_current_version(data, **kw):
@cronjobs.register @cronjobs.register
def update_addon_average_daily_users(): def update_addon_average_daily_users():
"""Update add-ons ADU totals.""" """Update add-ons ADU totals."""
raise_if_reindex_in_progress()
cursor = connections[multidb.get_slave()].cursor() cursor = connections[multidb.get_slave()].cursor()
q = """SELECT q = """SELECT
addon_id, AVG(`count`) addon_id, AVG(`count`)

Просмотреть файл

@ -1,3 +1,6 @@
import os
import datetime
from nose.tools import eq_ from nose.tools import eq_
import mock import mock
@ -5,7 +8,10 @@ import amo
import amo.tests import amo.tests
from addons import cron from addons import cron
from addons.models import Addon, AppSupport from addons.models import Addon, AppSupport
from django.core.management.base import CommandError
from files.models import File, Platform from files.models import File, Platform
from lib.es.management.commands.reindex import flag_database, unflag_database
from stats.models import UpdateCount
from versions.models import Version from versions.models import Version
@ -211,6 +217,35 @@ class AvgDailyUserCountTestCase(amo.tests.TestCase):
addon = Addon.objects.get(pk=3615) addon = Addon.objects.get(pk=3615)
eq_(addon.average_daily_users, addon.total_downloads) eq_(addon.average_daily_users, addon.total_downloads)
def test_adu_flag(self):
addon = Addon.objects.get(pk=3615)
now = datetime.datetime.now()
counter = UpdateCount.objects.create(addon=addon, date=now,
count=1234)
counter.save()
self.assertTrue(
addon.average_daily_users > addon.total_downloads + 10000,
'Unexpected ADU count. ADU of %d not greater than %d' % (
addon.average_daily_users, addon.total_downloads + 10000))
adu = cron.update_addon_average_daily_users
flag_database('new', 'old', 'alias')
try:
# Should fail.
self.assertRaises(CommandError, adu)
# Should work with the environ flag.
os.environ['FORCE_INDEXING'] = '1'
adu()
finally:
unflag_database()
del os.environ['FORCE_INDEXING']
addon = Addon.objects.get(pk=3615)
eq_(addon.average_daily_users, 1234)
class TestReindex(amo.tests.ESTestCase): class TestReindex(amo.tests.ESTestCase):

Просмотреть файл

@ -16,6 +16,7 @@ from bandwagon.models import Collection
from cake.models import Session from cake.models import Session
from constants.base import VALID_STATUSES from constants.base import VALID_STATUSES
from devhub.models import ActivityLog, LegacyAddonLog from devhub.models import ActivityLog, LegacyAddonLog
from lib.es.utils import raise_if_reindex_in_progress
from sharing import SERVICES_LIST, LOCAL_SERVICES_LIST from sharing import SERVICES_LIST, LOCAL_SERVICES_LIST
from stats.models import AddonShareCount, Contribution from stats.models import AddonShareCount, Contribution
@ -244,6 +245,7 @@ def weekly_downloads():
""" """
Update 7-day add-on download counts. Update 7-day add-on download counts.
""" """
raise_if_reindex_in_progress()
cursor = connection.cursor() cursor = connection.cursor()
cursor.execute(""" cursor.execute("""
SELECT addon_id, SUM(count) AS weekly_count SELECT addon_id, SUM(count) AS weekly_count

Просмотреть файл

@ -12,6 +12,7 @@ from addons.models import Addon
from .models import (AddonCollectionCount, CollectionCount, from .models import (AddonCollectionCount, CollectionCount,
UpdateCount) UpdateCount)
from . import tasks from . import tasks
from lib.es.utils import raise_if_reindex_in_progress
task_log = commonware.log.getLogger('z.task') task_log = commonware.log.getLogger('z.task')
cron_log = commonware.log.getLogger('z.cron') cron_log = commonware.log.getLogger('z.cron')
@ -20,6 +21,7 @@ cron_log = commonware.log.getLogger('z.cron')
@cronjobs.register @cronjobs.register
def update_addons_collections_downloads(): def update_addons_collections_downloads():
"""Update addons+collections download totals.""" """Update addons+collections download totals."""
raise_if_reindex_in_progress()
d = (AddonCollectionCount.objects.values('addon', 'collection') d = (AddonCollectionCount.objects.values('addon', 'collection')
.annotate(sum=Sum('count'))) .annotate(sum=Sum('count')))
@ -44,6 +46,7 @@ def update_collections_total():
@cronjobs.register @cronjobs.register
def update_global_totals(date=None): def update_global_totals(date=None):
"""Update global statistics totals.""" """Update global statistics totals."""
raise_if_reindex_in_progress()
if date: if date:
date = datetime.datetime.strptime(date, '%Y-%m-%d').date() date = datetime.datetime.strptime(date, '%Y-%m-%d').date()
@ -70,6 +73,7 @@ def addon_total_contributions():
@cronjobs.register @cronjobs.register
def index_latest_stats(index=None, aliased=True): def index_latest_stats(index=None, aliased=True):
raise_if_reindex_in_progress()
latest = UpdateCount.search(index).order_by('-date').values_dict() latest = UpdateCount.search(index).order_by('-date').values_dict()
if latest: if latest:
latest = latest[0]['date'] latest = latest[0]['date']

Просмотреть файл

@ -2,6 +2,7 @@ import datetime
import json import json
import logging import logging
from optparse import make_option from optparse import make_option
import os
import re import re
import sys import sys
import traceback import traceback
@ -25,6 +26,7 @@ from stats.search import setup_indexes as put_stats_mapping
from users.cron import reindex_users from users.cron import reindex_users
from lib.es.models import Reindexing from lib.es.models import Reindexing
from lib.es.utils import database_flagged
if django_settings.MARKETPLACE: if django_settings.MARKETPLACE:
from mkt.stats.cron import index_mkt_stats from mkt.stats.cron import index_mkt_stats
@ -214,11 +216,6 @@ def create_index(index, is_stats):
traceback.print_exc() traceback.print_exc()
def database_flagged():
"""Returns True if the Database is being indexed"""
return Reindexing.objects.exists()
@task_with_callbacks @task_with_callbacks
def flag_database(new_index, old_index, alias): def flag_database(new_index, old_index, alias):
"""Flags the database to indicate that the reindexing has started.""" """Flags the database to indicate that the reindexing has started."""
@ -363,13 +360,17 @@ class Command(BaseCommand):
# let's do it # let's do it
log('Running all indexation tasks') log('Running all indexation tasks')
tree.apply_async()
time.sleep(10) # give celeryd some time to flag the DB os.environ['FORCE_INDEXING'] = '1'
while database_flagged(): try:
sys.stdout.write('.') tree.apply_async()
sys.stdout.flush() time.sleep(10) # give celeryd some time to flag the DB
time.sleep(5) while database_flagged():
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(5)
finally:
del os.environ['FORCE_INDEXING']
sys.stdout.write('\n') sys.stdout.write('\n')

Просмотреть файл

@ -1,4 +1,7 @@
import os
from .models import Reindexing from .models import Reindexing
from django.core.management.base import CommandError
import elasticutils.contrib.django as elasticutils import elasticutils.contrib.django as elasticutils
@ -34,3 +37,19 @@ def index_objects(ids, model, search, index=None, transforms=None):
model.index(data, bulk=True, id=ob.id, index=index) model.index(data, bulk=True, id=ob.id, index=index)
elasticutils.get_es().flush_bulk(forced=True) elasticutils.get_es().flush_bulk(forced=True)
def database_flagged():
"""Returns True if the Database is being indexed"""
return Reindexing.objects.exists()
def raise_if_reindex_in_progress():
"""Checks if the database indexation flag is on.
If it's one, and if no "FORCE_INDEXING" variable is present in the env,
raises a CommandError.
"""
if database_flagged() and 'FORCE_INDEXING' not in os.environ:
raise CommandError("Indexation already occuring. Add a FORCE_INDEXING "
"variable in the environ to force it")

Просмотреть файл

@ -7,6 +7,7 @@ import cronjobs
import pyes import pyes
from stats.models import Contribution from stats.models import Contribution
from lib.es.utils import raise_if_reindex_in_progress
from mkt.webapps.models import Installed from mkt.webapps.models import Installed
from . import tasks from . import tasks
@ -15,6 +16,7 @@ cron_log = commonware.log.getLogger('mkt.cron')
@cronjobs.register @cronjobs.register
def index_latest_mkt_stats(index=None, aliased=True): def index_latest_mkt_stats(index=None, aliased=True):
raise_if_reindex_in_progress()
yesterday = datetime.date.today() - datetime.timedelta(days=1) yesterday = datetime.date.today() - datetime.timedelta(days=1)
try: try:

Просмотреть файл

@ -10,6 +10,7 @@ from django.db.models import Count
import commonware.log import commonware.log
import cronjobs import cronjobs
from celery.task.sets import TaskSet from celery.task.sets import TaskSet
from lib.es.utils import raise_if_reindex_in_progress
import amo import amo
from amo.utils import chunked from amo.utils import chunked
@ -23,6 +24,7 @@ log = commonware.log.getLogger('z.cron')
@cronjobs.register @cronjobs.register
def update_weekly_downloads(): def update_weekly_downloads():
"""Update the weekly "downloads" from the users_install table.""" """Update the weekly "downloads" from the users_install table."""
raise_if_reindex_in_progress()
interval = datetime.today() - timedelta(days=7) interval = datetime.today() - timedelta(days=7)
counts = (Installed.objects.values('addon') counts = (Installed.objects.values('addon')
.filter(created__gte=interval, .filter(created__gte=interval,

Просмотреть файл

@ -9,6 +9,8 @@ from nose.tools import eq_
import amo import amo
import amo.tests import amo.tests
from addons.models import Addon from addons.models import Addon
from django.core.management.base import CommandError
from lib.es.management.commands.reindex import flag_database, unflag_database
from users.models import UserProfile from users.models import UserProfile
from mkt.webapps.cron import clean_old_signed, update_weekly_downloads from mkt.webapps.cron import clean_old_signed, update_weekly_downloads
from mkt.webapps.models import Installed, Webapp from mkt.webapps.models import Installed, Webapp
@ -39,6 +41,27 @@ class TestWeeklyDownloads(amo.tests.TestCase):
update_weekly_downloads() update_weekly_downloads()
eq_(self.get_webapp().weekly_downloads, 2) eq_(self.get_webapp().weekly_downloads, 2)
def test_weekly_downloads_flagged(self):
eq_(self.get_webapp().weekly_downloads, 0)
self.add_install()
self.add_install(user=UserProfile.objects.get(pk=10482),
created=datetime.today() - timedelta(days=2))
flag_database('new', 'old', 'alias')
try:
# Should fail.
self.assertRaises(CommandError, update_weekly_downloads)
eq_(self.get_webapp().weekly_downloads, 0)
# Should work with the environ flag.
os.environ['FORCE_INDEXING'] = '1'
update_weekly_downloads()
finally:
unflag_database()
del os.environ['FORCE_INDEXING']
eq_(self.get_webapp().weekly_downloads, 2)
def test_recently(self): def test_recently(self):
self.add_install(created=datetime.today() - timedelta(days=6)) self.add_install(created=datetime.today() - timedelta(days=6))
update_weekly_downloads() update_weekly_downloads()