remove Approvals > ActivityLog and deduping (bug 647261)
This commit is contained in:
Andy McKay
Родитель
29baf30455
Коммит
91f8203b95
|
|
@ -25,7 +25,7 @@ from bandwagon.models import Collection
|
|||
from cake.models import Session
|
||||
from devhub.models import ActivityLog, LegacyAddonLog
|
||||
from editors.models import EventLog
|
||||
from files.models import Approval, File, TestResultCache
|
||||
from files.models import TestResultCache
|
||||
from reviews.models import Review
|
||||
from sharing import SERVICES_LIST
|
||||
from stats.models import AddonShareCount, Contribution
|
||||
|
|
@ -290,53 +290,6 @@ def _migrate_editor_eventlog(items, **kw):
|
|||
log.warning("Couldn't find review for %d" % item.changed_id)
|
||||
|
||||
|
||||
# TODO(davedash): remove aftr /editors is on zamboni
|
||||
@cronjobs.register
|
||||
def migrate_approvals():
|
||||
a = MigrationTracker('approvals')
|
||||
id = a.get()
|
||||
if not id:
|
||||
log.warning('No last position reported from redis for '
|
||||
'migrating approvals, exiting.')
|
||||
return
|
||||
|
||||
items = (Approval.objects.filter(pk__gt=id).order_by('id')
|
||||
.values_list('id', flat=True))
|
||||
|
||||
for chunk in chunked(items, 100):
|
||||
_migrate_approvals(chunk)
|
||||
a.set(chunk[-1])
|
||||
|
||||
|
||||
@task
|
||||
def _migrate_approvals(items, **kw):
|
||||
log.info('[%s@%s] Migrating approval items starting with id: %s' %
|
||||
(len(items), _migrate_approvals.rate_limit, items[0]))
|
||||
for item in Approval.objects.filter(pk__in=items):
|
||||
try:
|
||||
args = (item.addon, item.file.version)
|
||||
except File.DoesNotExist:
|
||||
log.warning("Couldn't find file for approval %d" % item.id)
|
||||
continue
|
||||
|
||||
kw = dict(user=item.user, created=item.created,
|
||||
details=dict(comments=item.comments,
|
||||
reviewtype=item.reviewtype,
|
||||
source=item.pk))
|
||||
if item.action == amo.STATUS_PUBLIC:
|
||||
amo.log(amo.LOG.APPROVE_VERSION, *args, **kw)
|
||||
elif item.action == amo.STATUS_LITE:
|
||||
amo.log(amo.LOG.PRELIMINARY_VERSION, *args, **kw)
|
||||
elif item.action == amo.STATUS_NULL:
|
||||
amo.log(amo.LOG.REJECT_VERSION, *args, **kw)
|
||||
elif item.action in (amo.STATUS_PENDING, amo.STATUS_NOMINATED):
|
||||
amo.log(amo.LOG.ESCALATE_VERSION, *args, **kw)
|
||||
elif item.action == amo.STATUS_UNREVIEWED:
|
||||
amo.log(amo.LOG.RETAIN_VERSION, *args, **kw)
|
||||
else:
|
||||
log.warning('Unknown action: %d' % item.action)
|
||||
|
||||
|
||||
@cronjobs.register
|
||||
def dissolve_outgoing_urls():
|
||||
"""Over time, some outgoing.m.o URLs have been encoded several times in the
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
import json
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
import commonware.log
|
||||
|
|
@ -7,10 +5,6 @@ import celery.task
|
|||
from celeryutils import task
|
||||
from hera.contrib.django_utils import flush_urls
|
||||
|
||||
from addons.models import Addon
|
||||
from devhub.models import ActivityLog
|
||||
from editors.helpers import LOG_STATUSES
|
||||
|
||||
|
||||
log = commonware.log.getLogger('z.task')
|
||||
|
||||
|
|
@ -41,44 +35,3 @@ def flush_front_end_cache_urls(urls, **kw):
|
|||
urls[index] = u"%s%s" % (settings.SITE_URL, url)
|
||||
|
||||
flush_urls(urls)
|
||||
|
||||
|
||||
@task
|
||||
def dedupe_approvals(items, **kw):
|
||||
log.info('[%s@%s] Deduping approval items starting with addon: %s' %
|
||||
(len(items), dedupe_approvals.rate_limit, items[0]))
|
||||
for addon in Addon.objects.filter(pk__in=items):
|
||||
last = {}
|
||||
qs = (ActivityLog.objects.for_addons(addon)
|
||||
.order_by('-created')
|
||||
.filter(action__in=LOG_STATUSES))
|
||||
log.info('Found %d logs for addon %d ordered by -created'
|
||||
% (qs.count(), addon.pk))
|
||||
for activity in qs:
|
||||
arguments = json.loads(activity._arguments)
|
||||
current = {
|
||||
'action': activity.action,
|
||||
'created': activity.created.date(),
|
||||
'user': activity.user.pk,
|
||||
'addon': arguments[0]['addons.addon'],
|
||||
'version': arguments[1]['versions.version'],
|
||||
}
|
||||
if activity._details:
|
||||
details = json.loads(activity._details)
|
||||
current.update({
|
||||
'reviewtype': details['reviewtype'],
|
||||
'comments': details['comments'],
|
||||
})
|
||||
|
||||
if last and last == current:
|
||||
log.info('Deleting duplicate activity log %s '
|
||||
'from addon %s' % (activity.pk, addon.pk))
|
||||
activity.delete()
|
||||
else:
|
||||
if last:
|
||||
diff = dict(set(current.iteritems()) -
|
||||
set(last.iteritems())).keys()
|
||||
log.info('Not deleting duplicate activity log %s '
|
||||
'from addon %d difference found in %s' %
|
||||
(activity.pk, addon.pk, diff))
|
||||
last = current.copy()
|
||||
|
|
|
|||
|
|
@ -5,15 +5,12 @@ import test_utils
|
|||
|
||||
import amo
|
||||
from amo.cron import gc, remove_extra_cats
|
||||
from amo.tasks import dedupe_approvals
|
||||
from addons.models import Addon, AddonCategory, Category
|
||||
from bandwagon.models import Collection
|
||||
from cake.models import Session
|
||||
from devhub.models import ActivityLog
|
||||
from files.models import TestResult, TestResultCache
|
||||
from stats.models import AddonShareCount, Contribution
|
||||
from users.models import UserProfile
|
||||
from versions.models import Version
|
||||
|
||||
|
||||
class GarbageTest(test_utils.TestCase):
|
||||
|
|
@ -84,47 +81,3 @@ class RemoveExtraCatTest(test_utils.TestCase):
|
|||
eq_(self.addon.categories.count(), 2)
|
||||
remove_extra_cats()
|
||||
eq_(self.addon.categories.count(), 2)
|
||||
|
||||
|
||||
class TestDedupeApprovals(test_utils.TestCase):
|
||||
fixtures = ['base/users']
|
||||
|
||||
def setUp(self):
|
||||
self.addon = Addon.objects.create(type=amo.ADDON_EXTENSION)
|
||||
self.version = Version.objects.create(addon=self.addon)
|
||||
amo.set_user(UserProfile.objects.get(username='editor'))
|
||||
|
||||
def test_dedupe(self):
|
||||
for x in range(0, 4):
|
||||
amo.log(amo.LOG.APPROVE_VERSION, self.addon, self.version)
|
||||
dedupe_approvals([self.addon.pk])
|
||||
eq_(ActivityLog.objects.for_addons(self.addon).count(), 1)
|
||||
|
||||
def test_dedupe_mix(self):
|
||||
for x in range(0, 4):
|
||||
amo.log(amo.LOG.APPROVE_VERSION, self.addon, self.version)
|
||||
for x in range(0, 3):
|
||||
amo.log(amo.LOG.REJECT_VERSION, self.addon, self.version)
|
||||
for x in range(0, 5):
|
||||
amo.log(amo.LOG.APPROVE_VERSION, self.addon, self.version)
|
||||
dedupe_approvals([self.addon.pk])
|
||||
eq_(ActivityLog.objects.for_addons(self.addon).count(), 3)
|
||||
|
||||
def test_dedupe_date(self):
|
||||
# Test that a log spanning
|
||||
old = amo.log(amo.LOG.APPROVE_VERSION, self.addon, self.version)
|
||||
old.update(created=datetime.today() - timedelta(days=1))
|
||||
amo.log(amo.LOG.APPROVE_VERSION, self.addon, self.version)
|
||||
dedupe_approvals([self.addon.pk])
|
||||
eq_(ActivityLog.objects.for_addons(self.addon).count(), 2)
|
||||
|
||||
def test_dedupe_not_id(self):
|
||||
date_one = datetime.today()
|
||||
date_two = datetime.today() - timedelta(days=1)
|
||||
for x in range(0, 4):
|
||||
log = amo.log(amo.LOG.APPROVE_VERSION, self.addon, self.version)
|
||||
log.update(created=date_one)
|
||||
log = amo.log(amo.LOG.REJECT_VERSION, self.addon, self.version)
|
||||
log.update(created=date_two)
|
||||
dedupe_approvals([self.addon.pk])
|
||||
eq_(ActivityLog.objects.for_addons(self.addon).count(), 2)
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ for dict_ in CRONS.values():
|
|||
break
|
||||
|
||||
|
||||
# TODO(andym) remove migrate_approvals when zamboni editor tools are live
|
||||
cron = """\
|
||||
#
|
||||
# !!AUTO-GENERATED!! Edit scripts/crontab/make-crons.py instead.
|
||||
|
|
@ -54,7 +53,6 @@ HOME = /tmp
|
|||
|
||||
# Every 20 minutes.
|
||||
*/20 * * * * $Z_CRON check_queues
|
||||
*/20 * * * * $Z_CRON migrate_approvals
|
||||
|
||||
# Every 30 minutes.
|
||||
*/30 * * * * $Z_CRON tag_jetpacks
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче