From d6f53ca38f42413b9d6287f8fd344406cd921bf5 Mon Sep 17 00:00:00 2001 From: Dave Dash Date: Wed, 24 Nov 2010 10:08:00 -0800 Subject: [PATCH] bug 592591, addonlogs->activity_log importer --- apps/addons/utils.py | 18 +++++++++++ apps/amo/cron.py | 69 ++++++++++++++++++++++++++++++++++++------- apps/amo/log.py | 35 ++++++++++++---------- apps/devhub/models.py | 1 + 4 files changed, 98 insertions(+), 25 deletions(-) diff --git a/apps/addons/utils.py b/apps/addons/utils.py index b25dec38cb..f8200af828 100644 --- a/apps/addons/utils.py +++ b/apps/addons/utils.py @@ -26,6 +26,8 @@ def add_redis(f): try: import redisutils + # TODO(davedash): This should be our persistence layer when that's + # set in production. redis = redisutils.connections['master'] pipe = redis.pipeline(transaction=True) ret = f(cls, redis, pipe, *args, **kw) @@ -77,3 +79,19 @@ class ReverseNameLookup(object): for hash in hashes: pipe.delete('%s:%s' % (cls.prefix, hash)) pipe.delete('%s:%d' % (cls.prefix, addon_id)) + + +#TODO(davedash): remove after remora +class ActivityLogMigrationTracker(object): + """This tracks what id of the addonlog we're on.""" + key = 'amo:activitylog:migration' + + @add_redis + def __init__(self, redis, pipe): + self.redis = redis + + def get(self): + return self.redis.get(self.key) + + def set(self, value): + return self.redis.set(self.key, value) diff --git a/apps/amo/cron.py b/apps/amo/cron.py index bf0ebca238..d3312a74a7 100644 --- a/apps/amo/cron.py +++ b/apps/amo/cron.py @@ -4,16 +4,21 @@ from subprocess import Popen, PIPE from django.conf import settings +from celeryutils import task import cronjobs import commonware.log import amo +from amo.utils import chunked +from addons.utils import ActivityLogMigrationTracker from bandwagon.models import Collection from cake.models import Session -from devhub.models import ActivityLog -from files.models import TestResult, TestResultCache +from devhub.models import ActivityLog, LegacyAddonLog +from files.models import TestResultCache from sharing import SERVICES_LIST from stats.models import AddonShareCount, Contribution +from users.models import UserProfile +from versions.models import Version log = commonware.log.getLogger('z.cron') @@ -36,14 +41,6 @@ def gc(test_result=True): AddonShareCount.objects.exclude( service__in=[s.shortname for s in SERVICES_LIST]).delete() - # XXX(davedash): I can't seem to run this during testing without triggering - # an error: "test_remora.nose_c doesn't exist" - # for some reason a ForeignKey attaches itself to TestResult during testing - # I suspect it's the name, but I don't have time to really figure this out. - if test_result: - log.debug('Cleaning up test results.') - TestResult.objects.filter(created__lt=one_hour_ago).delete() - log.debug('Cleaning up test results cache.') TestResultCache.objects.filter(date__lt=one_hour_ago).delete() @@ -98,3 +95,55 @@ def gc(test_result=True): log.debug('Cleaning up anonymous collections.') Collection.objects.filter(created__lt=two_days_ago, type=amo.COLLECTION_ANONYMOUS).delete() + + +@cronjobs.register +def migrate_logs(): + # Get the highest id we've looked at. + a = ActivityLogMigrationTracker() + id = a.get() or 0 + + items = LegacyAddonLog.objects.filter(pk__gt=id).values_list( + 'id', flat=True) + for chunk in chunked(items, 100): + _migrate_logs.delay(chunk) + a.set(chunk[-1]) + + +@task +def _migrate_logs(items, **kw): + print 'Processing: %d..%d' % (items[0], items[-1]) + for item in LegacyAddonLog.objects.filter(pk__in=items): + kw = dict(user=item.user, created=item.created) + if item.type not in amo.LOG_KEEP: + continue + elif item.type in [amo.LOG.CREATE_ADDON.id, amo.LOG.SET_INACTIVE.id, + amo.LOG.UNSET_INACTIVE.id, + amo.LOG.SET_PUBLIC_STATS.id, + amo.LOG.UNSET_PUBLIC_STATS.id, + amo.LOG.ADD_RECOMMENDED.id, + amo.LOG.REMOVE_RECOMMENDED.id]: + amo.log(amo.LOG_BY_ID[item.type], item.addon, **kw) + + elif item.type in [amo.LOG.ADD_USER_WITH_ROLE.id, + amo.LOG.REMOVE_USER_WITH_ROLE.id]: + amo.log(amo.LOG_BY_ID[item.type], item.addon, + (UserProfile, item.object1_id), + unicode(dict(amo.AUTHOR_CHOICES)[item.object2_id]), **kw) + elif item.type == amo.LOG.CHANGE_STATUS.id: + amo.log(amo.LOG_BY_ID[item.type], item.addon, + unicode(amo.STATUS_CHOICES[item.object1_id]), **kw) + # Items that require only a version + elif item.type in [amo.LOG.ADD_VERSION.id, + amo.LOG.APPROVE_VERSION.id, + amo.LOG.RETAIN_VERSION.id, + amo.LOG.ESCALATE_VERSION.id, + amo.LOG.REQUEST_VERSION.id]: + try: + v = Version.objects.get(pk=item.object1_id) + amo.log(amo.LOG_BY_ID[item.type], item.addon, v, **kw) + except Version.DoesNotExist: + print ('Version %d does not exist. No worries, it happens.' + % item.object1_id) + elif item.type == amo.LOG.DELETE_VERSION.id: + amo.log(amo.LOG_BY_ID[item.type], item.addon, item.name1, **kw) diff --git a/apps/amo/log.py b/apps/amo/log.py index 5ec1ad42ac..b6775a5dbc 100644 --- a/apps/amo/log.py +++ b/apps/amo/log.py @@ -51,20 +51,20 @@ class EDIT_CONTRIBUTIONS: class USER_DISABLE: id = 8 - format = _(u'{user.name} disabled addon {addon}') + format = _(u'{addon} set inactive.') keep = True class USER_ENABLE: id = 9 - format = _(u'{user.name} enabled addon {addon}') + format = _(u'{addon} activated.') keep = True # TODO(davedash): Log these types when pages are present class SET_PUBLIC_STATS: id = 10 - format = _(u'{user.name} set stats public for {addon}') + format = _(u'Stats set public for {addon}.') keep = True @@ -79,7 +79,7 @@ class UNSET_PUBLIC_STATS: class CHANGE_STATUS: id = 12 # L10n: {0} is the status - format = _(u'{user.name} changed {addon} status to {0}') + format = _(u'{addon} status changed to {0}') keep = True @@ -114,7 +114,9 @@ class EDIT_VERSION: class DELETE_VERSION: id = 18 - format = _(u'{user.name} deleted version {0} from {addon}') + # Note, {0} is a string not a version since the version is deleted. + # L10n: {0} is the version number + format = _(u'Version {0} deleted from {addon}.') keep = True @@ -137,14 +139,14 @@ class DELETE_FILE_FROM_VERSION: # TODO(davedash): When editor tools exist class APPROVE_VERSION: id = 21 - format = _(u'Version {0.version} of {addon} approved') + format = _(u'{addon} {version} approved.') keep = True # TODO(davedash): When editor tools exist class RETAIN_VERSION: id = 22 - format = _(u'{user.name} retained version {0.version} of {addon}') + format = _(u'{addon} {version} retained.') keep = True @@ -152,7 +154,7 @@ class RETAIN_VERSION: class ESCALATE_VERSION: id = 23 # L10n: {0.version} is the version of an addon. - format = _(u'{user.name} escalated review of {addon} {0.version}') + format = _(u'Review escalated for {addon} {version}.') keep = True @@ -160,8 +162,7 @@ class ESCALATE_VERSION: class REQUEST_VERSION: id = 24 # L10n: {0.version} is the version of an addon. - format = _(u'{user.name} requested more information regarding ' - '{addon} {0.version}') + format = _(u'More information regarding {addon} {version} was requested.') keep = True @@ -205,13 +206,13 @@ class REMOVE_RECOMMENDED_CATEGORY: class ADD_RECOMMENDED: id = 33 - format = _(u'{addon} is now featured') + format = _(u'{addon} is now featured.') keep = True class REMOVE_RECOMMENDED: id = 34 - format = _(u'{addon} is no longer featured') + format = _(u'{addon} is no longer featured.') keep = True @@ -265,10 +266,10 @@ LOGS = (CREATE_ADDON, EDIT_PROPERTIES, EDIT_DESCRIPTIONS, EDIT_CATEGORIES, ) LOG_BY_ID = dict((l.id, l) for l in LOGS) LOG = AttributeDict((l.__name__, l) for l in LOGS) -LOG_KEEP = (l.id for l in LOGS if hasattr(l, 'keep')) +LOG_KEEP = [l.id for l in LOGS if hasattr(l, 'keep')] -def log(action, *args): +def log(action, *args, **kw): """ e.g. amo.log(amo.LOG.CREATE_ADDON, []), amo.log(amo.LOG.ADD_FILE_TO_VERSION, file, version) @@ -278,7 +279,7 @@ def log(action, *args): from users.models import UserProfile from amo import get_user, logger_log - user = get_user() + user = kw.get('user', get_user()) if not user: logger_log.warning('Activity log called with no user: %s' % action.id) @@ -287,6 +288,10 @@ def log(action, *args): al = ActivityLog(user=user, action=action.id) al.arguments = args al.save() + if 'created' in kw: + al.created = kw['created'] + # Double save necessary since django resets the created date on save. + al.save() for arg in args: if isinstance(arg, Addon): diff --git a/apps/devhub/models.py b/apps/devhub/models.py index 4c78f96921..e6c67199cf 100644 --- a/apps/devhub/models.py +++ b/apps/devhub/models.py @@ -234,6 +234,7 @@ class LegacyAddonLog(models.Model): class Meta: db_table = 'addonlogs' + ordering = ('id',) class SubmitStep(models.Model):