a cronjob to clean up redis (bug 616865)

apps/amo/management/commands/clean_redis.py

@@ -0,0 +1,73 @@
+import itertools
+import logging
+import socket
+import time
+
+from django.core.management.base import BaseCommand
+
+import redisutils
+import redis as redislib
+
+log = logging.getLogger('z.redis')
+
+# We process the keys in chunks of size CHUNK.
+CHUNK = 3000
+# Remove any sets with less than MIN or more than MAX elements.
+MIN = 2
+MAX = 75
+# Expire keys after EXPIRE seconds.
+EXPIRE = 60 * 5
+
+# Calling redis can raise raise these errors.
+RedisError = redislib.RedisError, socket.error
+
+
+def vacuum(master, slave):
+    total = [1, 0]
+
+    def keys():
+        ks = slave.keys()
+        total[0] = len(ks)
+        log.info('There are %s keys to clean up.' % total[0])
+        ks = iter(ks)
+        while 1:
+            yield [ks.next() for _ in xrange(CHUNK)]
+
+    count = itertools.count()
+    for ks in keys():
+        pipe = slave.pipeline()
+        for k in ks:
+            pipe.scard(k)
+        try:
+            drop = [k for k, size in zip(ks, pipe.execute())
+                    if size < MIN or size > MAX]
+        except RedisError:
+            continue
+        num = count.next() * CHUNK
+        percent = round(float(num) / total[0] * 100, 1)
+        total[1] += len(drop)
+        log.debug('[%s %.1f%%] Dropping %s keys.' % (num, percent, len(drop)))
+        pipe = master.pipeline()
+        for k in drop:
+            pipe.expire(k, EXPIRE)
+        try:
+            pipe.execute()
+        except RedisError:
+            continue
+        time.sleep(1)  # Poor man's rate limiting.
+
+    log.info('Dropped %s keys [%.1f%%].' %
+              (total[1], round(float(total[1]) / total[0] * 100, 1)))
+
+
+class Command(BaseCommand):
+    help = "Clean up the redis used by cache machine."
+
+    def handle(self, *args, **kw):
+        try:
+            master = redisutils.connections['cache']
+            slave = redisutils.connections['cache_slave']
+        except Exception:
+            log.error('Could not connect to redis.', exc_info=True)
+            return
+        vacuum(master, slave)

scripts/crontab/make-crons.py

@@ -79,6 +79,7 @@ HOME = /tmp
 30 1 * * * $Z_CRON update_user_ratings
 30 2 * * * $Z_CRON addon_reviews_ratings
 30 3 * * * $DJANGO cleanup
+30 4 * * * $DJANGO clean_redis
 30 5 * * * $REMORA; php -f maintenance.php expired_resetcode
 30 6 * * * $REMORA; php -f maintenance.php category_totals
 30 7 * * * $REMORA; php -f maintenance.php collection_subscribers

scripts/crontab/preview

@@ -38,6 +38,7 @@ HOME = /tmp
 30 1 * * * cd /data/amo_python/src/preview/zamboni; /usr/bin/python26 manage.py cron update_user_ratings
 30 2 * * * cd /data/amo_python/src/preview/zamboni; /usr/bin/python26 manage.py cron addon_reviews_ratings
 30 3 * * * cd /data/amo_python/src/preview/zamboni; /usr/bin/python26 manage.py cleanup
+30 4 * * * cd /data/amo_python/src/preview/zamboni; /usr/bin/python26 manage.py clean_redis
 30 5 * * * cd /data/amo/www/addons.mozilla.org-preview/bin; php -f maintenance.php expired_resetcode
 30 6 * * * cd /data/amo/www/addons.mozilla.org-preview/bin; php -f maintenance.php category_totals
 30 7 * * * cd /data/amo/www/addons.mozilla.org-preview/bin; php -f maintenance.php collection_subscribers

scripts/crontab/prod

@@ -38,6 +38,7 @@ HOME = /tmp
 30 1 * * * apache cd /data/amo_python/src/prod/zamboni; /usr/bin/python26 manage.py cron update_user_ratings
 30 2 * * * apache cd /data/amo_python/src/prod/zamboni; /usr/bin/python26 manage.py cron addon_reviews_ratings
 30 3 * * * apache cd /data/amo_python/src/prod/zamboni; /usr/bin/python26 manage.py cleanup
+30 4 * * * apache cd /data/amo_python/src/prod/zamboni; /usr/bin/python26 manage.py clean_redis
 30 5 * * * apache cd /data/amo/www/addons.mozilla.org-remora/bin; php -f maintenance.php expired_resetcode
 30 6 * * * apache cd /data/amo/www/addons.mozilla.org-remora/bin; php -f maintenance.php category_totals
 30 7 * * * apache cd /data/amo/www/addons.mozilla.org-remora/bin; php -f maintenance.php collection_subscribers