reduce chunk size to eliminate timeouts. Also remove unused ratelimits

apps/stats/cron.py

@@ -28,7 +28,7 @@ def update_addons_collections_downloads():
          .annotate(sum=Sum('count')))
 
     ts = [tasks.update_addons_collections_downloads.subtask(args=[chunk])
-          for chunk in chunked(d, 600)]
+          for chunk in chunked(d, 250)]
     TaskSet(ts).apply_async()
 
 
@@ -40,7 +40,7 @@ def update_collections_total():
                                 .annotate(sum=Sum('count')))
 
     ts = [tasks.update_collections_total.subtask(args=[chunk])
-          for chunk in chunked(d, 1000)]
+          for chunk in chunked(d, 250)]
     TaskSet(ts).apply_async()
 
 
@@ -66,7 +66,7 @@ def update_global_totals(date=None):
 @cronjobs.register
 def addon_total_contributions():
     addons = Addon.objects.values_list('id', flat=True)
-    ts = [tasks.cron_total_contributions.subtask(args=chunk)
+    ts = [tasks.addon_total_contributions.subtask(args=chunk)
           for chunk in chunked(addons, 100)]
     TaskSet(ts).apply_async()
 

apps/stats/tasks.py

@@ -36,35 +36,29 @@ def addon_total_contributions(*addons, **kw):
         Addon.objects.filter(id=addon).update(total_contributions=total)
 
 
-@task(rate_limit='2/s')
+@task
-def cron_total_contributions(*addons, **kw):
-    "Rate limited version of `addon_total_contributions` suitable for cron."
-    addon_total_contributions(*addons)
-
-
-@task(rate_limit='2/s')
 def update_addons_collections_downloads(data, **kw):
-    log.info("[%s@%s] Updating addons+collections download totals." %
+    log.info("[%s] Updating addons+collections download totals." %
-                  (len(data), update_addons_collections_downloads.rate_limit))
+                  (len(data)))
     for var in data:
         (CollectionAddon.objects.filter(addon=var['addon'],
                                         collection=var['collection'])
                                 .update(downloads=var['sum']))
 
 
-@task(rate_limit='2/s')
+@task
 def update_collections_total(data, **kw):
-    log.info("[%s@%s] Updating collections' download totals." %
+    log.info("[%s] Updating collections' download totals." %
-                   (len(data), update_collections_total.rate_limit))
+                   (len(data)))
     for var in data:
         (Collection.objects.filter(pk=var['collection_id'])
          .update(downloads=var['sum']))
 
 
-@task(rate_limit='20/h')
+@task
 def update_global_totals(job, date, **kw):
-    log.info("[%s] Updating global statistics totals (%s) for (%s)" %
+    log.info("Updating global statistics totals (%s) for (%s)" %
-                   (update_global_totals.rate_limit, job, date))
+                   (job, date))
 
     jobs = _get_daily_jobs(date)
     jobs.update(_get_metrics_jobs(date))