Increase frequency of auto_approve command, prevent multiple runs in parallel (#5751)

Increase frequency of auto_approve command, prevent multiple runs in parallel

scripts/crontab/crontab.tpl

@@ -5,10 +5,12 @@ DJANGO_SETTINGS_MODULE='settings_local'
 
 HOME=/tmp
 
+# Every 10 minutes
+*/10 * * * * %(django)s auto_approve
+
 #once per hour
 5 * * * * %(z_cron)s update_collections_subscribers
 10 * * * * %(z_cron)s update_blog_posts
-15 * * * * %(django)s auto_approve
 20 * * * * %(z_cron)s addon_last_updated
 25 * * * * %(z_cron)s update_collections_votes
 45 * * * * %(z_cron)s update_addon_appsupport

src/olympia/editors/management/commands/auto_approve.py

@@ -14,12 +14,15 @@ from olympia.editors.helpers import ReviewHelper
 from olympia.editors.models import (
     AutoApprovalNotEnoughFilesError, AutoApprovalNoValidationResultError,
     AutoApprovalSummary, clear_reviewing_cache, set_reviewing_cache)
+from olympia.files.utils import atomic_lock
 from olympia.versions.models import Version
 from olympia.zadmin.models import get_config
 
 
 log = olympia.core.logger.getLogger('z.editors.auto_approve')
 
+LOCK_NAME = 'auto-approve'  # Name of the atomic_lock() used.
+
 
 class Command(BaseCommand):
     help = 'Auto-approve add-ons based on predefined criteria'
@@ -76,13 +79,22 @@ class Command(BaseCommand):
             else amo.AUTO_APPROVED)
 
         self.stats = Counter()
-        qs = self.fetch_candidates()
-        self.stats['total'] = len(qs)
 
-        for version in qs:
-            self.process(version)
+        # Get a lock before doing anything, we don't want to have multiple
+        # instances of the command running in parallel.
+        lock = atomic_lock(settings.TMP_PATH, LOCK_NAME, lifetime=15 * 60)
+        with lock as lock_attained:
+            if lock_attained:
+                qs = self.fetch_candidates()
+                self.stats['total'] = len(qs)
 
-        self.log_final_summary(self.stats)
+                for version in qs:
+                    self.process(version)
+
+                self.log_final_summary(self.stats)
+            else:
+                # We didn't get the lock...
+                log.error('auto-approve lock present, aborting.')
 
     @transaction.atomic
     def process(self, version):

src/olympia/editors/tests/test_commands.py

@@ -18,6 +18,7 @@ from olympia.editors.models import (
     AutoApprovalNotEnoughFilesError, AutoApprovalNoValidationResultError,
     AutoApprovalSummary, get_reviewing_cache)
 from olympia.files.models import FileValidation
+from olympia.files.utils import atomic_lock
 from olympia.zadmin.models import Config, get_config, set_config
 
 
@@ -406,3 +407,12 @@ class TestAutoApproveCommand(TestCase):
             'too_many_average_daily_users': 1,
             'too_few_approved_updates': 1,
         })
+
+    def test_prevent_multiple_runs_in_parallel(self):
+        # Create a lock manually, the command should exit immediately without
+        # doing anything.
+        with atomic_lock(settings.TMP_PATH, auto_approve.LOCK_NAME):
+            call_command('auto_approve')
+
+        assert self.log_final_summary_mock.call_count == 0
+        assert self.file.reload().status == amo.STATUS_AWAITING_REVIEW