Fix bug 1142850: Add cron job to update external files hourly.

bedrock/externalfiles/__init__.py

@@ -9,6 +9,7 @@ from os.path import abspath, basename, dirname, exists, join
 from datetime import datetime
 
 from django.conf import settings
+from django.core.cache import get_cache, InvalidCacheBackendError
 from django.utils.functional import cached_property
 from django.utils.http import parse_http_date_safe
 
@@ -21,12 +22,19 @@ UPDATED_FILE = '{0}.updated.txt'
 
 
 class ExternalFile(object):
+    cache_key = None
+
     def __init__(self, file_id):
         try:
             fileinfo = settings.EXTERNAL_FILES[file_id]
         except KeyError:
             raise ValueError('No external file with the {0} ID.'.format(file_id))
 
+        try:
+            self._cache = get_cache('externalfiles')
+        except InvalidCacheBackendError:
+            self._cache = get_cache('default')
+
         self.file_id = file_id
         self.url = fileinfo['url']
         self.name = fileinfo.get('name', basename(self.url))
@@ -131,3 +139,7 @@ class ExternalFile(object):
 
         log.info('Successfully updated {0}.'.format(self.name))
         return True
+
+    def clear_cache(self):
+        if self.cache_key:
+            self._cache.delete(self.cache_key)

bedrock/externalfiles/management/commands/update_externalfiles.py

@@ -22,13 +22,19 @@ class Command(BaseCommand):
                     dest='quiet',
                     default=False,
                     help='Do not print output to stdout.'),
+        make_option('--status',
+                    action='store_true',
+                    dest='status',
+                    default=False,
+                    help='Print only a final status to stdout. Mostly for scripts.')
     )
 
     def handle(self, *args, **options):
         file_ids = args or settings.EXTERNAL_FILES.keys()
+        updated = False
 
         def printout(msg, ending=None):
-            if not options['quiet']:
+            if not (options['quiet'] or options['status']):
                 self.stdout.write(msg, ending=ending)
 
         for fid in file_ids:
@@ -42,4 +48,11 @@ class Command(BaseCommand):
             if result is None:
                 printout('already up-to-date')
             else:
+                updated = True
                 printout('done')
+
+        if options['status']:
+            if updated:
+                self.stdout.write('updated')
+            else:
+                self.stdout.write('up-to-date')

bedrock/mozorg/credits.py

@@ -5,14 +5,14 @@
 import csv
 from operator import itemgetter
 
-from django.utils.functional import cached_property
-
 from ordereddict import OrderedDict
 
 from bedrock.externalfiles import ExternalFile
 
 
 class CreditsFile(ExternalFile):
+    cache_key = 'credits-file-sorted-names'
+
     def validate_content(self, content):
         rows = list(csv.reader(content.strip().encode('utf8').split('\n')))
         if len(rows) < 2200:  # it's 2273 as of now
@@ -23,7 +23,7 @@ class CreditsFile(ExternalFile):
 
         return content
 
-    @cached_property
+    @property
     def ordered(self):
         """
         Returns an OrderedDict of sorted lists of names by first letter of sortkey.
@@ -49,15 +49,20 @@ class CreditsFile(ExternalFile):
         :param credits_data: any iterable of CSV formatted strings.
         :return: list of lists
         """
-        names = []
-        for row in csv.reader(self.readlines()):
-            if len(row) == 1:
-                name = sortkey = row[0]
-            elif len(row) == 2:
-                name, sortkey = row
-            else:
-                continue
+        sorted_names = self._cache.get(self.cache_key)
+        if sorted_names is None:
+            names = []
+            for row in csv.reader(self.readlines()):
+                if len(row) == 1:
+                    name = sortkey = row[0]
+                elif len(row) == 2:
+                    name, sortkey = row
+                else:
+                    continue
 
-            names.append([name.decode('utf8'), sortkey.upper()])
+                names.append([name.decode('utf8'), sortkey.upper()])
 
-        return sorted(names, key=itemgetter(1))
+            sorted_names = sorted(names, key=itemgetter(1))
+            self._cache.set(self.cache_key, 3600)  # 1 hour
+
+        return sorted_names

bedrock/mozorg/tests/test_credits.py

@@ -13,6 +13,7 @@ from bedrock.mozorg.tests import TestCase
 class TestCredits(TestCase):
     def setUp(self):
         self.credits_file = credits.CreditsFile('credits')
+        self.credits_file.clear_cache()
 
     def test_credits_list(self):
         self.credits_file.readlines = Mock(return_value=[

bin/update-scripts/prod/update-externalfiles.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd /data/bedrock/src/www.mozilla.org-django
+
+return_value=$(venv/bin/python bedrock/manage.py update_externalfiles --status)
+
+if [ "$return_value" = "updated" ]; then
+    # file was updated, deploy
+    /data/bedrock/deploy www.mozilla.org-django/bedrock/bedrock/externalfiles/files_cache > /dev/null
+    echo "Successfully updated externalfiles."
+fi

bin/update-scripts/stage/update-externalfiles.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd /data/bedrock-stage/src/www.allizom.org-django
+
+return_value=$(venv/bin/python bedrock/manage.py update_externalfiles --status)
+
+if [ "$return_value" = "updated" ]; then
+    # file was updated, deploy
+    /data/bedrock/deploy www.allizom.org-django/bedrock/bedrock/externalfiles/files_cache > /dev/null
+    echo "Successfully updated externalfiles."
+fi

etc/cron.d/bedrock-prod.tmpl

@@ -10,6 +10,9 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 */10 * * * * {{ user }} {{ source }}/bin/update-scripts/prod/update-prod-php.sh {{ log }}
 */15 * * * * {{ user }} {{ source }}/bin/update-scripts/prod/update-prod-locale-cron.sh {{ log }}
 
+# bug 1142850
+0    * * * * {{ user }} {{ source }}/bin/update-scripts/prod/update-externalfiles.sh {{ log }}
+
 */5  * * * * {{ django_manage }} rnasync {{ log }}
 
 # bug 996144 & 1014586

etc/cron.d/bedrock-stage.tmpl

@@ -10,6 +10,9 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 */10 * * * * {{ user }} {{ source }}/bin/update-scripts/stage/update-stage-php.sh {{ log }}
 */15 * * * * {{ user }} {{ source }}/bin/update-scripts/stage/update-stage-locale.sh {{ log }}
 
+# bug 1142850
+0    * * * * {{ user }} {{ source }}/bin/update-scripts/stage/update-externalfiles.sh {{ log }}
+
 */5  * * * * {{ django_manage }} rnasync {{ log }}
 
 # bug 996144