From 5435b548ea2a1a4563f3009db29dfc1fc4d2e432 Mon Sep 17 00:00:00 2001
From: Paul McLanahan <pmac@mozilla.com>
Date: Fri, 13 Mar 2015 14:58:22 -0400
Subject: [PATCH] Fix bug 1142850: Add cron job to update external files
 hourly.

---
 bedrock/externalfiles/__init__.py             | 12 +++++++
 .../commands/update_externalfiles.py          | 15 ++++++++-
 bedrock/mozorg/credits.py                     | 31 +++++++++++--------
 bedrock/mozorg/tests/test_credits.py          |  1 +
 .../prod/update-externalfiles.sh              | 11 +++++++
 .../stage/update-externalfiles.sh             | 11 +++++++
 etc/cron.d/bedrock-prod.tmpl                  |  3 ++
 etc/cron.d/bedrock-stage.tmpl                 |  3 ++
 8 files changed, 73 insertions(+), 14 deletions(-)
 create mode 100644 bin/update-scripts/prod/update-externalfiles.sh
 create mode 100644 bin/update-scripts/stage/update-externalfiles.sh

diff --git a/bedrock/externalfiles/__init__.py b/bedrock/externalfiles/__init__.py
index 8e29afd055..c95bc38e77 100644
--- a/bedrock/externalfiles/__init__.py
+++ b/bedrock/externalfiles/__init__.py
@@ -9,6 +9,7 @@ from os.path import abspath, basename, dirname, exists, join
 from datetime import datetime
 
 from django.conf import settings
+from django.core.cache import get_cache, InvalidCacheBackendError
 from django.utils.functional import cached_property
 from django.utils.http import parse_http_date_safe
 
@@ -21,12 +22,19 @@ UPDATED_FILE = '{0}.updated.txt'
 
 
 class ExternalFile(object):
+    cache_key = None
+
     def __init__(self, file_id):
         try:
             fileinfo = settings.EXTERNAL_FILES[file_id]
         except KeyError:
             raise ValueError('No external file with the {0} ID.'.format(file_id))
 
+        try:
+            self._cache = get_cache('externalfiles')
+        except InvalidCacheBackendError:
+            self._cache = get_cache('default')
+
         self.file_id = file_id
         self.url = fileinfo['url']
         self.name = fileinfo.get('name', basename(self.url))
@@ -131,3 +139,7 @@ class ExternalFile(object):
 
         log.info('Successfully updated {0}.'.format(self.name))
         return True
+
+    def clear_cache(self):
+        if self.cache_key:
+            self._cache.delete(self.cache_key)
diff --git a/bedrock/externalfiles/management/commands/update_externalfiles.py b/bedrock/externalfiles/management/commands/update_externalfiles.py
index 28f3e9340e..e9f7a51e9b 100644
--- a/bedrock/externalfiles/management/commands/update_externalfiles.py
+++ b/bedrock/externalfiles/management/commands/update_externalfiles.py
@@ -22,13 +22,19 @@ class Command(BaseCommand):
                     dest='quiet',
                     default=False,
                     help='Do not print output to stdout.'),
+        make_option('--status',
+                    action='store_true',
+                    dest='status',
+                    default=False,
+                    help='Print only a final status to stdout. Mostly for scripts.')
     )
 
     def handle(self, *args, **options):
         file_ids = args or settings.EXTERNAL_FILES.keys()
+        updated = False
 
         def printout(msg, ending=None):
-            if not options['quiet']:
+            if not (options['quiet'] or options['status']):
                 self.stdout.write(msg, ending=ending)
 
         for fid in file_ids:
@@ -42,4 +48,11 @@ class Command(BaseCommand):
             if result is None:
                 printout('already up-to-date')
             else:
+                updated = True
                 printout('done')
+
+        if options['status']:
+            if updated:
+                self.stdout.write('updated')
+            else:
+                self.stdout.write('up-to-date')
diff --git a/bedrock/mozorg/credits.py b/bedrock/mozorg/credits.py
index 933d3738e5..53bec2c0fb 100644
--- a/bedrock/mozorg/credits.py
+++ b/bedrock/mozorg/credits.py
@@ -5,14 +5,14 @@
 import csv
 from operator import itemgetter
 
-from django.utils.functional import cached_property
-
 from ordereddict import OrderedDict
 
 from bedrock.externalfiles import ExternalFile
 
 
 class CreditsFile(ExternalFile):
+    cache_key = 'credits-file-sorted-names'
+
     def validate_content(self, content):
         rows = list(csv.reader(content.strip().encode('utf8').split('\n')))
         if len(rows) < 2200:  # it's 2273 as of now
@@ -23,7 +23,7 @@ class CreditsFile(ExternalFile):
 
         return content
 
-    @cached_property
+    @property
     def ordered(self):
         """
         Returns an OrderedDict of sorted lists of names by first letter of sortkey.
@@ -49,15 +49,20 @@ class CreditsFile(ExternalFile):
         :param credits_data: any iterable of CSV formatted strings.
         :return: list of lists
         """
-        names = []
-        for row in csv.reader(self.readlines()):
-            if len(row) == 1:
-                name = sortkey = row[0]
-            elif len(row) == 2:
-                name, sortkey = row
-            else:
-                continue
+        sorted_names = self._cache.get(self.cache_key)
+        if sorted_names is None:
+            names = []
+            for row in csv.reader(self.readlines()):
+                if len(row) == 1:
+                    name = sortkey = row[0]
+                elif len(row) == 2:
+                    name, sortkey = row
+                else:
+                    continue
 
-            names.append([name.decode('utf8'), sortkey.upper()])
+                names.append([name.decode('utf8'), sortkey.upper()])
 
-        return sorted(names, key=itemgetter(1))
+            sorted_names = sorted(names, key=itemgetter(1))
+            self._cache.set(self.cache_key, 3600)  # 1 hour
+
+        return sorted_names
diff --git a/bedrock/mozorg/tests/test_credits.py b/bedrock/mozorg/tests/test_credits.py
index 041d85c443..b094361214 100644
--- a/bedrock/mozorg/tests/test_credits.py
+++ b/bedrock/mozorg/tests/test_credits.py
@@ -13,6 +13,7 @@ from bedrock.mozorg.tests import TestCase
 class TestCredits(TestCase):
     def setUp(self):
         self.credits_file = credits.CreditsFile('credits')
+        self.credits_file.clear_cache()
 
     def test_credits_list(self):
         self.credits_file.readlines = Mock(return_value=[
diff --git a/bin/update-scripts/prod/update-externalfiles.sh b/bin/update-scripts/prod/update-externalfiles.sh
new file mode 100644
index 0000000000..13e18100f8
--- /dev/null
+++ b/bin/update-scripts/prod/update-externalfiles.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd /data/bedrock/src/www.mozilla.org-django
+
+return_value=$(venv/bin/python bedrock/manage.py update_externalfiles --status)
+
+if [ "$return_value" = "updated" ]; then
+    # file was updated, deploy
+    /data/bedrock/deploy www.mozilla.org-django/bedrock/bedrock/externalfiles/files_cache > /dev/null
+    echo "Successfully updated externalfiles."
+fi
diff --git a/bin/update-scripts/stage/update-externalfiles.sh b/bin/update-scripts/stage/update-externalfiles.sh
new file mode 100644
index 0000000000..5e0ac05427
--- /dev/null
+++ b/bin/update-scripts/stage/update-externalfiles.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd /data/bedrock-stage/src/www.allizom.org-django
+
+return_value=$(venv/bin/python bedrock/manage.py update_externalfiles --status)
+
+if [ "$return_value" = "updated" ]; then
+    # file was updated, deploy
+    /data/bedrock/deploy www.allizom.org-django/bedrock/bedrock/externalfiles/files_cache > /dev/null
+    echo "Successfully updated externalfiles."
+fi
diff --git a/etc/cron.d/bedrock-prod.tmpl b/etc/cron.d/bedrock-prod.tmpl
index 9e964984a4..8cb88771c5 100644
--- a/etc/cron.d/bedrock-prod.tmpl
+++ b/etc/cron.d/bedrock-prod.tmpl
@@ -10,6 +10,9 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 */10 * * * * {{ user }} {{ source }}/bin/update-scripts/prod/update-prod-php.sh {{ log }}
 */15 * * * * {{ user }} {{ source }}/bin/update-scripts/prod/update-prod-locale-cron.sh {{ log }}
 
+# bug 1142850
+0    * * * * {{ user }} {{ source }}/bin/update-scripts/prod/update-externalfiles.sh {{ log }}
+
 */5  * * * * {{ django_manage }} rnasync {{ log }}
 
 # bug 996144 & 1014586
diff --git a/etc/cron.d/bedrock-stage.tmpl b/etc/cron.d/bedrock-stage.tmpl
index aac83b7377..1467c677f1 100644
--- a/etc/cron.d/bedrock-stage.tmpl
+++ b/etc/cron.d/bedrock-stage.tmpl
@@ -10,6 +10,9 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 */10 * * * * {{ user }} {{ source }}/bin/update-scripts/stage/update-stage-php.sh {{ log }}
 */15 * * * * {{ user }} {{ source }}/bin/update-scripts/stage/update-stage-locale.sh {{ log }}
 
+# bug 1142850
+0    * * * * {{ user }} {{ source }}/bin/update-scripts/stage/update-externalfiles.sh {{ log }}
+
 */5  * * * * {{ django_manage }} rnasync {{ log }}
 
 # bug 996144