Add backend for fxos feed links: bug 1128587

bedrock/firefox/helpers.py

@@ -0,0 +1,23 @@
+from django.core.cache import cache
+from django.conf import settings
+
+import jingo
+
+from bedrock.firefox.models import FirefoxOSFeedLink
+
+
+@jingo.register.function
+def firefox_os_feed_links(locale, force_cache_refresh=False):
+    if locale in settings.FIREFOX_OS_FEED_LOCALES:
+        cache_key = 'firefox-os-feed-links-' + locale
+        if not force_cache_refresh:
+            links = cache.get(cache_key)
+            if links:
+                return links
+        links = list(
+            FirefoxOSFeedLink.objects.filter(locale=locale).order_by(
+                '-id').values_list('link', 'title')[:10])
+        cache.set(cache_key, links)
+        return links
+    elif '-' in locale:
+        return firefox_os_feed_links(locale.split('-')[0])

bedrock/firefox/migrations/0001_initial.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'FirefoxOSFeedLink'
+        db.create_table(u'firefox_firefoxosfeedlink', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('link', self.gf('django.db.models.fields.URLField')(max_length=2000)),
+            ('title', self.gf('django.db.models.fields.CharField')(max_length=2000)),
+            ('locale', self.gf('django.db.models.fields.CharField')(max_length=10, db_index=True)),
+        ))
+        db.send_create_signal(u'firefox', ['FirefoxOSFeedLink'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'FirefoxOSFeedLink'
+        db.delete_table(u'firefox_firefoxosfeedlink')
+
+
+    models = {
+        u'firefox.firefoxosfeedlink': {
+            'Meta': {'object_name': 'FirefoxOSFeedLink'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'link': ('django.db.models.fields.URLField', [], {'max_length': '2000'}),
+            'locale': ('django.db.models.fields.CharField', [], {'max_length': '10', 'db_index': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '2000'})
+        }
+    }
+
+    complete_apps = ['firefox']

bedrock/firefox/models.py

@@ -0,0 +1,10 @@
+from django.db import models
+
+
+class FirefoxOSFeedLink(models.Model):
+    link = models.URLField(max_length=2000)
+    title = models.CharField(max_length=2000)
+    locale = models.CharField(max_length=10, db_index=True)
+
+    def __unicode__(self):
+        return self.title

bedrock/firefox/templates/firefox/os/index-2015.html

@@ -0,0 +1,14 @@
+{% extends "firefox/base-resp.html" %}
+
+{% add_lang_files "firefox/os/devices" %}
+
+{% block content %}
+{% set feed_links = firefox_os_feed_links(LANG) %}
+{% if feed_links %}
+<ul>
+  {% for link, title in feed_links %}
+  <li><a href="{{ link }}">{{ title }}</a></li>
+  {% endfor %}
+</ul>
+{% endif %}
+{% endblock %}

bedrock/firefox/tests/test_base.py

@@ -3,7 +3,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import json
-
 import os
 from urlparse import parse_qsl, urlparse
 
@@ -11,7 +10,6 @@ from django.conf import settings
 from django.http import HttpResponse
 from django.test.client import Client, RequestFactory
 from django.test.utils import override_settings
-from django.utils import simplejson
 from funfactory.helpers import static
 
 from funfactory.urlresolvers import reverse
@@ -338,7 +336,7 @@ class TestFirefoxPartners(TestCase):
         self.assertEqual(resp.status_code, 400)
 
         # decode JSON response
-        resp_data = simplejson.loads(resp.content)
+        resp_data = json.loads(resp.content)
 
         self.assertEqual(resp_data['msg'], 'bad_request')
         self.assertTrue(post_patch.called)
@@ -353,8 +351,7 @@ class TestFirefoxPartners(TestCase):
             }, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
         self.assertEqual(resp.status_code, 400)
 
-        # decode JSON response
-        resp_data = simplejson.loads(resp.content)
+        resp_data = json.loads(resp.content)
 
         self.assertEqual(resp_data['msg'], 'Form invalid')
         self.assertFalse(post_patch.called)
@@ -375,8 +372,7 @@ class TestFirefoxPartners(TestCase):
             }, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
         self.assertEqual(resp.status_code, 200)
 
-        # decode JSON response
-        resp_data = simplejson.loads(resp.content)
+        resp_data = json.loads(resp.content)
 
         self.assertEqual(resp_data['msg'], 'ok')
         post_patch.assert_called_once_with(ANY, {

bedrock/firefox/tests/test_helpers.py

@@ -0,0 +1,53 @@
+from django.test.utils import override_settings
+
+from mock import patch
+from nose.tools import assert_false, eq_
+
+from bedrock.mozorg.tests import TestCase
+from bedrock.firefox import helpers
+
+
+@override_settings(FIREFOX_OS_FEED_LOCALES=['xx'])
+@patch('bedrock.firefox.helpers.cache')
+@patch('bedrock.firefox.helpers.FirefoxOSFeedLink')
+class FirefoxOSFeedLinksTest(TestCase):
+    def test_no_feed_for_locale(self, FirefoxOSFeedLink, cache):
+        """
+        Should return None without checking cache or db.
+        """
+        eq_(helpers.firefox_os_feed_links('yy'), None)
+        assert_false(cache.get.called)
+        assert_false(FirefoxOSFeedLink.objects.filter.called)
+
+    def test_force_cache_refresh(self, FirefoxOSFeedLink, cache):
+        """
+        Should force cache update of first 10 values without cache.get()
+        """
+
+        (FirefoxOSFeedLink.objects.filter.return_value.order_by.return_value
+         .values_list.return_value) = range(20)
+        eq_(helpers.firefox_os_feed_links('xx', force_cache_refresh=True),
+            range(10))
+        assert_false(cache.get.called)
+        FirefoxOSFeedLink.objects.filter.assert_called_with(locale='xx')
+        cache.set.assert_called_with('firefox-os-feed-links-xx', range(10))
+
+    def test_cache_miss(self, FirefoxOSFeedLink, cache):
+        """
+        Should update cache with first 10 items from db query
+        """
+        cache.get.return_value = None
+        (FirefoxOSFeedLink.objects.filter.return_value.order_by.return_value
+         .values_list.return_value) = range(20)
+        eq_(helpers.firefox_os_feed_links('xx'), range(10))
+        cache.get.assert_called_with('firefox-os-feed-links-xx')
+        FirefoxOSFeedLink.objects.filter.assert_called_with(locale='xx')
+        cache.set.assert_called_with('firefox-os-feed-links-xx', range(10))
+
+    def test_hyphenated_cached(self, FirefoxOSFeedLink, cache):
+        """
+        Should construct cache key with only first part of hyphenated locale.
+        """
+        eq_(helpers.firefox_os_feed_links('xx-xx'), cache.get.return_value)
+        cache.get.assert_called_with('firefox-os-feed-links-xx')
+        assert_false(FirefoxOSFeedLink.objects.filter.called)

bedrock/firefox/urls.py

@@ -73,7 +73,7 @@ urlpatterns = patterns('',
     # This dummy page definition makes it possible to link to /firefox/ (Bug 878068)
     url('^firefox/$', views.fx_home_redirect, name='firefox'),
 
-    page('firefox/os', 'firefox/os/index.html'),
+    url('^firefox/os/$', views.firefox_os_index, name='firefox.os.index'),
     page('firefox/os/releases', 'firefox/os/releases.html'),
 
 

bedrock/firefox/views.py

@@ -14,6 +14,8 @@ from django.views.decorators.vary import vary_on_headers
 from django.views.generic.base import TemplateView
 
 import basket
+import waffle
+
 from funfactory.helpers import static
 from funfactory.urlresolvers import reverse
 from lib import l10n_utils
@@ -223,6 +225,13 @@ def all_downloads(request, channel):
     return l10n_utils.render(request, 'firefox/all.html', context)
 
 
+def firefox_os_index(request):
+    if waffle.switch_is_active('firefox-os-index-2015'):
+        return l10n_utils.render(request, 'firefox/os/index-2015.html')
+    else:
+        return l10n_utils.render(request, 'firefox/os/index.html')
+
+
 @csrf_protect
 def firefox_partners(request):
     # If the current locale isn't in our list, return the en-US value

bedrock/settings/base.py

@@ -2216,3 +2216,15 @@ REST_FRAMEWORK = {
 
     'DEFAULT_FILTER_BACKENDS': ('rna.filters.TimestampedFilterBackend',)
 }
+
+FIREFOX_OS_FEEDS = (
+    ('de', 'https://blog.mozilla.org/press-de/category/firefox-os/feed/'),
+    ('en-US', 'https://blog.mozilla.org/blog/category/firefox-os/feed/'),
+    ('es-ES', 'https://blog.mozilla.org/press-es/category/firefox-os/feed/'),
+    ('es', 'https://blog.mozilla.org/press-latam/category/firefox-os/feed/'),
+    ('fr', 'https://blog.mozilla.org/press-fr/category/firefox-os/feed/'),
+    ('it', 'https://blog.mozilla.org/press-it/category/firefox-os/feed/'),
+    ('pl', 'https://blog.mozilla.org/press-pl/category/firefox-os/feed/'),
+    ('pt-BR', 'https://blog.mozilla.org/press-br/category/firefox-os/feed/'),
+)
+FIREFOX_OS_FEED_LOCALES = [feed[0] for feed in FIREFOX_OS_FEEDS]

etc/cron.d/bedrock-dev.tmpl

@@ -12,3 +12,6 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 # bug 1014586
 3 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
 
+# bug 1128587
+38 * * * * {{ django_manage }} runscript update_firefox_os_feeds > /dev/null 2>&1
+

etc/cron.d/bedrock-prod.tmpl

@@ -23,3 +23,6 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 # bug 1087533
 42 * * * * {{ django_cron }} update_reps_ical > /dev/null 2>&1
 
+# bug 1128587
+38 * * * * {{ django_manage }} runscript update_firefox_os_feeds > /dev/null 2>&1
+

etc/cron.d/bedrock-stage.tmpl

@@ -26,3 +26,6 @@ MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
 # bug 1087533
 42 * * * * {{ django_cron }} update_reps_ical > /dev/null 2>&1
 
+# bug 1128587
+38 * * * * {{ django_manage }} runscript update_firefox_os_feeds > /dev/null 2>&1
+

scripts/tests.py

@@ -0,0 +1,46 @@
+from django.test.utils import override_settings
+
+from mock import Mock, patch
+
+from bedrock.firefox.models import FirefoxOSFeedLink
+from bedrock.mozorg.tests import TestCase
+
+from scripts import update_firefox_os_feeds
+
+
+class TestUpdateFirefoxOSFeeds(TestCase):
+    @override_settings(FIREFOX_OS_FEEDS=[('xx', 'http://example.com/feed'),
+                                         ('yy', 'http://example.com/feed2')])
+    @patch('scripts.update_firefox_os_feeds.create_or_update_fxos_feed_links')
+    @patch('scripts.update_firefox_os_feeds.parse')
+    def test_run_exception_handling(self, parse, create_or_update_fxos_feed_links):
+        parse.side_effect = [Exception('test'), 'parsed feed']
+        update_firefox_os_feeds.run()
+        create_or_update_fxos_feed_links.assert_called_with('yy', 'parsed feed')
+
+    @patch('scripts.update_firefox_os_feeds.create_or_update_fxos_feed_link')
+    def test_create_or_update_fxos_feed_links(
+            self, create_or_update_fxos_feed_link):
+        update_firefox_os_feeds.create_or_update_fxos_feed_links(
+            'xx',
+            {'entries': [{}, {'link': 'http://example.com', 'title': 'Title'}]})
+        create_or_update_fxos_feed_link.assert_called_with(
+            'xx', 'http://example.com', 'Title')
+
+    @patch('scripts.update_firefox_os_feeds.FirefoxOSFeedLink.objects')
+    def test_create_or_update_fxos_feed_link(self, objects):
+        objects.get_or_create.return_value = ('feed_link', True)
+        update_firefox_os_feeds.create_or_update_fxos_feed_link(
+            'xx', 'http://example.com', 'Title')
+        objects.get_or_create.assert_called_with(
+            locale='xx', link='http://example.com', defaults={'title': 'Title'})
+
+    @patch('scripts.update_firefox_os_feeds.FirefoxOSFeedLink.objects')
+    def test_create_or_update_fxos_feed_link_update_title(self, objects):
+        feed_link = Mock(title='Old Title')
+        objects.get_or_create.return_value = (feed_link, False)
+        update_firefox_os_feeds.create_or_update_fxos_feed_link(
+            'xx', 'http://example.com', 'Title')
+        objects.get_or_create.assert_called_with(
+            locale='xx', link='http://example.com', defaults={'title': 'Title'})
+        feed_link.save.assert_called_with(update_fields=['title'])

scripts/update_firefox_os_feeds.py

@@ -0,0 +1,33 @@
+from __future__ import print_function
+
+from django.conf import settings
+
+from feedparser import parse
+
+from bedrock.firefox.models import FirefoxOSFeedLink
+
+
+def create_or_update_fxos_feed_link(locale, link, title):
+    feed_link, created = FirefoxOSFeedLink.objects.get_or_create(
+        locale=locale, link=link, defaults={'title': title}) 
+    if not created and title != feed_link.title:
+        feed_link.title = title
+        feed_link.save(update_fields=['title'])
+
+
+def create_or_update_fxos_feed_links(locale, parsed_feed):
+    for entry in parsed_feed.get('entries', []):
+        link = entry.get('link')
+        title = entry.get('title')
+        if link and title:
+            create_or_update_fxos_feed_link(locale, link, title)
+
+
+def run(*args):
+    for locale, url in settings.FIREFOX_OS_FEEDS:
+        try:
+            parsed_feed = parse(url)
+        except:
+            print('Unable to fetch and parse ' + url)
+        else:
+            create_or_update_fxos_feed_links(locale, parsed_feed)