Capturing so, so many encoding errors. Also added positional parameters to the |fe filter (because they were missing).

apps/search/clients.py

@@ -2,6 +2,7 @@ import logging
 import socket
 
 from django.conf import settings
+from django.utils.encoding import smart_unicode
 
 from .sphinxapi import SphinxClient
 
@@ -88,7 +89,6 @@ class SearchClient(object):
                 sc.SetFilter(f['filter'], f['value'],
                              f.get('exclude', False))
 
-
         try:
             result = sc.Query(query, self.index)
         except socket.timeout:
@@ -120,7 +120,7 @@ class SearchClient(object):
             {'limit': settings.SEARCH_SUMMARY_LENGTH
                 * settings.SEARCH_SUMMARY_LENGTH_MULTIPLIER})[0]
 
-        excerpt = raw_excerpt
+        excerpt = smart_unicode(raw_excerpt)
         for p in self.compiled_patterns:
             excerpt = p[0].sub(p[1], excerpt)
 
@@ -130,7 +130,7 @@ class SearchClient(object):
                 + self.truncate_pattern.sub('',
                     excerpt[settings.SEARCH_SUMMARY_LENGTH:])
             if excerpt[-1] != '.':
-                excerpt += '...'
+                excerpt += u'...'
 
         return excerpt
 

apps/search/helpers.py

@@ -1,5 +1,5 @@
 from django.conf import settings
-from django.utils.encoding import force_unicode
+from django.utils.encoding import smart_unicode
 
 import jinja2
 from jingo import register
@@ -21,7 +21,8 @@ def spellcheck(string, locale='en-US'):
 def suggestions(context, string, locale='en-US'):
     d = DidYouMean(locale, dict_dir=settings.DICT_DIR,
                    words=settings.WORD_LIST)
-    words = [(jinja2.escape(w.new), w.corrected) for w in d.suggest(string)]
+    words = [(jinja2.escape(smart_unicode(w.new)), w.corrected)
+             for w in d.suggest(string)]
 
     newwords = []
     newquery = []
@@ -32,7 +33,7 @@ def suggestions(context, string, locale='en-US'):
         else:
             newwords.append(w[0])
 
-    markup = '<a href="{url}">{text}</a>'
+    markup = u'<a href="{url}">{text}</a>'
 
     q = u' '.join(newquery)
     text = u' '.join(newwords)

apps/search/tests/test_search.py

@@ -4,6 +4,7 @@ Tests for the search (sphinx) app.
 import os
 import shutil
 import time
+import json
 
 from django.test import client
 from django.db import connection
@@ -11,13 +12,19 @@ from django.db import connection
 from nose import SkipTest
 from nose.tools import assert_raises
 import test_utils
-import json
+import jingo
 
 from manage import settings
 from sumo.urlresolvers import reverse
 import search as constants
 from search.utils import start_sphinx, stop_sphinx, reindex
 from search.clients import WikiClient, ForumClient, SearchError
+from sumo.models import WikiPage
+
+
+def render(s, context={}):
+    t = jingo.env.from_string(s)
+    return t.render(**context)
 
 
 def create_extra_tables():
@@ -218,6 +225,20 @@ class SearchTest(SphinxTestCase):
                             results[-1]['attrs'][test_for[i]])
             i += 1
 
+    def test_unicode_excerpt(self):
+        """Unicode characters in the excerpt should not be a problem."""
+        wc = WikiClient()
+        q = 'contribute'
+        results = wc.query(q)
+        self.assertNotEquals(0, len(results))
+        page = WikiPage.objects.get(pk=results[0]['id'])
+        try:
+            excerpt = wc.excerpt(page.data, q)
+            render('{{ c }}', {'c': excerpt})
+        except UnicodeDecodeError:
+            self.fail('Raised UnicodeDecodeError.')
+
+
 def test_sphinx_down():
     """
     Tests that the client times out when Sphinx is down.

apps/search/views.py

@@ -1,17 +1,15 @@
-# Create your views here.
-
+import urllib
 import time
 import re
 import json
 
-from django.utils.datastructures import MultiValueDict
 from django import forms
 from django.conf import settings
 from django.http import HttpResponse
 
 import jingo
+import jinja2
 from tower import ugettext as _
-from flatqs import flatten
 
 from sumo.utils import paginate
 from sumo.models import ForumThread, WikiPage, Forum, Category
@@ -209,14 +207,22 @@ def search(request):
         try:
             if documents[i]['attrs'].get('category', False):
                 wiki_page = WikiPage.objects.get(pk=documents[i]['id'])
-                result = {'search_summary': wc.excerpt(wiki_page.data, q),
+
+                excerpt = wc.excerpt(wiki_page.data, q)
+                summary = jinja2.Markup(excerpt)
+
+                result = {'search_summary': summary,
                     'url': wiki_page.get_url(),
                     'title': wiki_page.name,
                     }
                 results.append(result)
             else:
                 forum_thread = ForumThread.objects.get(pk=documents[i]['id'])
-                result = {'search_summary': fc.excerpt(forum_thread.data, q),
+
+                excerpt = fc.excerpt(forum_thread.data, q)
+                summary = jinja2.Markup(excerpt)
+
+                result = {'search_summary': summary,
                     'url': forum_thread.get_url(),
                     'title': forum_thread.name,
                     }
@@ -226,12 +232,16 @@ def search(request):
         except (WikiPage.DoesNotExist, ForumThread.DoesNotExist):
             continue
 
-    refine_query = MultiValueDict()
-    for name, field in search_form.fields.items():
-        refine_query[name] = request.GET.getlist(name)
+    items = [(k, v) for k in search_form.fields
+             for v in request.GET.getlist(k) if v]
 
-    refine_query = '?a=1&w=' + str(where) + '&' \
-        + flatten(refine_query, encode=False)
+    try:
+        qsa = urllib.urlencode(items)
+    except UnicodeEncodeError:
+        qsa = urllib.urlencode([(k, v.encode('utf8')) for k, v
+                                in items])
+
+    refine_query = u'?a=1&w=%s&%s' % (where, qsa)
 
     if request.GET.get('format') == 'json':
         callback = request.GET.get('callback', '').strip()

apps/sumo/helpers.py

@@ -1,12 +1,12 @@
 import cgi
 import urlparse
+import urllib
 
-from django.utils.datastructures import MultiValueDict
+from django.utils.encoding import smart_unicode
 
 import jinja2
 
 from jingo import register, env
-from flatqs import flatten
 
 from sumo.urlresolvers import reverse
 
@@ -33,14 +33,25 @@ def urlparams(url_, hash=None, **query):
     url = urlparse.urlparse(url_)
     fragment = hash if hash is not None else url.fragment
 
-    query_dict = MultiValueDict()
+    items = []
     if url.query:
         for k, v in cgi.parse_qsl(url.query):
-            query_dict.update({k: v})
+            items.append((k, v))
     for k, v in query.items():
-        query_dict.update({k: v})
+        items.append((k, v))
+
+    items = [(k, v) for k, v in items if v is not None]
+
+    def encoder(v):
+        if hasattr(v, 'encode'):
+            return v.encode('raw_unicode_escape')
+        return v
+
+    try:
+        query_string = urllib.urlencode(items)
+    except UnicodeEncodeError:
+        query_string = urllib.urlencode([(k, encoder(v)) for k, v in items])
 
-    query_string = flatten(query_dict, encode=False)
     new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
                                query_string, fragment)
     return jinja2.Markup(new.geturl())
@@ -87,7 +98,11 @@ def fe(str, *args, **kwargs):
     """Format a safe string with potentially unsafe arguments, then return a
     safe string."""
 
-    for i in kwargs:
-        kwargs[i] = jinja2.escape(kwargs[i])
+    str = unicode(str)
 
-    return jinja2.Markup(str.format(**kwargs))
+    args = [jinja2.escape(smart_unicode(v)) for v in args]
+
+    for k in kwargs:
+        kwargs[k] = jinja2.escape(smart_unicode(kwargs[k]))
+
+    return jinja2.Markup(str.format(*args, **kwargs))

apps/sumo/tests/test_helpers.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from nose.tools import eq_
 
 import jingo
@@ -8,7 +10,29 @@ def render(s, context={}):
     return t.render(**context)
 
 
+def setup():
+    jingo.load_helpers()
+
+
 def test_fe_helper():
     context = {'var': '<bad>'}
     template = '{{ "<em>{t}</em>"|fe(t=var) }}'
     eq_('<em>&lt;bad&gt;</em>', render(template, context))
+
+
+def test_fe_positional():
+    context = {'var': '<bad>'}
+    template = '{{ "<em>{0}</em>"|fe(var) }}'
+    eq_('<em>&lt;bad&gt;</em>', render(template, context))
+
+
+def test_fe_unicode():
+    context = {'var': u'Français'}
+    template = '{{ "Speak {0}"|fe(var) }}'
+    eq_(u'Speak Français', render(template, context))
+
+
+def test_urlparams_unicode():
+    context = {'var': u'Fran\xc3\xa7ais'}
+    template = '{{ url("search")|urlparams(q=var) }}'
+    eq_(u'/en-US/search?q=Fran%C3%A7ais', render(template, context))

apps/sumo/utils.py

@@ -1,6 +1,6 @@
-from django.core import paginator
+import urllib
 
-from flatqs import flatten
+from django.core import paginator
 
 
 def paginate(request, queryset, per_page=20):
@@ -20,10 +20,15 @@ def paginate(request, queryset, per_page=20):
         paginated = p.page(1)
 
     base = request.build_absolute_uri(request.path)
-    request_copy = request.GET.copy()
+
+    items = [(k, v) for k in request.GET if k != 'page'
+             for v in request.GET.getlist(k) if v]
+
     try:
-        del request_copy['page']
-    except KeyError:
-        pass
-    paginated.url = u'%s?%s' % (base, flatten(request_copy, encode=False))
+        qsa = urllib.urlencode(items)
+    except UnicodeEncodeError:
+        qsa = urllib.urlencode([(k, v.encode('utf8')) for k, v
+                                in items])
+
+    paginated.url = u'%s?%s' % (base, qsa)
     return paginated