Handle icons served with `Content-Type: ...; charset=utf-8` (bug 1056231)

appvalidator/testcases/webappbase.py

@@ -133,7 +133,7 @@ def _normalize_url(err, url):
 
 
 def try_get_resource(err, package, url, filename, resource_type="URL",
-                     max_size=True, simulate=False):
+                     max_size=True, simulate=False, binary=False):
 
     # Try to process data URIs first.
     if url.startswith("data:"):
@@ -211,10 +211,10 @@ def try_get_resource(err, package, url, filename, resource_type="URL",
             # Some versions of requests don't support close().
             pass
 
-        if request.encoding:
-            # If a encoding was specified, decode raw data with it. Needed to
-            # properly feed unicode data to HTMLParser when content is in UTF-8
-            # for instance.
+        if request.encoding and not binary:
+            # If a encoding was specified, and we are not trying to fetch a
+            # binary, decode raw data with it. Needed to properly feed unicode
+            # data to HTMLParser when content is in UTF-8 for instance.
             data = data.decode(request.encoding)
 
         http_cache[url] = data
@@ -374,8 +374,8 @@ def test_app_resources(err, package):
                 filename="manifest.webapp")
             break
 
-        icon_data = try_get_resource(err, package,
-                                     url, "manifest.webapp", "icon")
+        icon_data = try_get_resource(err, package, url, "manifest.webapp",
+                                     "icon", binary=True)
         if not icon_data:
             continue
 

tests/test_webapp_resources.py

@@ -204,6 +204,21 @@ class TestDataOutput(TestCase):
             u"é" * 100)
         self.assert_silent()
 
+    @patch("appvalidator.testcases.webappbase.requests.get")
+    @patch("appvalidator.constants.MAX_RESOURCE_SIZE", 100)
+    def test_unicode_binary(self, r_g):
+        normal_response_object = Mock()
+        # The u"é" is *not* encoded in UTF-8 this time, so it would throw an
+        # UnicodeEncodeError if we'd try to decode it.
+        normal_response_object.raw.read.side_effect = [u"é" * 100, ""]
+        normal_response_object.encoding = "UTF-8"
+        normal_response_object.status_code = 200
+        r_g.return_value = normal_response_object
+
+        eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", "",
+                                     binary=True), u"é" * 100)
+        self.assert_silent()
+
     @patch("appvalidator.testcases.webappbase.requests.get")
     @patch("appvalidator.constants.MAX_RESOURCE_SIZE", 100)
     def test_decode_gzip(self, r_g):
@@ -290,6 +305,9 @@ class TestResourcePolling(TestCase):
         self.setup_manifest()["icons"] = {"32": "fizz"}
         appbase.test_app_resources(self.err, None)
         eq_(tgr.call_args[0][2], "fizz")
+
+        # Icons must be fetched with "binary" keyword argument.
+        eq_(tgr.call_args[1]['binary'], True)
         eq_(test_icon.call_args[1]["data"].getvalue(), "foobar")
 
     @patch("appvalidator.testcases.webappbase.test_icon", Mock())