Use encoding from the response when getting remote data (bug 1012817)

This commit is contained in:
Mathieu Pillard 2014-05-26 16:16:55 +02:00
Родитель ce90728e60
Коммит 0c37da1110
3 изменённых файлов: 26 добавлений и 1 удалений

Просмотреть файл

@ -184,7 +184,7 @@ def try_get_resource(err, package, url, filename, resource_type="URL",
error="Resource too large",
description=["A requested resource returned too much data. "
"File sizes are limited to %dMB." %
(constants.MAX_RESOURCE_SIZE / 1204 / 1024),
(constants.MAX_RESOURCE_SIZE / 1024 / 1024),
"Requested resource: %s" % url],
filename=filename)
return
@ -195,6 +195,12 @@ def try_get_resource(err, package, url, filename, resource_type="URL",
# Some versions of requests don't support close().
pass
if request.encoding:
# If a encoding was specified, decode raw data with it. Needed to
# properly feed unicode data to HTMLParser when content is in UTF-8
# for instance.
data = data.decode(request.encoding)
http_cache[url] = data
if not data or request.status_code != 200:

Просмотреть файл

@ -54,6 +54,7 @@ def safe(func):
request = Mock()
request.text = "foo bar"
request.status_code = 200
request.encoding = 'UTF-8'
# The first bit is the return value. The second bit tells whatever
# is requesting the data that there's no more data.
request.raw.read.side_effect = [request.text, ""]

Просмотреть файл

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
from functools import wraps
import os
@ -178,6 +179,7 @@ class TestDataOutput(TestCase):
def test_just_right(self, r_g):
normal_response_object = Mock()
normal_response_object.raw.read.side_effect = ["x" * 100, ""]
normal_response_object.encoding = ""
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
@ -185,11 +187,26 @@ class TestDataOutput(TestCase):
"x" * 100)
self.assert_silent()
@patch("appvalidator.testcases.webappbase.requests.get")
@patch("appvalidator.constants.MAX_RESOURCE_SIZE", 100)
def test_unicodeness(self, r_g):
normal_response_object = Mock()
normal_response_object.raw.read.side_effect = [u"é".encode('utf-8')
* 100, ""]
normal_response_object.encoding = "UTF-8"
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", ""),
u"é" * 100)
self.assert_silent()
@patch("appvalidator.testcases.webappbase.requests.get")
@patch("appvalidator.constants.MAX_RESOURCE_SIZE", 100)
def test_empty(self, r_g):
empty_response = Mock()
empty_response.raw.read.return_value = ""
empty_response.encoding = ""
empty_response.status_code = 200
r_g.return_value = empty_response
@ -202,6 +219,7 @@ class TestDataOutput(TestCase):
def test_eventual_404(self, r_g):
error_response = Mock()
error_response.raw.read.side_effect = ["x" * 100, ""]
error_response.encoding = ""
error_response.status_code = 404
r_g.return_value = error_response