diff --git a/apps/jetpack/models.py b/apps/jetpack/models.py index 765392cc..2757df55 100644 --- a/apps/jetpack/models.py +++ b/apps/jetpack/models.py @@ -59,7 +59,7 @@ from elasticutils.utils import retry_on_timeout log = commonware.log.getLogger('f.jetpack') -EDITABLE_EXTENSIONS = ("html", "css", "js", "txt") +EDITABLE_EXTENSIONS = ("html", "css", "js", "txt", "xml", "json") def make_name(value=None): " wrap for slugify " @@ -2379,24 +2379,29 @@ class Attachment(BaseModel): try: with codecs.open(self.get_file_path(), **kwargs) as f: f.write(data) - except UnicodeDecodeError, err: - log.error('Attachment write failure (UTF8 decode): (%s)\n%s' % ( - self.pk, str(err))) - raise AttachmentWriteException( - 'Attachment failed to save properly
' - 'Unknown Unicode in file') - except UnicodeEncodeError, err: - log.error('Attachment write failure (UTF8 encode): (%s)\n%s' % ( - self.pk, str(err))) - exc_type, exc_value, exc_traceback = sys.exc_info() - tb = traceback.format_exception(exc_type, exc_value, exc_traceback) + except UnicodeDecodeError, UnicodeEncodeError: + data = unicode(data, 'utf-8') + try: + with codecs.open(self.get_file_path(), **kwargs) as f: + f.write(data) + except UnicodeDecodeError, err: + log.error('Attachment write failure (UTF8 decode): (%s)\n%s' % ( + self.pk, str(err))) + raise AttachmentWriteException( + 'Attachment failed to save properly
' + 'Unknown Unicode in file') + except UnicodeEncodeError, err: + log.error('Attachment write failure (UTF8 encode): (%s)\n%s' % ( + self.pk, str(err))) + exc_type, exc_value, exc_traceback = sys.exc_info() + tb = traceback.format_exception(exc_type, exc_value, exc_traceback) - mail_admins( - 'Unicode Encode in writing attachment', - "Attachment [%d] writing error\n\n %s" % (self.pk, tb)) - raise AttachmentWriteException( - 'Attachment failed to save properly
' - 'Unknown Unicode in file') + mail_admins( + 'Unicode Encode in writing attachment', + "Attachment [%d] writing error\n\n %s" % (self.pk, tb)) + raise AttachmentWriteException( + 'Attachment failed to save properly
' + 'Unknown Unicode in file') def export_code(self, static_dir): diff --git a/apps/jetpack/tests/attachment_test.py b/apps/jetpack/tests/attachment_test.py index 87ddf921..3f15cb4e 100644 --- a/apps/jetpack/tests/attachment_test.py +++ b/apps/jetpack/tests/attachment_test.py @@ -125,6 +125,27 @@ class AttachmentTest(TestCase): assert os.path.isfile(attachment.get_file_path()) assert attachment.read() + def test_attachment_with_kanji_from_web(self): + url = ("file://%s/apps/jetpack/tests/" + "sample_attachment_kanji_UTF8_without_BOM.json") % settings.ROOT + URLField.clean = Mock(return_value=url) + addon = Package.objects.create( + type='a', + author=self.author) + self.author.set_password('secure') + self.author.save() + self.client.login(username=self.author.username, password='secure') + response = self.client.post(addon.latest.get_add_attachment_url(), { + 'url': url, + 'filename': 'sample_attachment_kanji_UTF8_without_BOM.json', + 'force_contenttype': 'utf-8'}) + eq_(response.status_code, 200) + addon = Package.objects.get(author=self.author) + eq_(addon.latest.revision_number, 1) + attachment = addon.latest.attachments.get() + eq_('sample_attachment_kanji_UTF8_without_BOM', attachment.filename) + assert os.path.isfile(attachment.get_file_path()) + assert attachment.read() def test_create_image_attachment(self): " test simply shouldn't raise any errors " diff --git a/apps/jetpack/tests/sample_attachment_kanji_UTF8_with_BOM.json b/apps/jetpack/tests/sample_attachment_kanji_UTF8_with_BOM.json new file mode 100644 index 00000000..1dd52ee7 --- /dev/null +++ b/apps/jetpack/tests/sample_attachment_kanji_UTF8_with_BOM.json @@ -0,0 +1,3 @@ +{ "test": [ + {"country":"日本"} +]} \ No newline at end of file diff --git a/apps/jetpack/tests/sample_attachment_kanji_UTF8_without_BOM.json b/apps/jetpack/tests/sample_attachment_kanji_UTF8_without_BOM.json new file mode 100644 index 00000000..6e0352e6 --- /dev/null +++ b/apps/jetpack/tests/sample_attachment_kanji_UTF8_without_BOM.json @@ -0,0 +1,3 @@ +{ "test": [ + {"country":"日本"} +]}