failing tests added

* adding xml and json to the editable files
* try to save attachment, on failure encode and try to save again

apps/jetpack/models.py

@@ -59,7 +59,7 @@ from elasticutils.utils import retry_on_timeout
 
 log = commonware.log.getLogger('f.jetpack')
 
-EDITABLE_EXTENSIONS = ("html", "css", "js", "txt")
+EDITABLE_EXTENSIONS = ("html", "css", "js", "txt", "xml", "json")
 
 def make_name(value=None):
     " wrap for slugify "
@@ -2379,24 +2379,29 @@ class Attachment(BaseModel):
         try:
             with codecs.open(self.get_file_path(), **kwargs) as f:
                 f.write(data)
-        except UnicodeDecodeError, err:
-            log.error('Attachment write failure (UTF8 decode): (%s)\n%s' % (
-                self.pk, str(err)))
-            raise AttachmentWriteException(
-                'Attachment failed to save properly<br/>'
-                'Unknown Unicode in file')
-        except UnicodeEncodeError, err:
-            log.error('Attachment write failure (UTF8 encode): (%s)\n%s' % (
-                self.pk, str(err)))
-            exc_type, exc_value, exc_traceback = sys.exc_info()
-            tb = traceback.format_exception(exc_type, exc_value, exc_traceback)
+        except UnicodeDecodeError, UnicodeEncodeError:
+            data = unicode(data, 'utf-8')
+            try:
+                with codecs.open(self.get_file_path(), **kwargs) as f:
+                    f.write(data)
+            except UnicodeDecodeError, err:
+                log.error('Attachment write failure (UTF8 decode): (%s)\n%s' % (
+                    self.pk, str(err)))
+                raise AttachmentWriteException(
+                    'Attachment failed to save properly<br/>'
+                    'Unknown Unicode in file')
+            except UnicodeEncodeError, err:
+                log.error('Attachment write failure (UTF8 encode): (%s)\n%s' % (
+                    self.pk, str(err)))
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                tb = traceback.format_exception(exc_type, exc_value, exc_traceback)
 
-            mail_admins(
-                    'Unicode Encode in writing attachment',
-                    "Attachment [%d] writing error\n\n %s" % (self.pk, tb))
-            raise AttachmentWriteException(
-                'Attachment failed to save properly<br/>'
-                'Unknown Unicode in file')
+                mail_admins(
+                        'Unicode Encode in writing attachment',
+                        "Attachment [%d] writing error\n\n %s" % (self.pk, tb))
+                raise AttachmentWriteException(
+                    'Attachment failed to save properly<br/>'
+                    'Unknown Unicode in file')
 
 
     def export_code(self, static_dir):

apps/jetpack/tests/attachment_test.py

@@ -125,6 +125,27 @@ class AttachmentTest(TestCase):
         assert os.path.isfile(attachment.get_file_path())
         assert attachment.read()
 
+    def test_attachment_with_kanji_from_web(self):
+        url = ("file://%s/apps/jetpack/tests/"
+                "sample_attachment_kanji_UTF8_without_BOM.json") % settings.ROOT
+        URLField.clean = Mock(return_value=url)
+        addon = Package.objects.create(
+                type='a',
+                author=self.author)
+        self.author.set_password('secure')
+        self.author.save()
+        self.client.login(username=self.author.username, password='secure')
+        response = self.client.post(addon.latest.get_add_attachment_url(), {
+            'url': url,
+            'filename': 'sample_attachment_kanji_UTF8_without_BOM.json',
+            'force_contenttype': 'utf-8'})
+        eq_(response.status_code, 200)
+        addon = Package.objects.get(author=self.author)
+        eq_(addon.latest.revision_number, 1)
+        attachment = addon.latest.attachments.get()
+        eq_('sample_attachment_kanji_UTF8_without_BOM', attachment.filename)
+        assert os.path.isfile(attachment.get_file_path())
+        assert attachment.read()
 
     def test_create_image_attachment(self):
         " test simply shouldn't raise any errors "

apps/jetpack/tests/sample_attachment_kanji_UTF8_with_BOM.json

@@ -0,0 +1,3 @@
+{ "test": [
+  {"country":"日本"}
+]}

apps/jetpack/tests/sample_attachment_kanji_UTF8_without_BOM.json

@@ -0,0 +1,3 @@
+{ "test": [
+  {"country":"日本"}
+]}