Ref #2899: Use s3 for user media on legacy branch

Will require changes to server config

kitsune/gallery/tests/test_models.py

@@ -1,4 +1,7 @@
+from django.core.files.base import ContentFile
+
 from nose.tools import eq_
+from mock import patch
 
 from kitsune.gallery.models import Image
 from kitsune.gallery.tests import ImageFactory
@@ -6,16 +9,18 @@ from kitsune.sumo.tests import TestCase
 from kitsune.upload.tasks import generate_thumbnail
 
 
+@patch('kitsune.upload.tasks._create_image_thumbnail')
 class ImageTestCase(TestCase):
     def tearDown(self):
         Image.objects.all().delete()
         super(ImageTestCase, self).tearDown()
 
-    def test_thumbnail_url_if_set(self):
+    def test_thumbnail_url_if_set(self, create_thumbnail_mock):
         """thumbnail_url_if_set() returns self.thumbnail if set, or else
         returns self.file"""
         img = ImageFactory()
         eq_(img.file.url, img.thumbnail_url_if_set())
 
+        create_thumbnail_mock.return_value = ContentFile('the dude')
         generate_thumbnail(img, 'file', 'thumbnail')
         eq_(img.thumbnail.url, img.thumbnail_url_if_set())

kitsune/gallery/views.py

@@ -244,7 +244,7 @@ def _get_media_info(media_id, media_type):
     if media_type == 'image':
         media = get_object_or_404(Image, pk=media_id)
         try:
-            media_format = imghdr.what(media.file.path)
+            media_format = imghdr.what(media.file.file)
         except UnicodeEncodeError:
             pass
     elif media_type == 'video':

kitsune/settings.py

@@ -550,6 +550,7 @@ INSTALLED_APPS = (
     'authority',
     'timezones',
     'waffle',
+    'storages',
     'kitsune.access',
     'kitsune.sumo',
     'kitsune.search',
@@ -757,6 +758,15 @@ MAX_FILEPATH_LENGTH = 250
 # Default storage engine - ours does not preserve filenames
 DEFAULT_FILE_STORAGE = 'kitsune.upload.storage.RenameFileStorage'
 
+# AWS S3 Storage Settings
+AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID', default='')
+AWS_SECRET_ACCESS_KEY = config('AWS_SECRET_ACCESS_KEY', default='')
+AWS_STORAGE_BUCKET_NAME = config('AWS_STORAGE_BUCKET_NAME', default='')
+AWS_S3_CUSTOM_DOMAIN = config('AWS_S3_CUSTOM_DOMAIN', default='prod-cdn.sumo.mozilla.net')
+AWS_S3_OBJECT_PARAMETERS = {
+    'CacheControl': 'max-age=2592000',
+}
+
 # Auth and permissions related constants
 LOGIN_URL = '/users/login'
 LOGOUT_URL = '/users/logout'

kitsune/upload/storage.py

@@ -3,7 +3,13 @@ import itertools
 import os
 import time
 
-from django.core.files.storage import FileSystemStorage as DjangoStorage
+from django.conf import settings
+from django.core.files.storage import FileSystemStorage
+
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+DjangoStorage = S3Boto3Storage if settings.AWS_ACCESS_KEY_ID else FileSystemStorage
 
 
 class RenameFileStorage(DjangoStorage):

kitsune/upload/tasks.py

@@ -2,9 +2,11 @@ import logging
 import os
 import StringIO
 import subprocess
+from tempfile import NamedTemporaryFile
 
 from django.conf import settings
 from django.core.files.base import ContentFile
+from django.core.files.storage import default_storage
 
 from celery import task
 from PIL import Image
@@ -28,7 +30,7 @@ def generate_thumbnail(for_obj, from_field, to_field,
     to_ = getattr(for_obj, to_field)
 
     # Bail silently if nothing to generate from, image was probably deleted.
-    if not (from_ and os.path.isfile(from_.path)):
+    if not (from_ and default_storage.exists(from_.name)):
         log_msg = 'No file to generate from: {model} {id}, {from_f} -> {to_f}'
         log.info(log_msg.format(model=for_obj.__class__.__name__,
                                 id=for_obj.id, from_f=from_field,
@@ -38,24 +40,23 @@ def generate_thumbnail(for_obj, from_field, to_field,
     log_msg = 'Generating thumbnail for {model} {id}: {from_f} -> {to_f}'
     log.info(log_msg.format(model=for_obj.__class__.__name__, id=for_obj.id,
                             from_f=from_field, to_f=to_field))
-    thumb_content = _create_image_thumbnail(from_.path, longest_side=max_size)
-    file_path = from_.path
+    thumb_content = _create_image_thumbnail(from_.file, longest_side=max_size)
     if to_:  # Clean up old file before creating new one.
         to_.delete(save=False)
     # Don't modify the object.
-    to_.save(file_path, thumb_content, save=False)
+    to_.save(from_.name, thumb_content, save=False)
     # Use update to avoid race conditions with updating different fields.
     # E.g. when generating two thumbnails for different fields of a single
     # object.
     for_obj.update(**{to_field: to_.name})
 
 
-def _create_image_thumbnail(file_path, longest_side=settings.THUMBNAIL_SIZE,
+def _create_image_thumbnail(fileobj, longest_side=settings.THUMBNAIL_SIZE,
                             pad=False):
     """
     Returns a thumbnail file with a set longest side.
     """
-    original_image = Image.open(file_path)
+    original_image = Image.open(fileobj)
     original_image = original_image.convert('RGBA')
     file_width, file_height = original_image.size
 
@@ -109,14 +110,14 @@ def compress_image(for_obj, for_field):
     for_ = getattr(for_obj, for_field)
 
     # Bail silently if nothing to compress, image was probably deleted.
-    if not (for_ and os.path.isfile(for_.path)):
+    if not (for_ and default_storage.exists(for_.name)):
         log_msg = 'No file to compress for: {model} {id}, {for_f}'
         log.info(log_msg.format(model=for_obj.__class__.__name__,
                                 id=for_obj.id, for_f=for_field))
         return
 
     # Bail silently if not a PNG.
-    if not (os.path.splitext(for_.path)[1].lower() == '.png'):
+    if not (os.path.splitext(for_.name)[1].lower() == '.png'):
         log_msg = 'File is not PNG for: {model} {id}, {for_f}'
         log.info(log_msg.format(model=for_obj.__class__.__name__,
                                 id=for_obj.id, for_f=for_field))
@@ -126,7 +127,17 @@ def compress_image(for_obj, for_field):
     log.info(log_msg.format(model=for_obj.__class__.__name__, id=for_obj.id,
                             for_f=for_field))
 
-    file_path = for_.path
-    if settings.OPTIPNG_PATH is not None:
-        subprocess.call([settings.OPTIPNG_PATH,
-                         '-quiet', '-preserve', file_path])
+    _optipng(for_.name)
+
+
+def _optipng(file_name):
+    if not settings.OPTIPNG_PATH:
+        return
+
+    with default_storage.open(file_name, 'rb') as file_obj:
+        with NamedTemporaryFile(suffix='.png') as tmpfile:
+            tmpfile.write(file_obj.read())
+            subprocess.call([settings.OPTIPNG_PATH,
+                            '-quiet', '-preserve', tmpfile.name])
+            file_content = ContentFile(tmpfile.read())
+            default_storage.save(file_name, file_content)

kitsune/upload/tests/test_tasks.py

@@ -162,7 +162,7 @@ class CompressImageTestCase(TestCase):
             image.file.save(up_file.name, up_file, save=True)
         return image
 
-    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', '')
+    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', '/dude')
     @mock.patch.object(kitsune.upload.tasks.subprocess, 'call')
     def test_compressed_image_default(self, call):
         """uploaded image is compressed."""
@@ -170,7 +170,7 @@ class CompressImageTestCase(TestCase):
         compress_image(image, 'file')
         assert call.called
 
-    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', '')
+    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', '/dude')
     @mock.patch.object(kitsune.upload.tasks.subprocess, 'call')
     def test_compress_no_file(self, call):
         """compress_image does not fail when no file is provided."""
@@ -178,7 +178,7 @@ class CompressImageTestCase(TestCase):
         compress_image(image, 'file')
         assert not call.called
 
-    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', None)
+    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', '')
     @mock.patch.object(kitsune.upload.tasks.subprocess, 'call')
     def test_compress_no_compression_software(self, call):
         """compress_image does not fail when no compression software."""
@@ -186,7 +186,7 @@ class CompressImageTestCase(TestCase):
         compress_image(image, 'file')
         assert not call.called
 
-    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', '')
+    @mock.patch.object(settings._wrapped, 'OPTIPNG_PATH', '/dude')
     @mock.patch.object(kitsune.upload.tasks.subprocess, 'call')
     def test_compressed_image_animated(self, call):
         """uploaded animated gif image is not compressed."""

requirements/default.txt

@@ -324,8 +324,9 @@ pyparsing==1.5.5
 # sha256: RsUeuHi3h-gU7o-XN7CmIREDSutNHAZFCsWo6lpw5gI
 pyquery==1.2.9
 
-# sha256: bxlzSLRvuM358_z8Kn1al9qV2z4uhmfPZXIWJ0_hsAk
-python-dateutil==1.5
+# sha256: iRw4sqAvW7G-PkeThmyN9Jx9Gbqr-cG61iVH4LSGaso
+# sha256: lVEbrmNNabxzKbpV5kZJmoQrxOw0KtVKjNtlZFoKrTw
+python-dateutil==2.6.1
 
 # sha256: Df9jYEI_PsCMvjv683szlGGlSiHRO-DdXZyZmc5TEHg
 python-gflags==2.0
@@ -394,3 +395,28 @@ https://github.com/eventbrite/zendesk/archive/1b256c3b827a8f3c6bbd3bd1fc09b7b0e2
 # sha256: oxpREC6MLwR_iqYri9xVL3ox2cafLLFjkyJcPuDbcrU
 # sha256: JawRvUVE_vMjbzJDn2sd47z5MfizTHeGq0Dw7IVibus
 django-mozilla-product-details==0.12.1
+
+# sha256: vI5MH0g2CMXdEhIHL9QQQvPvLSooluxPsS28YsgplqA
+# sha256: q2vhU4zylRFAC86D0OXKdNLpNcrYIIYGO89eftrMFmE
+django-storages==1.6.5
+
+# sha256: OAV7BmmQFyzm678qXgRqVFUDeTWB_PFMqw44IcYRLrA
+# sha256: 95933KIoD3eA851ypQiPTPK2JsCSHnGF7WrBer_dfmw
+boto3==1.4.7
+
+# sha256: H20AGbah173V1EBv3-e2d290afX1WOLK1svsN_G4btQ
+# sha256: NTsM4TTa4b_9DEu-h3nid-PdW3Oq2vdnuX7T9D9uJtI
+botocore==1.7.41
+
+# sha256: 8RtEYfQldAodkI6aP3Nlw9LlafbKaKL_i8W82Wdu3WM
+# sha256: aoHUyapiyvBhy1F7TZrR3TADdM1HBpl6_5zWrt1h_GQ
+jmespath==0.9.3
+
+# sha256: x7FvTMpazSvVesliO_uj_s4Eckc5KJNQbQ0ubyViDrM
+# sha256: dvH1j0pH4sivoTXix2lYgGo6u8Qrch2H_Z0RQJx12Xk
+s3transfer==0.1.11
+
+# sha256: ekvUfq9lluEpXssRNhE5_r4psISoe_AFv4mfmkLtw8Y
+# sha256: Aq7EvZKrBn9v8no4o4pBFzvwG-2PiRV3aMFXP1PkdKY
+# sha256: UeZO8uv7Kcrh-qEzs3EBQ0luyiHFMPP3FCTXdod2QnQ
+docutils==0.14