move FileUpload creation to a method, add a hash, add upload tests

apps/devhub/tests/test_views.py

@@ -1,5 +1,8 @@
+import json
 import re
+import shutil
 import socket
+import tempfile
 from decimal import Decimal
 from urllib import urlencode
 
@@ -20,7 +23,7 @@ from applications.models import AppVersion
 from bandwagon.models import Collection
 from devhub.forms import ContribForm
 from devhub.models import ActivityLog, RssKey, SubmitStep
-from files.models import File, Platform
+from files.models import File, FileUpload, Platform
 from reviews.models import Review
 from tags.models import Tag
 from users.models import UserProfile
@@ -2058,3 +2061,49 @@ class TestSubmitSteps(test_utils.TestCase):
         doc = pq(self.client.get(url).content)
         self.assert_linked(doc, [])
         self.assert_highlight(doc, 7)
+
+
+class TestUpload(test_utils.TestCase):
+    fixtures = ['base/users']
+
+    def setUp(self):
+        self.url = reverse('devhub.upload')
+        self._addons_path = settings.ADDONS_PATH
+        settings.ADDONS_PATH = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(settings.ADDONS_PATH)
+        settings.ADDONS_PATH = self._addons_path
+
+    def post(self):
+        data = 'some data'
+        return self.client.post(self.url, data, content_type='text',
+                                HTTP_X_FILE_NAME='filename.xpi',
+                                HTTP_X_FILE_SIZE=len(data))
+
+    def test_no_x_filesize(self):
+        r = self.client.post(self.url, 'some data', content_type='text')
+        eq_(r.status_code, 400)
+
+    def test_create_fileupload(self):
+        self.post()
+        upload = FileUpload.objects.get()
+        eq_(upload.name, 'filename.xpi')
+        eq_(open(upload.path).read(), 'some data')
+
+    def test_fileupload_user(self):
+        self.client.login(username='regular@mozilla.com', password='password')
+        self.post()
+        user = UserProfile.objects.get(email='regular@mozilla.com')
+        eq_(FileUpload.objects.get().user, user)
+
+    def test_fileupload_validation(self):
+        self.post()
+        validation = json.loads(FileUpload.objects.get().validation)
+        eq_(validation['success'], False)
+
+    def test_redirect(self):
+        r = self.post()
+        upload = FileUpload.objects.get()
+        url = reverse('devhub.upload_detail', args=[upload.pk, 'json'])
+        self.assertRedirects(r, url)

apps/devhub/views.py

@@ -3,17 +3,13 @@ import collections
 import functools
 import json
 import os
-import uuid
 
 from django import http
-from django.conf import settings
-from django.db.models import F
 from django.shortcuts import get_object_or_404, redirect
 from django.utils.http import urlquote
 
 import commonware.log
 import jingo
-import path
 from tower import ugettext_lazy as _lazy, ugettext as _
 
 import amo
@@ -37,8 +33,6 @@ from . import forms, tasks, feeds
 
 log = commonware.log.getLogger('z.devhub')
 
-# Acceptable extensions.
-EXTENSIONS = ('.xpi', '.jar', '.xml')
 
 # We use a session cookie to make sure people see the dev agreement.
 DEV_AGREEMENT_COOKIE = 'yes-I-read-the-dev-agreement'
@@ -405,22 +399,16 @@ def profile(request, addon_id, addon):
 def upload(request):
     if request.method == 'POST':
         #TODO(gkoberger): Bug 610800 - Don't load uploads into memory.
-        upload = request.raw_post_data
-        upload_name = request.META['HTTP_X_FILE_NAME']
-        upload_size = request.META['HTTP_X_FILE_SIZE']
-        loc = path.path(settings.ADDONS_PATH) / 'temp' / uuid.uuid4().hex
-        if not loc.dirname().exists():
-            loc.dirname().makedirs()
-        ext = path.path(upload_name).ext
-        if ext in EXTENSIONS:
-            loc += ext
-        log.info('UPLOAD: %r (%s bytes) to %r' %
-                 (upload_name, upload_size, loc))
-        with open(loc, 'wb') as fd:
-            for chunk in upload:
-                fd.write(chunk)
-        user = getattr(request, 'amo_user', None)
-        fu = FileUpload.objects.create(path=loc, name=upload_name, user=user)
+        filedata = request.raw_post_data
+        try:
+            filename = request.META['HTTP_X_FILE_NAME']
+            size = request.META['HTTP_X_FILE_SIZE']
+        except KeyError:
+            return http.HttpResponseBadRequest()
+        fu = FileUpload.from_post([filedata], filename, size)
+        if request.user.is_authenticated():
+            fu.user = request.amo_user
+            fu.save()
         tasks.validator.delay(fu.pk)
         return redirect('devhub.upload_detail', fu.pk, 'json')
 

apps/files/models.py

@@ -1,4 +1,6 @@
+import hashlib
 import os
+import uuid
 import zipfile
 
 from django.conf import settings
@@ -15,6 +17,9 @@ from amo.urlresolvers import reverse
 
 log = commonware.log.getLogger('z.files')
 
+# Acceptable extensions.
+EXTENSIONS = ('.xpi', '.jar', '.xml')
+
 
 class File(amo.models.ModelBase):
     STATUS_CHOICES = amo.STATUS_CHOICES.items()
@@ -174,6 +179,7 @@ class FileUpload(amo.models.ModelBase):
     path = models.CharField(max_length=255)
     name = models.CharField(max_length=255,
                             help_text="The user's original filename")
+    hash = models.CharField(max_length=255, default='')
     user = models.ForeignKey('users.UserProfile', null=True)
     validation = models.TextField(null=True)
     task_error = models.TextField(null=True)
@@ -184,6 +190,23 @@ class FileUpload(amo.models.ModelBase):
     def __unicode__(self):
         return self.uuid
 
+    @classmethod
+    def from_post(cls, chunks, filename, size):
+        loc = path.path(settings.ADDONS_PATH) / 'temp' / uuid.uuid4().hex
+        if not loc.dirname().exists():
+            loc.dirname().makedirs()
+        ext = path.path(filename).ext
+        if ext in EXTENSIONS:
+            loc += ext
+        log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
+        hash = hashlib.sha256()
+        with open(loc, 'wb') as fd:
+            for chunk in chunks:
+                hash.update(chunk)
+                fd.write(chunk)
+        return cls.objects.create(path=loc, name=filename,
+                                  hash='sha256:%s' % hash.hexdigest())
+
 
 class TestCase(amo.models.ModelBase):
     test_group = models.ForeignKey('TestGroup')

apps/files/tests.py

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
+import hashlib
 import os
+import shutil
+import tempfile
 
 from django import forms
 from django.conf import settings
@@ -7,10 +10,10 @@ from django.conf import settings
 import test_utils
 from nose.tools import eq_
 
-import amo
+import amo.utils
 from addons.models import Addon
 from applications.models import Application, AppVersion
-from files.models import File
+from files.models import File, FileUpload
 from files.utils import parse_xpi
 from versions.models import Version
 
@@ -159,3 +162,31 @@ class TestParseXpi(test_utils.TestCase):
     # parse_theme
     # parse_langpack
     # parse_search_engine?
+
+
+class TestFileUpload(test_utils.TestCase):
+
+    def setUp(self):
+        self._addons_path = settings.ADDONS_PATH
+        settings.ADDONS_PATH = tempfile.mkdtemp()
+        self.data = 'file contents'
+
+    def tearDown(self):
+        shutil.rmtree(settings.ADDONS_PATH)
+        settings.ADDONS_PATH = self._addons_path
+
+    def upload(self):
+        # The data should be in chunks.
+        data = list(amo.utils.chunked(self.data, 3))
+        return FileUpload.from_post(data, 'filename.xpi',
+                                    len(self.data))
+
+    def test_from_post_write_file(self):
+        eq_(open(self.upload().path).read(), self.data)
+
+    def test_from_post_filename(self):
+        eq_(self.upload().name, 'filename.xpi')
+
+    def test_from_post_hash(self):
+        hash = hashlib.sha256(self.data).hexdigest()
+        eq_(self.upload().hash, 'sha256:%s' % hash)

migrations/98-fileupload-hash.sql

@@ -0,0 +1,2 @@
+ALTER TABLE file_uploads
+    ADD COLUMN `hash` varchar(255) DEFAULT NULL;