devhub uploads (bug 586696 & 586700)

apps/devhub/management/commands/requeue_uploads.py

@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Requeue any stranded add-on uploads after restarting Rabbit."
+
+    def handle(self, *args, **options):
+        from files.models import FileUpload
+        from devhub import tasks
+        qs = FileUpload.objects.filter(task_error=None, validation=None)
+        pks = qs.values_list('pk', flat=True)
+        print 'Restarting %s tasks.' % len(pks)
+        for pk in pks:
+            tasks.validator.delay(pk)

apps/devhub/tasks.py

@@ -0,0 +1,32 @@
+import logging
+import sys
+import time
+import traceback
+
+from celeryutils import task
+
+from amo.decorators import write
+from files.models import FileUpload
+
+log = logging.getLogger('z.devhub.task')
+
+
+@task
+@write
+def validator(upload_id, **kw):
+    log.info('VALIDATING: %s' % upload_id)
+    upload = FileUpload.objects.get(pk=upload_id)
+    try:
+        result = _validator(upload)
+        upload.update(validation=result)
+    except:
+        # Store the error with the FileUpload job, then raise
+        # it for normal logging.
+        tb = traceback.format_exception(*sys.exc_info())
+        upload.update(task_error=''.join(tb))
+        raise
+
+
+def _validator(upload):
+    time.sleep(3)
+    return 'oh nice'

apps/devhub/templates/devhub/upload.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+
+{% block content %}
+<form action="" method="post" enctype="multipart/form-data">
+  {{ csrf() }}
+  <input type="file" name="upload">
+  <button type="submit">Push</button>
+</form>
+{% endblock %}

apps/devhub/templates/devhub/validation.html

@@ -0,0 +1,7 @@
+{% extends "base.html" %}
+
+{% block content %}
+Upload: {{ upload.uuid }}<br>
+Validation: {{ upload.validation }}<br>
+Error: {{ upload.task_error }}
+{% endblock %}

apps/devhub/urls.py

@@ -16,5 +16,8 @@ urlpatterns = patterns('',
 
     url('^addons$', views.addons_dashboard, name='devhub.addons'),
     url('^addons/activity$', views.addons_activity,
-        name='devhub.addons.activity'),
+        name='devhub.addons_activity'),
+    url('^upload$', views.upload, name='devhub.upload'),
+    url('^upload/([^/]+)$', views.upload_detail,
+        name='devhub.upload_detail'),
 )

apps/devhub/views.py

@@ -1,16 +1,27 @@
 import functools
+import uuid
 
 from django import http
-from django.shortcuts import get_object_or_404
+from django.conf import settings
+from django.shortcuts import get_object_or_404, redirect
 
-from tower import ugettext as _, ugettext_lazy as _lazy
+import commonware.log
 import jingo
+import path
+from tower import ugettext as _, ugettext_lazy as _lazy
 
 import amo.utils
 from amo.decorators import login_required
 from access import acl
 from addons.models import Addon
 from addons.views import BaseFilter
+from files.models import FileUpload
+from . import tasks
+
+log = commonware.log.getLogger('z.devhub')
+
+# Acceptable extensions.
+EXTENSIONS = ('.xpi', '.jar', '.xml')
 
 
 def owner_required(f=None, require_owner=True):
@@ -77,3 +88,31 @@ def addons_edit(request, addon_id, addon):
         }
 
     return jingo.render(request, 'devhub/addons/edit.html', data)
+
+
+def upload(request):
+    if request.method == 'POST' and 'upload' in request.FILES:
+        upload = request.FILES['upload']
+        loc = path.path(settings.ADDONS_PATH) / 'temp' / uuid.uuid4().hex
+        if not loc.dirname().exists():
+            loc.dirname().makedirs()
+        ext = path.path(upload.name).ext
+        if ext in EXTENSIONS:
+            loc += ext
+        log.info('UPLOAD: %r (%s bytes) to %r' %
+                 (upload.name, upload.size, loc))
+        with open(loc, 'wb') as fd:
+            for chunk in upload:
+                fd.write(chunk)
+        user = getattr(request, 'amo_user', None)
+        fu = FileUpload.objects.create(path=loc, name=upload.name, user=user)
+        task = tasks.validator.delay(fu.pk)
+        return redirect('devhub.upload_detail', fu.pk)
+
+    return jingo.render(request, 'devhub/upload.html')
+
+
+def upload_detail(request, uuid):
+    upload = get_object_or_404(FileUpload.uncached, uuid=uuid)
+    return jingo.render(request, 'devhub/validation.html',
+                        dict(upload=upload))

apps/files/models.py

@@ -4,6 +4,8 @@ from django.conf import settings
 from django.db import models
 from django.utils import translation
 
+from uuidfield.fields import UUIDField
+
 import amo.models
 import amo.utils
 from amo.urlresolvers import reverse
@@ -100,6 +102,20 @@ class Platform(amo.models.ModelBase):
         db_table = 'platforms'
 
 
+class FileUpload(amo.models.ModelBase):
+    """Created when a file is uploaded for validation/submission."""
+    uuid = UUIDField(primary_key=True, auto=True)
+    path = models.CharField(max_length=255)
+    name = models.CharField(max_length=255,
+                            help_text="The user's original filename")
+    user = models.ForeignKey('users.UserProfile', null=True)
+    validation = models.TextField(null=True)
+    task_error = models.TextField(null=True)
+
+    class Meta(amo.models.ModelBase.Meta):
+        db_table = 'file_uploads'
+
+
 class TestCase(amo.models.ModelBase):
     test_group = models.ForeignKey('TestGroup')
     help_link = models.CharField(max_length=255, blank=True,

apps/reviews/models.py

@@ -1,19 +1,16 @@
 from datetime import datetime, timedelta
-import itertools
 
 from django.conf import settings
 from django.core.cache import cache
 from django.db import models
-from django.utils import translation
 
 import bleach
-from celery.decorators import task
+from celeryutils import task
 from tower import ugettext_lazy as _
 
 import amo.models
 from amo.urlresolvers import reverse
 from translations.fields import TranslatedField
-from translations.models import Translation
 from users.models import UserProfile
 from versions.models import Version
 

apps/users/tasks.py

@@ -1,20 +1,15 @@
-from datetime import datetime
 import os
-import random
 
 from django.conf import settings
-from django.contrib.auth.models import User as DjangoUser
-from django.db import IntegrityError
 from django.db import connection, transaction
 
 import commonware.log
-from celery.decorators import task
+from celeryutils import task
 from easy_thumbnails import processors
 from PIL import Image
 
 import amo.signals
 from . import cron
-from amo.utils import slugify
 from users.models import UserProfile
 
 task_log = commonware.log.getLogger('z.task')

migrations/75-piston.sql

@@ -3,8 +3,7 @@ CREATE TABLE `piston_nonce` (
     `token_key` varchar(18) NOT NULL,
     `consumer_key` varchar(18) NOT NULL,
     `key` varchar(255) NOT NULL
-)
-;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 CREATE TABLE `piston_consumer` (
     `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
     `name` varchar(255) NOT NULL,
@@ -13,8 +12,7 @@ CREATE TABLE `piston_consumer` (
     `secret` varchar(32) NOT NULL,
     `status` varchar(16) NOT NULL,
     `user_id` integer
-)
-;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 ALTER TABLE `piston_consumer` ADD CONSTRAINT `user_id_refs_id_aad30107` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
 CREATE TABLE `piston_token` (
     `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
@@ -28,8 +26,7 @@ CREATE TABLE `piston_token` (
     `consumer_id` integer NOT NULL,
     `callback` varchar(255),
     `callback_confirmed` bool NOT NULL
-)
-;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 ALTER TABLE `piston_token` ADD CONSTRAINT `user_id_refs_id_efc02d17` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
 ALTER TABLE `piston_token` ADD CONSTRAINT `consumer_id_refs_id_85f42355` FOREIGN KEY (`consumer_id`) REFERENCES `piston_consumer` (`id`);
 CREATE INDEX `piston_consumer_fbfc09f1` ON `piston_consumer` (`user_id`);

migrations/79-file-uploads.sql

@@ -0,0 +1,14 @@
+DROP TABLE IF EXISTS `file_uploads`;
+CREATE TABLE `file_uploads` (
+    `created` datetime NOT NULL,
+    `modified` datetime NOT NULL,
+    `uuid` char(32) NOT NULL PRIMARY KEY,
+    `path` varchar(255) NOT NULL,
+    `name` varchar(255) NOT NULL,
+    `user_id` int(11) unsigned,
+    `validation` longtext,
+    `task_error` longtext
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+ALTER TABLE `file_uploads`
+    ADD CONSTRAINT FOREIGN KEY (`user_id`) REFERENCES `users` (`id`);

requirements/prod.txt

@@ -16,6 +16,7 @@ django-celery==2.0.2
 # packaging doesn't pick it up.
 importlib==1.0.2
 mongoengine==0.3
+django-uuidfield==0.1
 
 -e git://github.com/jbalogh/django-multidb-router.git#egg=django-multidb-router
 -e git://github.com/jbalogh/django-cache-machine.git@85b0f3ed4#egg=django-cache-machine

settings.py

@@ -132,11 +132,11 @@ MEDIA_URL = '/media//'
 TMP_PATH = path('tmp')
 
 # Absolute path to a writable directory shared by all servers. No trailing
-# slash.
-# Example: /data/
+# slash.  Example: /data/
 NETAPP_STORAGE = TMP_PATH
 
-# Absolute path to writable directory for storing addons
+#  File path for storing XPI/JAR files (or any files associated with an
+#  add-on). Example: /mnt/netapp_amo/addons.mozilla.org-remora/files
 ADDONS_PATH = NETAPP_STORAGE + '/addons'
 
 # Absolute path to a writable directory shared by all servers. No trailing