import json import logging import os import shutil import socket import sys import traceback import urllib2 import urlparse from django.conf import settings from django.core.management import call_command from celeryutils import task from tower import ugettext as _ import amo from amo.decorators import write, set_modified_on from amo.utils import slugify, resize_image from addons.models import Addon from applications.management.commands import dump_apps from applications.models import Application, AppVersion from files.models import FileUpload, File, FileValidation from PIL import Image log = logging.getLogger('z.devhub.task') @task @write def validator(upload_id, **kw): if not settings.VALIDATE_ADDONS: return None log.info('VALIDATING: %s' % upload_id) upload = FileUpload.objects.get(pk=upload_id) try: result = run_validator(upload.path) upload.validation = result upload.save() # We want to hit the custom save(). except: # Store the error with the FileUpload job, then raise # it for normal logging. tb = traceback.format_exception(*sys.exc_info()) upload.update(task_error=''.join(tb)) raise @task @write def compatibility_check(upload_id, app_guid, appversion_str, **kw): if not settings.VALIDATE_ADDONS: return None log.info('COMPAT CHECK for upload %s / app %s version %s' % (upload_id, app_guid, appversion_str)) upload = FileUpload.objects.get(pk=upload_id) app = Application.objects.get(guid=app_guid) appver = AppVersion.objects.get(application=app, version=appversion_str) try: result = run_validator(upload.path, for_appversions={app_guid: [appversion_str]}, test_all_tiers=True, overrides={'targetapp_maxVersion': {app_guid: appversion_str}}) upload.validation = result upload.compat_with_app = app upload.compat_with_appver = appver upload.save() # We want to hit the custom save(). except: # Store the error with the FileUpload job, then raise # it for normal logging. tb = traceback.format_exception(*sys.exc_info()) upload.update(task_error=''.join(tb)) raise @task @write def file_validator(file_id, **kw): if not settings.VALIDATE_ADDONS: return None log.info('VALIDATING file: %s' % file_id) file = File.objects.get(pk=file_id) # Unlike upload validation, let the validator # raise an exception if there is one. result = run_validator(file.file_path) return FileValidation.from_json(file, result) def run_validator(file_path, for_appversions=None, test_all_tiers=False, overrides=None): """A pre-configured wrapper around the addon validator. *file_path* Path to addon / extension file to validate. *for_appversions=None* An optional dict of application versions to validate this addon for. The key is an application GUID and its value is a list of versions. *test_all_tiers=False* When False (default) the validator will not continue if it encounters fatal errors. When True, all tests in all tiers are run. See bug 615426 for discussion on this default. *overrides=None* Normally the validator gets info from install.rdf but there are a few things we need to override. See validator for supported overrides. Example: {'targetapp_maxVersion': {'': ''}} To validate the addon for compatibility with Firefox 5 and 6, you'd pass in:: for_appversions={amo.FIREFOX.guid: ['5.0.*', '6.0.*']} Not all application versions will have a set of registered compatibility tests. """ from validator.validate import validate # TODO(Kumar) remove this when validator is fixed, see bug 620503 from validator.testcases import scripting scripting.SPIDERMONKEY_INSTALLATION = settings.SPIDERMONKEY import validator.constants validator.constants.SPIDERMONKEY_INSTALLATION = settings.SPIDERMONKEY apps = dump_apps.Command.JSON_PATH if not os.path.exists(apps): call_command('dump_apps') return validate(file_path, for_appversions=for_appversions, format='json', # When False, this flag says to stop testing after one # tier fails. determined=test_all_tiers, approved_applications=apps, spidermonkey=settings.SPIDERMONKEY, overrides=overrides) @task(rate_limit='4/m') @write def flag_binary(ids, **kw): log.info('[%s@%s] Flagging binary addons starting with id: %s...' % (len(ids), flag_binary.rate_limit, ids[0])) addons = Addon.objects.filter(pk__in=ids).no_transforms() for addon in addons: try: log.info('Validating addon with id: %s' % addon.pk) file = File.objects.filter(version__addon=addon).latest('created') result = run_validator(file.file_path) binary = (json.loads(result)['metadata'] .get('contains_binary_extension', False)) log.info('Setting binary for addon with id: %s to %s' % (addon.pk, binary)) addon.update(binary=binary) except Exception, err: log.error('Failed to run validation on addon id: %s, %s' % (addon.pk, err)) @task @set_modified_on def resize_icon(src, dst, size, **kw): """Resizes addon icons.""" log.info('[1@None] Resizing icon: %s' % dst) try: if isinstance(size, list): for s in size: resize_image(src, '%s-%s.png' % (dst, s), (s, s), remove_src=False) os.remove(src) else: resize_image(src, dst, (size, size), remove_src=True) return True except Exception, e: log.error("Error saving addon icon: %s" % e) @task @set_modified_on def resize_preview(src, thumb_dst, full_dst, **kw): """Resizes preview images.""" log.info('[1@None] Resizing preview: %s' % thumb_dst) try: # Generate the thumb. size = amo.ADDON_PREVIEW_SIZES[0] resize_image(src, thumb_dst, size, remove_src=False) # Resize the original. size = amo.ADDON_PREVIEW_SIZES[1] resize_image(src, full_dst, size, remove_src=True) return True except Exception, e: log.error("Error saving preview: %s" % e) @task @set_modified_on def resize_preview_store_size(src, instance, **kw): """Resizes preview images and stores the sizes on the preview.""" thumb_dst, full_dst = instance.thumbnail_path, instance.image_path sizes = {} log.info('[1@None] Resizing preview and storing size: %s' % thumb_dst) try: sizes['thumbnail'] = resize_image(src, thumb_dst, amo.ADDON_PREVIEW_SIZES[0], remove_src=False) sizes['image'] = resize_image(src, full_dst, amo.ADDON_PREVIEW_SIZES[1], remove_src=False) instance.sizes = sizes instance.save() return True except Exception, e: log.error("Error saving preview: %s" % e) @task @write def get_preview_sizes(ids, **kw): log.info('[%s@%s] Getting preview sizes for addons starting at id: %s...' % (len(ids), get_preview_sizes.rate_limit, ids[0])) addons = Addon.objects.filter(pk__in=ids).no_transforms() for addon in addons: previews = addon.previews.all() log.info('Found %s previews for: %s' % (previews.count(), addon.pk)) for preview in previews: try: log.info('Getting size for preview: %s' % preview.pk) sizes = { 'thumbnail': Image.open(preview.thumbnail_path).size, 'image': Image.open(preview.image_path).size, } preview.update(sizes=sizes) except Exception, err: log.error('Failed to find size of preview: %s, error: %s' % (addon.pk, err)) @task @write def convert_purified(ids, **kw): log.info('[%s@%s] Converting fields to purified starting at id: %s...' % (len(ids), convert_purified.rate_limit, ids[0])) fields = ['the_reason', 'the_future'] for addon in Addon.objects.filter(pk__in=ids): flag = False for field in fields: value = getattr(addon, field) if value: value.clean() if (value.localized_string_clean != value.localized_string): flag = True if flag: log.info('Saving addon: %s to purify fields' % addon.pk) addon.save() @task def packager(data, feature_set, **kw): """Build an add-on based on input data.""" log.info('[1@None] Packaging add-on') # "Lock" the file by putting .lock in its name. from devhub.views import packager_path xpi_path = packager_path('%s.lock' % data['uuid']) log.info('Saving package to: %s' % xpi_path) from packager.main import packager data['slug'] = slugify(data['name']).replace('-', '_') features = set([k for k, v in feature_set.items() if v]) packager(data, xpi_path, features) # Unlock the file and make it available. try: shutil.move(xpi_path, packager_path(data['uuid'])) except IOError: log.error('Error unlocking add-on: %s' % xpi_path) def failed_validation(*messages): """Return a validation object that looks like the add-on validator.""" return json.dumps({'errors': 1, 'success': False, 'messages': messages}) def _fetch_manifest(url): try: response = urllib2.urlopen(url, timeout=5) except urllib2.HTTPError, e: raise Exception(_('%s responded with %s (%s).') % (url, e.code, e.msg)) except urllib2.URLError, e: # Unpack the URLError to try and find a useful message. if isinstance(e.reason, socket.timeout): raise Exception(_('Connection to "%s" timed out.') % url) elif isinstance(e.reason, socket.gaierror): raise Exception(_('Could not contact host at "%s".') % url) else: raise Exception(str(e.reason)) content_type = 'application/x-web-app-manifest+json' if not response.headers.get('Content-Type', '').startswith(content_type): if 'Content-Type' in response.headers: raise Exception(_('Your manifest must be served with the HTTP ' 'header "Content-Type: %s". We saw "%s".') % (content_type, response.headers['Content-Type'])) else: raise Exception(_('Your manifest must be served with the HTTP ' 'header "Content-Type: %s".') % content_type) # Read one extra byte. Reject if it's too big so we don't have issues # downloading huge files. content = response.read(settings.MAX_WEBAPP_UPLOAD_SIZE + 1) if len(content) > settings.MAX_WEBAPP_UPLOAD_SIZE: raise Exception(_('Your manifest must be less than %s bytes.') % settings.MAX_WEBAPP_UPLOAD_SIZE) return content @task def fetch_manifest(url, upload_pk=None, **kw): log.info(u'[1@None] Fetching manifest: %s.' % url) upload = FileUpload.objects.get(pk=upload_pk) try: content = _fetch_manifest(url) except Exception, e: # Drop a message in the validation slot and bail. upload.update(validation=failed_validation(e.message)) raise upload.add_file([content], url, len(content)) # Send the upload to the validator. validator(upload.pk)