Merge branch 'master' into bug-681906-upload_in_dashboard

This commit is contained in:
Piotr Zalewa 2011-09-02 09:16:50 +01:00
Родитель 02dbaf39c0 8830c58b0c
Коммит 258bf60b3a
24 изменённых файлов: 379 добавлений и 93 удалений

3
.gitmodules поставляемый
Просмотреть файл

@ -7,3 +7,6 @@
[submodule "lib/addon-sdk-1.0"]
path = lib/addon-sdk-1.0
url = git://github.com/mozilla/addon-sdk.git
[submodule "lib/addon-sdk-1.1rc1"]
path = lib/addon-sdk-1.1rc1
url = git://github.com/mozilla/addon-sdk.git

Просмотреть файл

Просмотреть файл

Просмотреть файл

@ -0,0 +1,68 @@
import commonware
import shutil
import os
from mock import Mock
from nose.tools import eq_
from test_utils import TestCase
from django.contrib.auth.models import User
from django.conf import settings
from amo.tasks import upload_to_amo
from base.templatetags.base_helpers import hashtag
from jetpack.models import (Package, PackageRevision, STATUS_UNREVIEWED,
STATUS_PUBLIC)
from utils.amo import AMOOAuth
log = commonware.log.getLogger('f.test')
class UploadTest(TestCase):
fixtures = ['mozilla_user', 'users', 'core_sdk', 'packages']
ADDON_AMO_ID = 1
def setUp(self):
self.author = User.objects.get(username='john')
self.addonrev = Package.objects.get(name='test-addon',
author__username='john').latest
self.hashtag = hashtag()
self.amo = AMOOAuth(domain=settings.AMOOAUTH_DOMAIN,
port=settings.AMOOAUTH_PORT,
protocol=settings.AMOOAUTH_PROTOCOL,
prefix=settings.AMOOAUTH_PREFIX)
def test_create_new_amo_addon(self):
AMOOAuth._send = Mock(return_value={
'status': STATUS_UNREVIEWED,
'id': self.ADDON_AMO_ID})
upload_to_amo(self.addonrev.pk, self.hashtag)
# checking status and other attributes
addonrev = Package.objects.get(name='test-addon',
author__username='john').latest
eq_(addonrev.package.amo_id, self.ADDON_AMO_ID)
eq_(addonrev.amo_version_name, 'initial')
eq_(addonrev.amo_status, STATUS_UNREVIEWED)
# check if right API was called
assert 'POST' in AMOOAuth._send.call_args[0]
assert self.amo.url('addon') in AMOOAuth._send.call_args[0]
def test_update_amo_addon(self):
AMOOAuth._send = Mock(return_value={'status': STATUS_UNREVIEWED})
# set add-on as uploaded
self.addonrev.amo_status = STATUS_PUBLIC
self.addonrev.amo_version_name = self.addonrev.get_version_name()
self.addonrev.package.amo_id = self.ADDON_AMO_ID
# create a new revision
self.addonrev.save()
upload_to_amo(self.addonrev.pk, self.hashtag)
# checking status and other attributes
addonrev = Package.objects.get(name='test-addon',
author__username='john').latest
eq_(addonrev.amo_version_name,
'initial.rev%d' % addonrev.revision_number)
eq_(addonrev.amo_status, STATUS_UNREVIEWED)
# check if right API was called
assert 'POST' in AMOOAuth._send.call_args[0]
assert self.amo.url('version') % self.ADDON_AMO_ID in AMOOAuth._send.call_args[0]

24
apps/base/forms.py Normal file
Просмотреть файл

@ -0,0 +1,24 @@
from django import forms
from django.forms.util import ErrorDict
class CleanForm(forms.Form):
def full_clean(self):
"""
Cleans self.data and populates self._errors and self.cleaned_data.
Does not remove cleaned_data if there are errors.
"""
self._errors = ErrorDict()
if not self.is_bound: # Stop further processing.
return
self.cleaned_data = {}
# If the form is permitted to be empty, and none of the form data
# has changed from the initial data, short circuit any validation.
if self.empty_permitted and not self.has_changed():
return
self._clean_fields()
self._clean_form()
self._post_clean()

Просмотреть файл

@ -54,5 +54,18 @@
{% else %}
<p>Can't connect to ElasticSearch cluster.</p>
{% endif %}
<h2 class="UI_Heading" style="padding-top: 3em">Memcached</h2>
{% if memcached %}
<ul>
{% for ip, port, result in memcached %}
<li>
{{ ip }}:{{ port }}
{{ result|yesno:"Success, FAILED"}}
</li>
{% endfor %}
</ul>
{% else %}
<p>There are no memcached servers!</p>
{% endif %}
{% endblock %}

Просмотреть файл

@ -1,4 +1,5 @@
import os
import socket
import simplejson
import commonware.log
@ -48,6 +49,7 @@ def monitor(request):
status = True
data = {}
# Check Read/Write
filepaths = [
(settings.UPLOAD_DIR, os.R_OK | os.W_OK, 'We want read + write.'),
]
@ -83,22 +85,64 @@ def monitor(request):
}
data['filepaths'] = filepath_results
template = loader.get_template('monitor.html')
# Check celery
try:
data['celery_responses'] = CeleryResponse.objects.all()
except:
status = False
# Check ElasticSearch
try:
es = get_es()
data['es_health'] = es.cluster_health()
data['es_health']['version'] = es.collect_info()['server']['version']['number']
except:
if data['es_health']['status'] =='red':
status = False
log.warning('ElasticSearch cluster health was red.')
except Exception, e:
status = False
log.critical('Failed to connect to ElasticSearch: %s' % e)
# Check memcached
memcache = getattr(settings, 'CACHES', {}).get('default')
memcache_results = []
if memcache and 'memcached' in memcache['BACKEND']:
hosts = memcache['LOCATION']
if not isinstance(hosts, (tuple, list)):
hosts = [hosts]
for host in hosts:
ip, port = host.split(':')
try:
s = socket.socket()
s.connect((ip, int(port)))
except Exception, e:
status = False
result = False
log.critical('Failed to connect to memcached (%s): %s'
% (host, e))
else:
result = True
finally:
s.close()
memcache_results.append((ip, port, result))
if len(memcache_results) < 2:
status = False
log.warning('You should have 2+ memcache servers. '
'You have %d.' % len(memcache_results))
if not memcache_results:
status = False
log.info('Memcached is not configured.')
data['memcached'] = memcache_results
# Check Redis
# TODO: we don't currently use redis
context = RequestContext(request, data)
status = 200 if status else 500
template = loader.get_template('monitor.html')
return HttpResponse(template.render(context), status=status)

Просмотреть файл

@ -204,7 +204,7 @@ class PackageRevision(BaseModel):
# TODO: update jetpack ID if needed
else:
# create addon on AMO
log.info('AMOOAUTHAPI: creating addon %s version %s' % (
log.info('AMOOAUTHAPI: creating addon %s amo_version %s' % (
self, self.amo_version_name))
data.update({'platform': 'all'})
try:
@ -222,7 +222,6 @@ class PackageRevision(BaseModel):
self.package.save()
os.remove(xpi_path)
log.debug(self.amo_status)
if error:
raise error
@ -1033,7 +1032,9 @@ class PackageRevision(BaseModel):
if save:
# save as new version
self.save()
return self.dependencies.add(dep)
ret = self.dependencies.add(dep)
dep.package.refresh_index()
return ret
def compare_dependency_conflicts(self, dep, as_upgrade=False):
"""
@ -1117,7 +1118,9 @@ class PackageRevision(BaseModel):
'dependency (%s) removed' % dep.name)
# save as new version
self.save()
return self.dependencies.remove(dep)
ret = self.dependencies.remove(dep)
dep.package.refresh_index()
return ret
raise DependencyException(
'There is no such library in this %s' \
% self.package.get_type_name())
@ -1740,6 +1743,11 @@ class Package(BaseModel, SearchMixin):
except PackageRevision.DoesNotExist:
pass
if self.is_library():
data['times_depended'] = (Package.objects
.filter(latest__dependencies__in=self.revisions.all())
.count())
try:
es.index(data, settings.ES_INDEX, self._meta.db_table, id=self.id,
bulk=bulk)

Просмотреть файл

@ -5,6 +5,8 @@ import simplejson
import os
import commonware
from django.http import Http404
from base.shortcuts import get_object_with_related_or_404
from jetpack.models import Package, PackageRevision
from jetpack.errors import ManifestNotValid
@ -24,6 +26,10 @@ def get_package_revision(id_name, type_id,
package = get_object_with_related_or_404(Package, id_number=id_name,
type=type_id)
package_revision = package.latest if latest else package.version
if not package_revision:
log.critical("Package %s by %s has no latest or version "
"revision" % (package, package.author))
raise Http404
elif revision_number:
# get version given by revision number
@ -35,6 +41,11 @@ def get_package_revision(id_name, type_id,
package_revision = get_object_with_related_or_404(PackageRevision,
package__id_number=id_name, package__type=type_id,
version_name=version_name)
# For unknown reason some revisions are not linked to any package
if not package_revision.package:
log.critical("PackageRevision %d by %s is not related to any "
"Package" % (package_revision.pk, package_revision.author))
raise Http404
return package_revision

Просмотреть файл

@ -44,9 +44,11 @@
<li id="download" title="Download" class="UI_Editor_Menu_Button Icon_download">
<a target="_new" href="{{ revision.get_download_xpi_url }}"><span></span></a>
</li>
{% comment %}
<li id="upload_to_amo" title="Upload" class="UI_Editor_Menu_Button Icon_upload">
<a target="_new" href="{{ revision.get_upload_to_amo_url }}"><span></span></a>
<a target="_}ew" href="{{ revision.get_upload_to_amo_url }}"><span></span></a>
</li>
{% endcomment %}
<li class="UI_Editor_Menu_Separator"></li>
{% endblock %}

Просмотреть файл

@ -22,7 +22,6 @@ from jetpack.errors import FilenameExistException
log = commonware.log.getLogger('f.test')
class AttachmentTest(TestCase):
"""Testing attachment methods."""
@ -106,6 +105,8 @@ class TestViews(TestCase):
if not os.path.exists(settings.UPLOAD_DIR):
os.makedirs(settings.UPLOAD_DIR)
self.tempdir = tempfile.mkdtemp()
self.author = User.objects.get(username='john')
self.author.set_password('password')
self.author.save()
@ -119,6 +120,9 @@ class TestViews(TestCase):
self.change_url = self.get_change_url(self.revision.revision_number)
self.client.login(username=self.author.username, password='password')
def tearDown(self):
shutil.rmtree(self.tempdir)
def test_attachment_error(self):
res = self.client.post(self.add_url, {})
eq_(res.status_code, 403)
@ -149,14 +153,15 @@ class TestViews(TestCase):
def upload(self, url, data, filename):
# A post that matches the JS and uses raw_post_data.
f = open('upload_attachment', 'w')
attachment = os.path.join(self.tempdir, 'upload_attachment')
f = open(attachment, 'w')
f.write(data)
f.close()
f = open('upload_attachment', 'r')
f = open(attachment, 'r')
resp = self.client.post(url, { 'upload_attachment': f },
HTTP_X_FILE_NAME=filename)
f.close()
os.unlink('upload_attachment')
os.unlink(attachment)
return resp
def test_attachment_path(self):

Просмотреть файл

@ -2,33 +2,18 @@ from django import forms
from django.forms.util import ErrorDict
from django.contrib.auth.models import User
from base.forms import CleanForm
TYPE_CHOICES = (
('l', 'Libraries'),
('a', 'Add-ons'),
)
class SearchForm(forms.Form):
class SearchForm(CleanForm):
q = forms.CharField(required=False)
page = forms.IntegerField(required=False, initial=1)
type = forms.ChoiceField(required=False, choices=TYPE_CHOICES)
author = forms.ModelChoiceField(required=False, queryset=User.objects.all())
copies = forms.IntegerField(required=False, initial=0)
used = forms.IntegerField(required=False, initial=0)
def full_clean(self):
"""
Cleans self.data and populates self._errors and self.cleaned_data.
Does not remove cleaned_data if there are errors.
"""
self._errors = ErrorDict()
if not self.is_bound: # Stop further processing.
return
self.cleaned_data = {}
# If the form is permitted to be empty, and none of the form data
# has changed from the initial data, short circuit any validation.
if self.empty_permitted and not self.has_changed():
return
self._clean_fields()
self._clean_form()
self._post_clean()

Просмотреть файл

@ -7,27 +7,28 @@ def package_search(searchq='', user=None, score_on=None, **filters):
# This is a filtered query, that says we want to do a query, but not have
# to deal with version_text='initial' or 'copy'
# nested awesomenezz!
notInitialOrCopy = ~(F(version_name='initial') | F(version_name='copy'))
qs = (Package.search().filter(notInitialOrCopy, **filters)
.facet(types={'terms': {'field': 'type'},
'facet_filter': notInitialOrCopy.filters}))
qs = Package.search().filter(notInitialOrCopy, **filters)
# Add type facet (minus any type filter)
facetFilter = dict((k, v) for k, v in filters.items() if k != 'type')
if facetFilter:
facetFilter = notInitialOrCopy & F(**facetFilter)
else:
facetFilter = notInitialOrCopy
qs = qs.facet(types={'terms': {'field': 'type'},
'facet_filter': facetFilter.filters})
if searchq:
qs = qs.query(or_=package_query(searchq))
if user and user.is_authenticated():
qs = qs.facet(author={'terms': {
'field': 'author',
'script':'term == %d ? true : false' % user.id}
})
#if score_on:
# q.score(script='_score * doc[\'%s\'].value' % score_on)
return qs

Просмотреть файл

@ -1,15 +1,22 @@
{% load base_helpers %}
<section id="SearchResults">
{% if addons %}
<h2 class="UI_Heading">Add-on Results ({{ addon_total }})</h2>
{% for package in addons %}
{% include "_package_result.html" %}
{% endfor %}
{% if addon_total > 5 %}
<p class="see-more"><a href="?{% querystring type='a' %}">See all {{ addon_total}} matching add-ons &rarr;</a></p>
{% endif %}
{% endif %}
{% if libraries %}
<h2 class="UI_Heading">Library Results ({{ library_total }})</h2>
{% for package in libraries %}
{% include "_package_result.html" %}
{% endfor %}
{% if library_total > 5 %}
<p class="see-more"><a href="?{% querystring type='l' %}">See all {{ library_total }} matching libraries &rarr;</a></p>
{% endif %}
{% endif %}
{% if not addons and not libraries %}

Просмотреть файл

@ -51,5 +51,17 @@
</div>
</li>
{% endif %}
{% if max_times_depended %}
<li id="UsedFilter">
Used by
<span class="slider-value">{{ query.used }}</span>
or more packages:
<div class="slider">
<div class="knob"></div>
<span class="range start">0</span>
<span class="range end">{{ max_times_depended }}</span>
</div>
</li>
{% endif %}
</ul>
</section>

Просмотреть файл

@ -10,6 +10,7 @@ from elasticutils import F
from jetpack.models import Package
from search.helpers import package_search
from search.cron import setup_mapping
log = commonware.log.getLogger('f.test.search')
@ -28,7 +29,20 @@ def create_package(name, type, **kwargs):
**kwargs)
class TestSearch(ESTestCase):
class MappedESTestCase(ESTestCase):
"""
FlightDeck has special mapping that needs to be put into the index for
the tests to work, so put the mapping each time the index is re-created.
"""
@classmethod
def setup_class(cls):
super(MappedESTestCase, cls).setup_class()
setup_mapping()
class TestSearch(MappedESTestCase):
fixtures = ('mozilla_user', 'users', 'core_sdk')
def test_index(self, name='zool'):
@ -111,7 +125,23 @@ class TestSearch(ESTestCase):
eq_(r['hits']['total'], 0)
class PackageSearchTest(ESTestCase):
class PackageSearchTest(MappedESTestCase):
fixtures = ('mozilla_user', 'users', 'core_sdk',)
def test_times_depended_on(self):
foo = create_library('foooooo')
bar = create_addon('barrrr')
bar.latest.dependency_add(foo.latest)
self.es.refresh()
qs = Package.search().filter(times_depended__gte=1)
eq_(len(qs), 1)
eq_(qs[0], foo)
class PackageHelperSearchTest(MappedESTestCase):
"""
search.helpers.package_search has some built-in sane defaults when
searching for Packages.
@ -145,6 +175,26 @@ class PackageSearchTest(ESTestCase):
data = package_search('foo')
eq_(1, len(data))
def test_type_facet_filter(self):
""")
Type facet should not have a type filter in it's facet_filter.
"""
buzz = create_addon('buzz lightyear')
buzz.latest.set_version('Infinity')
toystory = create_library('the toy story')
toystory.latest.set_version('1')
self.es.refresh()
data = package_search(type='a')
eq_(1, len(data))
types = dict((f['term'], f['count']) for f in data.facets['types'])
eq_(1, types.get('a'))
eq_(1, types.get('l'))
def test_custom_scoring(self):
raise SkipTest()
baz = create_addon('score baz')

Просмотреть файл

@ -29,7 +29,7 @@ class TestSearchViews(ESTestCase):
"""Should not error if non-int value is passed for the page number"""
create_addon('derp')
url = '%s?q=%s&page=%s' % (reverse('search_by_type', args=['addon']),
url = '%s?q=%s&page=%s' % (reverse('search'),
'test', '-^')
resp = self.client.get(url)

Просмотреть файл

@ -28,16 +28,29 @@ def search(request):
author = query.get('author')
if author:
filters['author'] = author.id
if query.get('copies'):
filters['copies_count__gte'] = query['copies']
else:
query['copies'] = 0
if query.get('used') and type_ != 'a':
# Add-ons can't be depended upon, so this query would filter out
# every single Add-on
filters['times_depended__gte'] = query['used']
else:
query['used'] = 0
results = {}
facets = {}
copies_facet = {'terms': {'field': 'copies_count'}}
times_depended_facet = {'terms': {'field': 'times_depended'}}
facets_ = {'copies': copies_facet, 'times_depended': times_depended_facet}
if type_:
filters['type'] = type_
qs = package_search(q, **filters).facet(copies={'terms':
{'field':'copies_count'}})
qs = package_search(q, **filters).facet(**facets_)
try:
results['pager'] = Paginator(qs, per_page=limit).page(page)
except EmptyPage:
@ -48,8 +61,9 @@ def search(request):
else:
# combined view
results['addons'] = package_search(q, type='a', **filters).facet(
copies={'terms':{'field':'copies_count'}})[:5]
results['libraries'] = package_search(q, type='l', **filters)[:5]
**facets_)[:5]
results['libraries'] = package_search(q, type='l', **filters).facet(
**facets_)[:5]
facets = _facets(results['addons'].facets)
facets['everyone_total'] = facets['combined_total']
template = 'aggregate.html'
@ -101,10 +115,19 @@ def _facets(facets):
max_ = copies_steps.pop()
max_copies = max(max_copies, max_)
max_times_depended = 0
if 'times_depended' in facets:
depended_steps = [t['term'] for t in facets['times_depended']]
if depended_steps:
depended_steps.sort()
max_ = depended_steps.pop()
max_times_depended = max(max_times_depended, max_)
return {
'addon_total': type_totals.get('a', 0),
'library_total': type_totals.get('l', 0),
'my_total': my_total,
'combined_total': type_totals.get('a', 0) + type_totals.get('l', 0),
'max_copies': max_copies,
'max_times_depended': max_times_depended
}

Просмотреть файл

@ -26,3 +26,10 @@ From the help::
optional arguments:
-h, --help show this help message and exit
-d DIR, --dir DIR path to the vendor directory
However, because ``vendor`` is a submodule, vending-machine should not
be used in it's default behavior. Instead, FlightDeck-lib should be
checked out to a separate folder, and you should set the ``-d`` argument
of vend::
vend -d ./FlightDeck-lib add elasticutils

1
lib/addon-sdk-1.1rc1 Submodule

@ -0,0 +1 @@
Subproject commit 4cd0c39219e3bb5dc1c1c8ea0676304807d8fe27

Просмотреть файл

@ -18,7 +18,7 @@ body {
border-color:#35467E #2C3C70 #1E2A52;
cursor:pointer;
height:5px;
margin:10px 0;
margin:10px 0 30px;
position:relative;
}
@ -158,6 +158,13 @@ body {
opacity:1;
}
#SearchResults .see-more {
margin:10px 0;
}
#SearchResults .see-more a {
color:#478CDE;
}
#SearchResults .UI_Pagination {
margin-top:40px;
}

Просмотреть файл

@ -17,7 +17,7 @@ var SearchResult = new Class({
this.content = tree;
this.show();
}.bind(this)
}).send();
}).send('xhr');
return this;
},
@ -44,13 +44,16 @@ var SearchResult = new Class({
newSidebar.replaces(sidebar);
}
if (!this.slider) {
this.slider = SearchResult.setupUI();
if (!this.ui) {
SearchResult.setupUI(this);
} else {
this.slider.sanityCheck = false;
var loc = new URI(this.url);
this.slider.set(loc.getData('copies') || 0);
this.slider.sanityCheck = true;
Object.each(this.ui.sliders, function(slider, name) {
slider.sanityCheck = false;
slider.set(loc.getData(name) || 0);
slider.sanityCheck = true;
});
}
return this;
@ -80,29 +83,37 @@ SearchResult.page = function(url) {
SearchResult.fetch(String(window.location));
};
SearchResult.setupUI = function() {
var copies = $('CopiesFilter');
if (copies) {
var cSlider = copies.getElement('.slider'),
cKnob = cSlider.getElement('.knob'),
cValue = copies.getElement('.slider-value'),
cRangeEnd = cSlider.getElement('.range.end'),
end = cRangeEnd.get('text').toInt();
SearchResult.setupUI = function(result) {
var ui = { sliders: {} };
if (result) result.ui = ui;
var initialStep = Math.max(0, cValue.get('text').toInt() || 0);
var filters = ['Copies', 'Used'];
filters.forEach(function(filter) {
var container = $(filter + 'Filter'),
dataKey = filter.toLowerCase();
var copiesSlider = new Slider(cSlider, cKnob, {
if (container) {
var sliderEl = container.getElement('.slider'),
knobEl = sliderEl.getElement('.knob'),
valueEl = container.getElement('.slider-value'),
rangeEndEl = sliderEl.getElement('.range.end'),
end = rangeEndEl.get('text').toInt();
var initialStep = Math.max(0, valueEl.get('text').toInt() || 0);
var slider = new Slider(sliderEl, knobEl, {
//snap: true,
range: [0, end],
initialStep: initialStep,
onChange: function(step) {
cValue.set('text', step);
valueEl.set('text', step);
},
onComplete: function(step) {
if (!this.sanityCheck) return;
var loc = new URI(String(window.location));
loc.setData('copies', step);
loc.setData(dataKey, step);
SearchResult.page(loc);
}
});
@ -112,10 +123,11 @@ SearchResult.setupUI = function() {
// check our sanity by stopping all onComplete's that happen
// during initialization, since sanityCheck get's set to true
// _after_ construction.
copiesSlider.sanityCheck = true;
slider.sanityCheck = true;
return copiesSlider;
ui.sliders[dataKey] = slider;
}
});
};

Просмотреть файл

@ -88,3 +88,6 @@ def update_flightdeck(ctx):
# Run management commands like this:
# manage_cmd(ctx, 'cmd')
# For 0.9.10
manage_cmd(ctx, 'cron index_all')

Просмотреть файл

@ -12,7 +12,7 @@ ROOT = os.path.dirname(os.path.abspath(__file__))
path = lambda *a: os.path.join(ROOT, *a)
# Set the project version
PROJECT_VERSION = "0.9.9b"
PROJECT_VERSION = "0.9.10"
# TODO: This should be handled by prod in a settings_local. By default, we
# shouldn't be in prod mode