зеркало из https://github.com/mozilla/FlightDeck.git
Merge pull request #49 from pennyfx/timeouts
Query timeout retry, force lowercase queries, added backend support for sorting
This commit is contained in:
Коммит
07ecd1b5c5
|
@ -9,6 +9,13 @@ TYPE_CHOICES = (
|
|||
('a', 'Add-ons'),
|
||||
)
|
||||
|
||||
SORT_CHOICES = (
|
||||
('score','score'),
|
||||
('activity','activity'),
|
||||
('forked','forked'),
|
||||
('used','used')
|
||||
)
|
||||
|
||||
class SearchForm(CleanForm):
|
||||
q = forms.CharField(required=False)
|
||||
page = forms.IntegerField(required=False, initial=1)
|
||||
|
@ -17,3 +24,4 @@ class SearchForm(CleanForm):
|
|||
copies = forms.IntegerField(required=False, initial=0)
|
||||
used = forms.IntegerField(required=False, initial=0)
|
||||
activity = forms.IntegerField(required=False, initial=0)
|
||||
sort = forms.ChoiceField(required=False, choices=SORT_CHOICES)
|
||||
|
|
|
@ -67,7 +67,7 @@ def get_activity_scale():
|
|||
|
||||
ACTIVITY_CACHE_KEY = 'search:activity:average'
|
||||
|
||||
def _get_average_activity():
|
||||
def _get_average_activity():
|
||||
average = cache.get(ACTIVITY_CACHE_KEY)
|
||||
if average:
|
||||
return average
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
import commonware
|
||||
import random
|
||||
import datetime
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
|
@ -8,10 +10,13 @@ from pyes import StringQuery, FieldQuery, FieldParameter
|
|||
from elasticutils.tests import ESTestCase
|
||||
from elasticutils import F
|
||||
|
||||
from jetpack.tasks import calculate_activity_rating
|
||||
from jetpack.models import Package
|
||||
from jetpack.models import PackageRevision
|
||||
from search.helpers import package_search
|
||||
from search.cron import setup_mapping
|
||||
|
||||
|
||||
log = commonware.log.getLogger('f.test.search')
|
||||
|
||||
|
||||
|
@ -212,5 +217,28 @@ class PackageHelperSearchTest(MappedESTestCase):
|
|||
and should be the first result.
|
||||
"""
|
||||
eq_([p.name for p in data['pager'].object_list], [quux.name, baz.name])
|
||||
|
||||
|
||||
|
||||
def test_sorting_by_activity(self):
|
||||
|
||||
#create 10 packages and a random number of revisions
|
||||
now = datetime.datetime.utcnow()
|
||||
addons = []
|
||||
for i in range(10):
|
||||
addon = create_addon('addon-{0}'.format(i))
|
||||
addons.append(addon)
|
||||
for k in range(1,random.randrange(1,50)):
|
||||
r = addon.revisions.create(author=addon.author, revision_number=k)
|
||||
r.created_at=now-datetime.timedelta(random.randrange(1,365))
|
||||
super(PackageRevision, r).save()
|
||||
|
||||
calculate_activity_rating([a.pk for a in addons])
|
||||
|
||||
self.es.refresh()
|
||||
|
||||
qs = package_search().order_by('activity')
|
||||
|
||||
last_score = 1
|
||||
for r in qs:
|
||||
eq_(r.activity<last_score, True)
|
||||
last_score = r.activity
|
||||
|
||||
|
|
|
@ -7,9 +7,16 @@ from django.template import RequestContext
|
|||
from jetpack.models import Package
|
||||
from .helpers import package_search, get_activity_scale
|
||||
from .forms import SearchForm
|
||||
from pyes.urllib3.connectionpool import TimeoutError
|
||||
|
||||
log = commonware.log.getLogger('f.search')
|
||||
|
||||
SORT_MAPPING = {
|
||||
'score':'_score',
|
||||
'activity':'activity',
|
||||
'forked':'copies_count',
|
||||
'used':'times_depended',
|
||||
}
|
||||
|
||||
def search(request):
|
||||
form = SearchForm(request.GET)
|
||||
|
@ -21,7 +28,7 @@ def search(request):
|
|||
page = query.get('page') or 1
|
||||
limit = 20
|
||||
activity_map = get_activity_scale()
|
||||
|
||||
sort = SORT_MAPPING.get(query.get('sort'), '_score')
|
||||
|
||||
filters = {}
|
||||
filters['user'] = request.user
|
||||
|
@ -44,35 +51,56 @@ def search(request):
|
|||
|
||||
if query.get('activity'):
|
||||
filters['activity__gte'] = activity_map.get(str(query['activity']), 0)
|
||||
|
||||
results = {}
|
||||
facets = {}
|
||||
|
||||
|
||||
copies_facet = {'terms': {'field': 'copies_count'}}
|
||||
times_depended_facet = {'terms': {'field': 'times_depended'}}
|
||||
facets_ = {'copies': copies_facet, 'times_depended': times_depended_facet}
|
||||
if type_:
|
||||
filters['type'] = type_
|
||||
qs = package_search(q, **filters).facet(**facets_)
|
||||
try:
|
||||
results['pager'] = Paginator(qs, per_page=limit).page(page)
|
||||
except EmptyPage:
|
||||
results['pager'] = Paginator(qs, per_page=limit).page(1)
|
||||
facets = _facets(results['pager'].object_list.facets)
|
||||
facets['everyone_total'] = len(qs)
|
||||
template = 'results.html'
|
||||
else:
|
||||
# combined view
|
||||
results['addons'] = package_search(q, type='a', **filters).facet(
|
||||
**facets_)[:5]
|
||||
results['libraries'] = package_search(q, type='l', **filters).facet(
|
||||
**facets_)[:5]
|
||||
facets = _facets(results['addons'].facets)
|
||||
facets['everyone_total'] = facets['combined_total']
|
||||
template = 'aggregate.html'
|
||||
|
||||
|
||||
|
||||
|
||||
def retry_on_timeout(fn ,args, max_retry=1):
|
||||
tries = 0;
|
||||
while True:
|
||||
try:
|
||||
tries += 1
|
||||
return fn(*args)
|
||||
except TimeoutError as e:
|
||||
log.error("ES query({3}) timeout: '{0}' - {1} - {2}"
|
||||
.format(form.cleaned_data,
|
||||
"retrying" if tries<max_retry else 'forget it',
|
||||
e, tries
|
||||
))
|
||||
if tries==max_retry:
|
||||
raise e
|
||||
continue
|
||||
|
||||
def execute_search(type_, q, limit, page, filters, facets_):
|
||||
template = ''
|
||||
results={}
|
||||
facets={}
|
||||
|
||||
if type_:
|
||||
filters['type'] = type_
|
||||
qs = package_search(q, **filters).order_by(sort).facet(**facets_)
|
||||
try:
|
||||
results['pager'] = Paginator(qs, per_page=limit).page(page)
|
||||
except EmptyPage:
|
||||
results['pager'] = Paginator(qs, per_page=limit).page(1)
|
||||
facets = _facets(results['pager'].object_list.facets)
|
||||
facets['everyone_total'] = len(qs)
|
||||
template = 'results.html'
|
||||
else:
|
||||
# combined view
|
||||
results['addons'] = package_search(q, type='a', **filters) \
|
||||
.order_by(sort).facet(**facets_)[:5]
|
||||
results['libraries'] = package_search(q, type='l', **filters) \
|
||||
.order_by(sort).facet(**facets_)[:5]
|
||||
facets = _facets(results['addons'].facets)
|
||||
facets['everyone_total'] = facets['combined_total']
|
||||
template = 'aggregate.html'
|
||||
return template,results,facets
|
||||
|
||||
template, results, facets = retry_on_timeout(execute_search,
|
||||
[type_,q,limit,page,filters,facets_] , 2)
|
||||
|
||||
ctx = {
|
||||
'q': q,
|
||||
'page': 'search',
|
||||
|
@ -80,9 +108,10 @@ def search(request):
|
|||
'query': query,
|
||||
'type': types.get(type_, None)
|
||||
}
|
||||
|
||||
ctx.update(results)
|
||||
ctx.update(facets)
|
||||
|
||||
|
||||
if request.is_ajax():
|
||||
template = 'ajax/' + template
|
||||
return _render(request, template, ctx)
|
||||
|
|
Загрузка…
Ссылка в новой задаче