Merge pull request #49 from pennyfx/timeouts

Query timeout retry, force lowercase queries, added backend support for sorting
This commit is contained in:
Arron Schaar 2011-10-31 13:18:54 -07:00
Родитель afb5ee614b c6f5889892
Коммит 07ecd1b5c5
4 изменённых файлов: 96 добавлений и 31 удалений

Просмотреть файл

@ -9,6 +9,13 @@ TYPE_CHOICES = (
('a', 'Add-ons'),
)
SORT_CHOICES = (
('score','score'),
('activity','activity'),
('forked','forked'),
('used','used')
)
class SearchForm(CleanForm):
q = forms.CharField(required=False)
page = forms.IntegerField(required=False, initial=1)
@ -17,3 +24,4 @@ class SearchForm(CleanForm):
copies = forms.IntegerField(required=False, initial=0)
used = forms.IntegerField(required=False, initial=0)
activity = forms.IntegerField(required=False, initial=0)
sort = forms.ChoiceField(required=False, choices=SORT_CHOICES)

Просмотреть файл

@ -67,7 +67,7 @@ def get_activity_scale():
ACTIVITY_CACHE_KEY = 'search:activity:average'
def _get_average_activity():
def _get_average_activity():
average = cache.get(ACTIVITY_CACHE_KEY)
if average:
return average

Просмотреть файл

@ -1,4 +1,6 @@
import commonware
import random
import datetime
from django.contrib.auth.models import User
@ -8,10 +10,13 @@ from pyes import StringQuery, FieldQuery, FieldParameter
from elasticutils.tests import ESTestCase
from elasticutils import F
from jetpack.tasks import calculate_activity_rating
from jetpack.models import Package
from jetpack.models import PackageRevision
from search.helpers import package_search
from search.cron import setup_mapping
log = commonware.log.getLogger('f.test.search')
@ -212,5 +217,28 @@ class PackageHelperSearchTest(MappedESTestCase):
and should be the first result.
"""
eq_([p.name for p in data['pager'].object_list], [quux.name, baz.name])
def test_sorting_by_activity(self):
#create 10 packages and a random number of revisions
now = datetime.datetime.utcnow()
addons = []
for i in range(10):
addon = create_addon('addon-{0}'.format(i))
addons.append(addon)
for k in range(1,random.randrange(1,50)):
r = addon.revisions.create(author=addon.author, revision_number=k)
r.created_at=now-datetime.timedelta(random.randrange(1,365))
super(PackageRevision, r).save()
calculate_activity_rating([a.pk for a in addons])
self.es.refresh()
qs = package_search().order_by('activity')
last_score = 1
for r in qs:
eq_(r.activity<last_score, True)
last_score = r.activity

Просмотреть файл

@ -7,9 +7,16 @@ from django.template import RequestContext
from jetpack.models import Package
from .helpers import package_search, get_activity_scale
from .forms import SearchForm
from pyes.urllib3.connectionpool import TimeoutError
log = commonware.log.getLogger('f.search')
SORT_MAPPING = {
'score':'_score',
'activity':'activity',
'forked':'copies_count',
'used':'times_depended',
}
def search(request):
form = SearchForm(request.GET)
@ -21,7 +28,7 @@ def search(request):
page = query.get('page') or 1
limit = 20
activity_map = get_activity_scale()
sort = SORT_MAPPING.get(query.get('sort'), '_score')
filters = {}
filters['user'] = request.user
@ -44,35 +51,56 @@ def search(request):
if query.get('activity'):
filters['activity__gte'] = activity_map.get(str(query['activity']), 0)
results = {}
facets = {}
copies_facet = {'terms': {'field': 'copies_count'}}
times_depended_facet = {'terms': {'field': 'times_depended'}}
facets_ = {'copies': copies_facet, 'times_depended': times_depended_facet}
if type_:
filters['type'] = type_
qs = package_search(q, **filters).facet(**facets_)
try:
results['pager'] = Paginator(qs, per_page=limit).page(page)
except EmptyPage:
results['pager'] = Paginator(qs, per_page=limit).page(1)
facets = _facets(results['pager'].object_list.facets)
facets['everyone_total'] = len(qs)
template = 'results.html'
else:
# combined view
results['addons'] = package_search(q, type='a', **filters).facet(
**facets_)[:5]
results['libraries'] = package_search(q, type='l', **filters).facet(
**facets_)[:5]
facets = _facets(results['addons'].facets)
facets['everyone_total'] = facets['combined_total']
template = 'aggregate.html'
def retry_on_timeout(fn ,args, max_retry=1):
tries = 0;
while True:
try:
tries += 1
return fn(*args)
except TimeoutError as e:
log.error("ES query({3}) timeout: '{0}' - {1} - {2}"
.format(form.cleaned_data,
"retrying" if tries<max_retry else 'forget it',
e, tries
))
if tries==max_retry:
raise e
continue
def execute_search(type_, q, limit, page, filters, facets_):
template = ''
results={}
facets={}
if type_:
filters['type'] = type_
qs = package_search(q, **filters).order_by(sort).facet(**facets_)
try:
results['pager'] = Paginator(qs, per_page=limit).page(page)
except EmptyPage:
results['pager'] = Paginator(qs, per_page=limit).page(1)
facets = _facets(results['pager'].object_list.facets)
facets['everyone_total'] = len(qs)
template = 'results.html'
else:
# combined view
results['addons'] = package_search(q, type='a', **filters) \
.order_by(sort).facet(**facets_)[:5]
results['libraries'] = package_search(q, type='l', **filters) \
.order_by(sort).facet(**facets_)[:5]
facets = _facets(results['addons'].facets)
facets['everyone_total'] = facets['combined_total']
template = 'aggregate.html'
return template,results,facets
template, results, facets = retry_on_timeout(execute_search,
[type_,q,limit,page,filters,facets_] , 2)
ctx = {
'q': q,
'page': 'search',
@ -80,9 +108,10 @@ def search(request):
'query': query,
'type': types.get(type_, None)
}
ctx.update(results)
ctx.update(facets)
if request.is_ajax():
template = 'ajax/' + template
return _render(request, template, ctx)