[mkt-index] Upgraded search dependent libs (bug 867288)

This upgrades pyelasticsearch and elasticutils to their latest versions
and backports some legacy code so code paths don't break due to
backwards incompatible changes. This should result in similar
functionality on the new libraries and allow us to upgrade to newer ways
of doing things incrementally.

apps/amo/models.py

@@ -317,6 +317,17 @@ class SearchMixin(object):
     def search(cls, index=None):
         return search.ES(cls, index or cls._get_index())
 
+    # For compatibility with elasticutils > v0.5.
+    # TODO: Remove these when we've moved mkt to its own index.
+
+    @classmethod
+    def get_index(cls):
+        return cls._get_index()
+
+    @classmethod
+    def get_mapping_type_name(cls):
+        return cls._meta.db_table
+
 
 class ModelBase(SearchMixin, caching.base.CachingMixin, models.Model):
     """

apps/amo/search.py

@@ -1,12 +1,13 @@
 import logging
 from operator import itemgetter
 
+from django.conf import settings as dj_settings
+
 from django_statsd.clients import statsd
+from elasticutils import S as EU_S
 from pyes import ES as pyes_ES
 from pyes import VERSION as PYES_VERSION
 
-from django.conf import settings as dj_settings
-
 
 log = logging.getLogger('z.es')
 
@@ -332,3 +333,34 @@ class ObjectSearchResults(SearchResults):
     def __iter__(self):
         objs = dict((obj.id, obj) for obj in self.objects)
         return (objs[id] for id in self.ids if id in objs)
+
+
+class TempS(EU_S):
+    # Temporary class override to mimic ElasticUtils v0.5 behavior.
+    # TODO: Remove this when we've moved mkt to its own index.
+
+    def _do_search(self):
+        """
+        Perform the search, then convert that raw format into a SearchResults
+        instance and return it.
+        """
+        if not self._results_cache:
+            hits = self.raw()
+            if self.as_list:
+                ResultClass = ListSearchResults
+            elif self.as_dict or self.type is None:
+                ResultClass = DictSearchResults
+            else:
+                ResultClass = ObjectSearchResults
+            self._results_cache = ResultClass(self.type, hits, self.fields)
+        return self._results_cache
+
+    def _build_query(self):
+        query = super(TempS, self)._build_query()
+        if 'fields' in query:
+            if 'id' not in query['fields']:
+                query['fields'].append('id')
+        else:
+            query['fields'] = ['id']
+
+        return query

apps/amo/templates/services/monitor.html

@@ -89,7 +89,7 @@
 <div class="notification-box {{ status(status_summary.elastic) }}">
   <h2>[Elastic Search] Health Check ({{ elastic_timer }}ms)</h2>
   <ul>
-    <li><code>{{ settings.ES_HOSTS|pprint }}</code></li>
+    <li><code>{{ settings.ES_URLS|pprint }}</code></li>
     <li><code>{{ settings.ES_INDEXES|pprint }}</code></li>
     {% if status_summary.elastic %}
       {% for k, v in elastic_results|dictsort %}

apps/amo/tests/__init__.py

@@ -39,7 +39,7 @@ from applications.models import Application, AppVersion
 from bandwagon.models import Collection
 from files.helpers import copyfileobj
 from files.models import File, Platform
-from lib.es.signals import reset, process
+from lib.es.signals import process, reset
 from market.models import AddonPremium, Price, PriceCurrency
 from translations.models import Translation
 from versions.models import ApplicationsVersions, Version
@@ -200,7 +200,7 @@ def stop_es_mock():
     for patch in ES_patchers:
         patch.stop()
 
-    if hasattr(elasticutils._local, 'es'):
+    if hasattr(elasticutils, '_local') and hasattr(elasticutils._local, 'es'):
         delattr(elasticutils._local, 'es')
 
 
@@ -421,7 +421,7 @@ class TestCase(RedisTest, test_utils.TestCase):
         assert 'API-Status' in res['Access-Control-Expose-Headers']
         assert 'API-Version' in res['Access-Control-Expose-Headers']
 
-        verbs = map(str.upper, verbs) + ['OPTIONS', ]
+        verbs = map(str.upper, verbs) + ['OPTIONS']
         actual = res['Access-Control-Allow-Methods'].split(', ')
         self.assertSetEqual(verbs, actual)
         eq_(res['Access-Control-Allow-Headers'],

apps/search/views.py

@@ -1,13 +1,13 @@
 from django import http
 from django.conf import settings
 from django.db.models import Q
+from django.utils import translation
 from django.utils.encoding import smart_str
 from django.views.decorators.vary import vary_on_headers
-from django.utils import translation
 
-from elasticutils.contrib.django import F, S
 import commonware.log
 import jingo
+from elasticutils.contrib.django import F
 from mobility.decorators import mobile_template
 from tower import ugettext as _
 
@@ -17,9 +17,10 @@ import browse.views
 from addons.models import Addon, Category
 from amo.decorators import json_view
 from amo.helpers import locale_url, urlparams
+from amo.search import TempS as S
 from amo.utils import sorted_groupby
 from bandwagon.models import Collection
-from versions.compare import dict_from_int, version_int, version_dict
+from versions.compare import dict_from_int, version_dict, version_int
 
 import mkt
 from mkt.webapps.models import Webapp

lib/es/management/commands/reindex.py

@@ -1,12 +1,12 @@
 import datetime
 import json
 import logging
-from optparse import make_option
 import os
 import re
 import sys
-import traceback
 import time
+import traceback
+from optparse import make_option
 
 import requests
 from celery_tasktree import task_with_callbacks, TaskTree
@@ -15,21 +15,20 @@ from django.conf import settings as django_settings
 from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
 
-from amo.utils import timestamp_index
-
 from addons.cron import reindex_addons, reindex_apps
+from amo.utils import timestamp_index
 from apps.addons.search import setup_mapping as put_amo_mapping
 from bandwagon.cron import reindex_collections
 from compat.cron import compatibility_report
+from lib.es.models import Reindexing
+from lib.es.utils import database_flagged
 from stats.search import setup_indexes as put_stats_mapping
 from users.cron import reindex_users
 
-from lib.es.models import Reindexing
-from lib.es.utils import database_flagged
-
 
 _INDEXES = {}
 
+
 def index_stats(index=None, aliased=True):
     """Indexes the previous 365 days."""
     call_command('index_stats', addons=None)
@@ -55,10 +54,8 @@ logger = logging.getLogger('z.elasticsearch')
 DEFAULT_NUM_REPLICAS = 0
 DEFAULT_NUM_SHARDS = 3
 
-if hasattr(django_settings, 'ES_HOSTS'):
-    base_url = django_settings.ES_HOSTS[0]
-    # The configuration comes with no scheme.
-    base_url = 'http://%s' % base_url
+if hasattr(django_settings, 'ES_URLS'):
+    base_url = django_settings.ES_URLS[0]
 else:
     base_url = 'http://127.0.0.1:9200'
 

lib/settings_base.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 # Django settings for zamboni project.
 
-import os
 import logging
+import os
 import socket
 
 from django.utils.functional import lazy
@@ -1360,6 +1360,7 @@ BUILDER_VERSIONS_URL = ('https://builder.addons.mozilla.org/repackage/' +
 
 ## elasticsearch
 ES_HOSTS = ['127.0.0.1:9200']
+ES_URLS = ['http://%s' % h for h in ES_HOSTS]
 ES_INDEXES = {'default': 'amo',
               'update_counts': 'amo_stats',
               'download_counts': 'amo_stats',

mkt/lookup/tests/test_views.py

@@ -1,6 +1,6 @@
+import json
 from datetime import datetime, timedelta
 from decimal import Decimal
-import json
 
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -196,10 +196,6 @@ class SearchTestMixin(object):
         res = self.client.get(self.url)
         self.assertLoginRedirects(res, self.url)
 
-    def test_no_results(self):
-        data = self.search(q='__garbage__', expect_results=False)
-        eq_(data['results'], [])
-
 
 class TestAcctSearch(ESTestCase, SearchTestMixin):
     fixtures = ['base/users']

mkt/lookup/views.py

@@ -1,10 +1,10 @@
-from datetime import datetime, timedelta
 import hashlib
 import uuid
+from datetime import datetime, timedelta
 
 from django.conf import settings
 from django.db import connection
-from django.db.models import Sum, Count, Q
+from django.db.models import Count, Q, Sum
 from django.http import Http404
 from django.shortcuts import get_object_or_404, redirect
 
@@ -16,14 +16,14 @@ from tower import ugettext as _
 
 import amo
 from addons.models import Addon
-from amo.decorators import (login_required, permission_required, post_required,
-                            json_view)
+from amo.decorators import (json_view, login_required, permission_required,
+                            post_required)
+from amo.search import TempS as S
 from amo.urlresolvers import reverse
 from amo.utils import paginate
 from apps.access import acl
 from apps.bandwagon.models import Collection
 from devhub.models import ActivityLog
-from elasticutils.contrib.django import S
 from lib.pay_server import client
 from market.models import AddonPaymentData, Refund
 from mkt.constants.payments import (COMPLETED, FAILED, PENDING,
@@ -34,7 +34,7 @@ from mkt.lookup.tasks import (email_buyer_refund_approved,
                               email_buyer_refund_pending)
 from mkt.site import messages
 from mkt.webapps.models import Installed
-from stats.models import DownloadCount, Contribution
+from stats.models import Contribution, DownloadCount
 from users.models import UserProfile
 
 log = commonware.log.getLogger('z.lookup')
@@ -335,8 +335,8 @@ def app_search(request):
         qs = (Addon.objects.filter(type=addon_type, guid=q)
                            .values(*non_es_fields))
         if not qs.count():
-            qs = (S(Addon).query(type=addon_type,
-                                 or_=_expand_query(q, fields))
+            qs = (S(Addon).query(must=True, type=addon_type)
+                          .query(should=True, **_expand_query(q, fields))
                           .values_dict(*fields)[:20])
     for app in qs:
         app['url'] = reverse('lookup.app_summary', args=[app['id']])

mkt/webapps/models.py

@@ -16,7 +16,7 @@ from django.dispatch import receiver
 from django.utils.http import urlquote
 
 import commonware.log
-from elasticutils.contrib.django import F, S
+from elasticutils.contrib.django import F
 from tower import ugettext as _
 
 import amo
@@ -28,6 +28,7 @@ from addons.models import (Addon, AddonDeviceType, Category, Flag,
 from addons.signals import version_changed
 from amo.decorators import skip_cache
 from amo.helpers import absolutify
+from amo.search import TempS as S
 from amo.storage_utils import copy_stored_file
 from amo.urlresolvers import reverse
 from amo.utils import JSONEncoder, smart_path
@@ -40,7 +41,7 @@ from versions.models import Version
 
 import mkt
 from mkt import regions
-from mkt.constants import apps, APP_FEATURES, APP_IMAGE_SIZES
+from mkt.constants import APP_FEATURES, APP_IMAGE_SIZES, apps
 from mkt.webapps.utils import get_locale_properties, get_supported_locales
 from mkt.zadmin.models import FeaturedApp
 

requirements/prod.txt

@@ -36,7 +36,6 @@ django-statsd-mozilla==0.3.8.6
 django-storages==1.1.4
 django-tastypie==0.9.11
 easy-thumbnails==1.1
-elasticutils==0.5
 fastchardet==0.2.0
 feedparser==5.0.1
 fudge==1.0.3
@@ -62,7 +61,7 @@ oauth2==1.5.211
 oauthlib==0.4.0
 ordereddict==1.1
 PyBrowserID==0.6
-pyelasticsearch==0.4.1
+pyelasticsearch==0.5
 pyes==0.16.0
 PyJWT-mozilla==0.1.4.2
 PyMySQL==0.5
@@ -77,6 +76,7 @@ receipts==0.2.6
 requests==1.2.0
 schematic==0.2
 signing_clients==0.1.5
+six==1.3.0
 slumber==0.5.3
 SQLAlchemy==0.7.5
 statsd==1.0.0
@@ -91,6 +91,7 @@ tastypie-services==0.2.2
 -e git://github.com/miracle2k/django-tables.git@546f339308103880c823b2056830fcdc9220edd0#egg=django-tables
 -e git://github.com/washort/gelato.models.git@6297036871419651032f7724b6b536c19a228ec6#egg=gelato.models
 -e git://github.com/mozilla/happyforms.git@729612c2a824a7e8283d416d2084bf506c671e24#egg=happyforms
+-e git://github.com/mozilla/elasticutils.git@5bde211443dbe524d940974d477119f51e3ac357#egg=elasticutils
 # Temporary fork.
 -e git://github.com/jsocol/jingo-minify.git@475b6955e741a0a31692e721ceb34a8a4c1b2194#egg=jingo-minify
 -e git://github.com/mozilla/nuggets.git@96e80a64aa4bfcfef4f43fc3ab6966450ccd7325#egg=nuggets

sites/altdev/settings_base.py

@@ -153,6 +153,7 @@ BUILDER_VERSIONS_URL = "https://builder-addons-dev.allizom.org/repackage/sdk-ver
 
 
 ES_HOSTS = splitstrip(private.ES_HOSTS)
+ES_URLS = ['http://%s' % h for h in ES_HOSTS]
 ES_INDEXES = {'default': 'marketplace_altdev',
               'update_counts': 'marketplace_altdev_stats',
               'download_counts': 'marketplace_altdev_stats'}

sites/dev/settings_base.py

@@ -130,6 +130,7 @@ BUILDER_VERSIONS_URL = "https://builder-addons-dev.allizom.org/repackage/sdk-ver
 
 
 ES_HOSTS = splitstrip(private.ES_HOSTS)
+ES_URLS = ['http://%s' % h for h in ES_HOSTS]
 ES_INDEXES = {'default': 'addons_dev',
               'update_counts': 'addons_dev_stats',
               'download_counts': 'addons_dev_stats'}

sites/landfill/settings_base.py

@@ -153,6 +153,7 @@ BUILDER_VERSIONS_URL = "https://builder-addons-dev.allizom.org/repackage/sdk-ver
 
 
 ES_HOSTS = splitstrip(private.ES_HOSTS)
+ES_URLS = ['http://%s' % h for h in ES_HOSTS]
 ES_INDEXES = {'default': 'addons_landfill',
               'update_counts': 'addons_landfill_stats',
               'download_counts': 'addons_landfill_stats'}

sites/prod/settings_base.py

@@ -137,6 +137,7 @@ CRONJOB_LOCK_PREFIX = 'addons'
 BUILDER_SECRET_KEY = private.BUILDER_SECRET_KEY
 
 ES_HOSTS = splitstrip(private.ES_HOSTS)
+ES_URLS = ['http://%s' % h for h in ES_HOSTS]
 ES_INDEXES = {'default': 'addons',
               'update_counts': 'addons_stats',
               'download_counts': 'addons_stats'}

sites/stage/settings_base.py

@@ -148,6 +148,7 @@ BUILDER_VERSIONS_URL = "https://builder-addons.allizom.org/repackage/sdk-version
 
 
 ES_HOSTS = splitstrip(private.ES_HOSTS)
+ES_URLS = ['http://%s' % h for h in ES_HOSTS]
 ES_INDEXES = {'default': 'marketplace_stage',
               'update_counts': 'marketplace_stage_stats',
               'download_counts': 'marketplace_stage_stats'}