Add backend to sign ES HTTP requests

.travis.yml

@@ -14,6 +14,7 @@ env:
   DEBUG=True
   CELERY_TASK_ALWAYS_EAGER=True
   DEV=True
+  ES_CONNECTION=local
 services:
   - elasticsearch
   - mysql

mozillians/common/search.py

@@ -0,0 +1,42 @@
+import boto3
+import os
+
+from botocore.auth import SigV4Auth
+from requests_aws4auth import AWS4Auth
+from elasticsearch import RequestsHttpConnection
+
+
+class AWS4AuthEncoded(AWS4Auth):
+    def __call__(self, request):
+        request = super(AWS4AuthEncoded, self).__call__(request)
+
+        for header_name in request.headers:
+            self._encode_header_to_utf8(request, header_name)
+
+        return request
+
+    def _encode_header_to_utf8(self, request, header_name):
+        value = request.headers[header_name]
+
+        if isinstance(value, unicode):
+            value = value.encode('utf-8')
+
+        if isinstance(header_name, unicode):
+            del request.headers[header_name]
+            header_name = header_name.encode('utf-8')
+
+        request.headers[header_name] = value
+
+
+class AWSRequestsHttpConnection(RequestsHttpConnection):
+    def perform_request(self, *args, **kwargs):
+        credentials = boto3.session.Session().get_credentials()
+        signed_creds = SigV4Auth(credentials, 'es', os.environ['AWS_ES_REGION'])
+
+        secure_auth = AWS4AuthEncoded(
+            credentials.access_key, credentials.secret_key,
+            os.environ['AWS_ES_REGION'], 'es',
+            session_token=signed_creds.credentials.token
+        )
+        self.session.auth = secure_auth
+        return super(AWSRequestsHttpConnection, self).perform_request(*args, **kwargs)

mozillians/env-dist

@@ -40,6 +40,7 @@ BASKET_API_KEY = 'basket_api_key'
 BASKET_MANAGERS = None
 AXES_BEHIND_REVERSE_PROXY = False
 ES_HOST=es
+ES_CONNECTION=local
 
 # Captcha
 NORECAPTCHA_SITE_KEY = "6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI"

mozillians/settings.py

@@ -422,6 +422,14 @@ ES_PROTOCOL = config('ES_PROTOCOL', default='http://')
 
 def _lazy_haystack_setup():
     from django.conf import settings
+    from elasticsearch import RequestsHttpConnection
+    from mozillians.common.search import AWSRequestsHttpConnection
+
+    es_connection = config('ES_CONNECTION', default='aws')
+    es_connection_class = {
+        'aws': AWSRequestsHttpConnection,
+        'local': RequestsHttpConnection
+    }
 
     es_url = '%s%s' % (settings.ES_PROTOCOL, settings.ES_HOST)
     es_index_name = config('ES_INDEX_NAME', default='mozillians_haystack')
@@ -429,17 +437,26 @@ def _lazy_haystack_setup():
         'default': {
             'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
             'URL': es_url,
-            'INDEX_NAME': es_index_name
+            'INDEX_NAME': es_index_name,
+            'KWARGS': {
+                'connection_class': es_connection_class[es_connection]
+            }
         },
         'tmp': {
             'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
             'URL': es_url,
-            'INDEX_NAME': 'tmp_{}'.format(es_index_name)
+            'INDEX_NAME': 'tmp_{}'.format(es_index_name),
+            'KWARGS': {
+                'connection_class': es_connection_class[es_connection]
+            }
         },
         'current': {
             'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
             'URL': es_url,
-            'INDEX_NAME': 'current_{}'.format(es_index_name)
+            'INDEX_NAME': 'current_{}'.format(es_index_name),
+            'KWARGS': {
+                'connection_class': es_connection_class[es_connection]
+            }
         }
     }
 

requirements/prod.txt

@@ -595,3 +595,6 @@ backports.ssl_match_hostname==3.7.0.1 \
     --hash=sha256:bb82e60f9fbf4c080eabd957c39f0641f0fc247d9a16e31e26d594d8f42b9fd2
 mysqlclient==1.4.4 \
     --hash=sha256:9c737cc55a5dc8dd3583a942d5a9b21be58d16f00f5fefca4e575e7d9682e98c
+requests-aws4auth==0.9 \
+    --hash=sha256:c9973af472d6d358ee301f077608361e078642aa019785139b588d526f50a23c \
+    --hash=sha256:e20e4941ccd5706973068f9214d40cb2e669461536b3a57b9ac824ae87744c2c