Merge pull request #1018 from mythmon/log-http-301s

Fix #1013 - Log HTTP to HTTPS redirects, and add cache headers.

docs/ops/config.rst

@@ -194,6 +194,13 @@ in other Django projects.
 
    The time in seconds to set in cache headers for permanent redirects.
 
+.. envvar:: DJANGO_HTTPS_REDIRECT_CACHE_TIME
+
+   :default: ``2592000`` (30 days)
+
+   The time in seconds to set in cache headers for permanent redirects
+   to change from HTTP to HTTPS.
+
 .. envvar:: DJANGO_LOGGING_USE_JSON
 
     :default: ``True``

recipe-server/normandy/base/middleware.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import re
 
@@ -8,6 +9,7 @@ from django.utils.cache import patch_cache_control
 from django.utils.deprecation import MiddlewareMixin
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.middleware.common import CommonMiddleware
+from django.middleware.security import SecurityMiddleware
 
 from mozilla_cloud_services_logger.django.middleware import (
     RequestSummaryLogger as OriginalRequestSummaryLogger
@@ -15,6 +17,12 @@ from mozilla_cloud_services_logger.django.middleware import (
 from whitenoise.middleware import WhiteNoiseMiddleware
 
 
+DEBUG_HTTP_TO_HTTPS_REDIRECT = 'normandy.base.middleware.D001'
+
+
+logger = logging.getLogger(__name__)
+
+
 def request_received_at_middleware(get_response):
     """
     Adds a 'received_at' property to requests with a datetime showing
@@ -91,3 +99,32 @@ class NormandyCommonMiddleware(CommonMiddleware):
     """
 
     response_redirect_class = HttpResponsePermanentRedirectCached
+
+
+class NormandySecurityMiddleware(SecurityMiddleware):
+    """Logs HTTP to HTTPS redirects, and adds cache headers to them."""
+
+    def process_request(self, request):
+        response = super().process_request(request)
+        if response is not None:
+            assert type(response) is http.HttpResponsePermanentRedirect
+            patch_cache_control(response, public=True, max_age=settings.HTTPS_REDIRECT_CACHE_TIME)
+
+            # Pull out just the HTTP headers from the rest of the request meta
+            headers = {
+                key.lstrip('HTTP_'): value
+                for (key, value) in request.META.items()
+                if key.startswith('HTTP_') or key == 'CONTENT_TYPE' or key == 'CONTENT_LENGTH'
+            }
+
+            logger.debug(
+                f'Served HTTP to HTTPS redirect for {request.path}',
+                extra={
+                    'code': DEBUG_HTTP_TO_HTTPS_REDIRECT,
+                    'method': request.method,
+                    'body': request.body.decode('utf-8'),
+                    'path': request.path,
+                    'headers': headers,
+                }
+            )
+        return response

recipe-server/normandy/base/tests/test_middleware.py

@@ -1,6 +1,17 @@
 from random import randint
 
-from normandy.base.middleware import NormandyCommonMiddleware
+import pytest
+
+from normandy.base.middleware import (
+    NormandyCommonMiddleware,
+    NormandySecurityMiddleware,
+    DEBUG_HTTP_TO_HTTPS_REDIRECT,
+)
+
+
+@pytest.fixture
+def mock_logger(mocker):
+    return mocker.patch('normandy.base.middleware.logger')
 
 
 class TestNormandyCommonMiddleware(object):
@@ -21,3 +32,58 @@ class TestNormandyCommonMiddleware(object):
         assert res['Location'] == url + '/'
         cache_control = set(res['Cache-Control'].split(', '))
         assert cache_control == {'public', f'max-age={cache_time}'}
+
+
+class TestNormandySecurityMiddleware(object):
+
+    @pytest.fixture
+    def enable_ssl_redirect(self, settings):
+        settings.SECURE_SSL_REDIRECT = True
+        settings.SECURE_REDIRECT_EXEMPT = []
+
+    def test_it_works(self, rf, enable_ssl_redirect):
+        middleware = NormandySecurityMiddleware()
+        req = rf.get('/', secure=False)
+        res = middleware.process_request(req)
+
+        assert res is not None
+        assert res.status_code == 301
+        assert res['Location'].startswith('https:')
+
+    def test_it_includes_cache_headers(self, rf, enable_ssl_redirect, settings):
+        cache_time = randint(100, 1000)
+        settings.HTTPS_REDIRECT_CACHE_TIME = cache_time
+
+        middleware = NormandySecurityMiddleware()
+        req = rf.get('/', secure=False)
+        res = middleware.process_request(req)
+
+        cache_control = set(res['Cache-Control'].split(', '))
+        assert cache_control == {'public', f'max-age={cache_time}'}
+
+    def test_it_logs(self, rf, enable_ssl_redirect, mock_logger):
+        middleware = NormandySecurityMiddleware()
+        req = rf.post(
+            path='/',
+            data='this is the body',
+            content_type='text/plain',
+            HTTP_X_HELLO='world',
+            secure=False,
+        )
+        middleware.process_request(req)
+
+        mock_logger.debug.assert_called_with(
+            'Served HTTP to HTTPS redirect for /',
+            extra={
+                'code': DEBUG_HTTP_TO_HTTPS_REDIRECT,
+                'method': 'POST',
+                'path': '/',
+                'body': 'this is the body',
+                'headers': {
+                    'X_HELLO': 'world',
+                    'CONTENT_TYPE': 'text/plain',
+                    'CONTENT_LENGTH': 16,
+                    'COOKIE': '',
+                },
+            }
+        )

recipe-server/normandy/settings.py

@@ -39,7 +39,7 @@ class Core(Configuration):
     MIDDLEWARE = [
         'normandy.base.middleware.request_received_at_middleware',
         'normandy.base.middleware.RequestSummaryLogger',
-        'django.middleware.security.SecurityMiddleware',
+        'normandy.base.middleware.NormandySecurityMiddleware',
         'normandy.base.middleware.NormandyWhiteNoiseMiddleware',
         'normandy.base.middleware.NormandyCommonMiddleware',
         'django.middleware.clickjacking.XFrameOptionsMiddleware',
@@ -174,6 +174,9 @@ class Base(Core):
     DEBUG = values.BooleanValue(False)
     ADMINS = values.SingleNestedListValue([])
     SILENCED_SYSTEM_CHECKS = values.ListValue([
+        # We've subclassed Django's security middleware, so Django's
+        # checks can't tell we are using the middleware.
+        'security.W001',
         # Check CSRF cookie http only. disabled because we read the
         # CSRF cookie in JS for forms in React.
         'security.W017',
@@ -330,6 +333,7 @@ class Base(Core):
     API_CACHE_TIME = values.IntegerValue(30)
     API_CACHE_ENABLED = values.BooleanValue(True)
     PERMANENT_REDIRECT_CACHE_TIME = values.IntegerValue(60 * 60 * 24 * 30)
+    HTTPS_REDIRECT_CACHE_TIME = values.IntegerValue(60 * 60 * 24 * 30)
 
     # If true, approvals must come from two separate users. If false, the same
     # user can approve their own request.
@@ -397,6 +401,7 @@ class ProductionReadOnly(Production):
     ]
     ADMIN_ENABLED = values.BooleanValue(False)
     SILENCED_SYSTEM_CHECKS = values.ListValue([
+        'security.W001',  # Security middle ware check
         'security.W003',  # CSRF middleware check
         'security.W017',  # Check CSRF cookie http only
     ])
@@ -419,6 +424,7 @@ class ProductionInsecure(Production):
 
     # These checks aren't useful for a purposefully insecure environment
     SILENCED_SYSTEM_CHECKS = values.ListValue([
+        'security.W001',  # security middleware check
         'security.W004',  # check hsts seconds
         'security.W008',  # Secure SSL redirect
         'security.W009',  # Secret key length