Release for 20191205

services/fxa/hubhandler.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
 import os
 import sys
 import newrelic.agent
@@ -11,7 +15,7 @@ serverless_wsgi.TEXT_MIME_TYPES.append("application/custom+json")
 
 # First some funky path manipulation so that we can work properly in
 # the AWS environment
-sys.path.insert(0, join(dirname(realpath(__file__)), 'src'))
+sys.path.insert(0, join(dirname(realpath(__file__)), "src"))
 
 newrelic.agent.initialize()
 
@@ -37,9 +41,12 @@ XRayMiddleware(hub_app.app, xray_recorder)
 @newrelic.agent.lambda_handler()
 def handle(event, context):
     try:
-        logger.info("handling hub event", subhub_event=event, context=context)
         return serverless_wsgi.handle_request(hub_app.app, event, context)
     except Exception as e:  # pylint: disable=broad-except
-        logger.exception("exception occurred", subhub_event=event, context=context, error=e)
+        logger.exception(
+            "exception occurred", subhub_event=event, context=context, error=e
+        )
         # TODO: Add Sentry exception catch here
         raise
+    finally:
+        logger.info("handling hub event", subhub_event=event, context=context)

services/fxa/miahandler.py

@@ -1,5 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
 import os
 import sys
 import newrelic.agent
@@ -10,7 +12,7 @@ from os.path import join, dirname, realpath
 
 # First some funky path manipulation so that we can work properly in
 # the AWS environment
-sys.path.insert(0, join(dirname(realpath(__file__)), 'src'))
+sys.path.insert(0, join(dirname(realpath(__file__)), "src"))
 
 newrelic.agent.initialize()
 
@@ -33,11 +35,13 @@ patch_all()
 @newrelic.agent.lambda_handler()
 def handle_mia(event, context):
     try:
-        logger.info("handling mia event", subhub_event=event, context=context)
-        processing_duration=int(os.getenv('PROCESS_EVENTS_HOURS', '6'))
+        processing_duration = int(os.getenv("PROCESS_EVENTS_HOURS", "6"))
         events_check.process_events(processing_duration)
     except Exception as e:  # pylint: disable=broad-except
-        logger.exception("exception occurred", subhub_event=event, context=context, error=e)
+        logger.exception(
+            "exception occurred", subhub_event=event, context=context, error=e
+        )
         # TODO: Add Sentry exception catch here
         raise
-
+    finally:
+        logger.info("handling mia event", subhub_event=event, context=context)

services/fxa/subhandler.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
 import os
 import sys
 import newrelic.agent
@@ -11,7 +15,7 @@ serverless_wsgi.TEXT_MIME_TYPES.append("application/custom+json")
 
 # First some funky path manipulation so that we can work properly in
 # the AWS environment
-sys.path.insert(0, join(dirname(realpath(__file__)), 'src'))
+sys.path.insert(0, join(dirname(realpath(__file__)), "src"))
 
 newrelic.agent.initialize()
 
@@ -37,9 +41,12 @@ XRayMiddleware(sub_app.app, xray_recorder)
 @newrelic.agent.lambda_handler()
 def handle(event, context):
     try:
-        logger.info("handling sub event", subhub_event=event, context=context)
         return serverless_wsgi.handle_request(sub_app.app, event, context)
     except Exception as e:  # pylint: disable=broad-except
-        logger.exception("exception occurred", subhub_event=event, context=context, error=e)
+        logger.exception(
+            "exception occurred", subhub_event=event, context=context, error=e
+        )
         # TODO: Add Sentry exception catch here
         raise
+    finally:
+        logger.info("handling sub event", sub_event=event, context=context)

src/hub/vendor/controller.py

@@ -24,7 +24,7 @@ logger = get_logger()
 
 class StripeHubEventPipeline:
     def __init__(self, payload) -> None:
-        assert isinstance(payload, dict)
+        assert isinstance(payload, dict)  # nosec
         self.payload: dict = payload
 
     def run(self) -> None:
@@ -47,7 +47,7 @@ class StripeHubEventPipeline:
             pass
 
 
-def view() -> tuple:
+def view() -> Response:
     try:
         payload = request.data
         logger.info("check payload", payload=payload)

src/shared/authentication.py

@@ -10,8 +10,6 @@ logger = get_logger()
 
 
 def test_token(test_api_token, cfg_api_token):
-    logger.info(f"test api token {test_api_token}")
-
     # Make sure the config API token has a meaningful value set,
     # to avoid an auth bypass on empty comparisons
     if cfg_api_token in (None, "None", ""):

src/shared/headers.py

@@ -9,7 +9,6 @@
 # be logged into the application's produced logs.
 HEADERS_WHITE_LIST = ["Content-Length", "Content-Type", "Host", "X-Amzn-Trace-Id"]
 
-
 # `dump_headers` is a method to dump from headers from the `requests` library's
 # headers and compare against a known list of safe headers for utilization in
 # items such as logging and metrics.  It is an O(n) algorithm so as the amount

src/shared/log.py

@@ -8,7 +8,10 @@ from shared.cfg import CFG
 from typing import Any
 
 LOGGER = None
-
+CENSORED_EVENT_VALUES_BY_EVENT_KEY = {
+    "headers": ["Authorization", "X-Forwarded-For"],
+    "multiValueHeaders": ["Authorization"],
+}
 
 dict_config = {
     "version": 1,
@@ -47,6 +50,26 @@ def event_uppercase(logger, method_name, event_dict):
     return event_dict
 
 
+def censor_event_dict(event_dict):
+    for k, v in event_dict.items():
+        if isinstance(v, dict):
+            censor_event_dict(v)
+        else:
+            for event_key, event_values in CENSORED_EVENT_VALUES_BY_EVENT_KEY.items():
+                if event_dict is not None:
+                    _event_key = event_dict.get(event_key)
+                    if _event_key:
+                        for event_value in event_values:
+                            _event_value = _event_key.get(event_value)
+                            if _event_key:
+                                event_dict[event_key][event_value] = "*CENSORED*"
+            return event_dict
+
+
+def censor_header(logger, method_name, event_dict):
+    return censor_event_dict(event_dict)
+
+
 def get_logger() -> Any:
     global LOGGER
     if not LOGGER:
@@ -66,6 +89,8 @@ def get_logger() -> Any:
                 stdlib.add_logger_name,
                 # Uppercase structlog's event name which shouldn't be convoluted with AWS events.
                 event_uppercase,
+                # Censor secure data
+                censor_header,
                 # Allow for string interpolation
                 stdlib.PositionalArgumentsFormatter(),
                 # Render timestamps to ISO 8601