From f367df8baa9deba3c361620f87c98eb88c2e234c Mon Sep 17 00:00:00 2001
From: Jonathan Claudius <jclaudius@mozilla.com>
Date: Tue, 28 May 2019 15:17:05 -0400
Subject: [PATCH 01/58] Add prototype LDAP password spray alert

---
 alerts/ldap_password_spray.py | 47 +++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 alerts/ldap_password_spray.py

diff --git a/alerts/ldap_password_spray.py b/alerts/ldap_password_spray.py
new file mode 100644
index 00000000..6e061908
--- /dev/null
+++ b/alerts/ldap_password_spray.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+
+from lib.alerttask import AlertTask
+from mozdef_util.query_models import SearchQuery, TermMatch
+
+
+class AlertLdapPasswordSpray(AlertTask):
+    def main(self):
+        # TODO: evaluate what the threshold should be to detect a password spray
+        search_query = SearchQuery(minutes=60)
+
+        search_query.add_must([
+            TermMatch('category', 'ldap'),
+            TermMatch('tags', 'ldap'),
+            TermMatch('details.response.error', 'LDAP_INVALID_CREDENTIALS')
+        ])
+
+        self.filtersManual(search_query)
+        self.searchEventsAggregated('details.client', samplesLimit=10)
+
+        # TODO: evaluate what the threshold should be for this alert
+        self.walkAggregations(threshold=1)
+
+    # Set alert properties
+    def onAggregation(self, aggreg):
+        category = 'ldap'
+        tags = ['ldap']
+        severity = 'WARNING'
+
+        user_dn_list = set()
+
+        for event in aggreg['allevents']:
+            for request in event['_source']['details']['requests']:
+                user_dn_list.add(request['details'][0])
+
+        summary = 'Possible Password Spray Attack in Progress from {0} using the following distinguished names: {1}'.format(
+            aggreg['value'],
+            ",".join(sorted(user_dn_list))
+        )
+
+        return self.createAlertDict(summary, category, tags, aggreg['events'], severity)

From ce4b7b938862c36789db511670af85acd55bbb7e Mon Sep 17 00:00:00 2001
From: Jonathan Claudius <jclaudius@mozilla.com>
Date: Thu, 6 Jun 2019 08:08:54 -0400
Subject: [PATCH 02/58] Move config items to config file

---
 alerts/ldap_password_spray.conf |  3 +++
 alerts/ldap_password_spray.py   | 41 ++++++++++++++++++++++++---------
 2 files changed, 33 insertions(+), 11 deletions(-)
 create mode 100644 alerts/ldap_password_spray.conf

diff --git a/alerts/ldap_password_spray.conf b/alerts/ldap_password_spray.conf
new file mode 100644
index 00000000..aca4ae85
--- /dev/null
+++ b/alerts/ldap_password_spray.conf
@@ -0,0 +1,3 @@
+[options]
+threshold_count = 1
+search_depth_min = 60
\ No newline at end of file
diff --git a/alerts/ldap_password_spray.py b/alerts/ldap_password_spray.py
index 6e061908..ce92409a 100644
--- a/alerts/ldap_password_spray.py
+++ b/alerts/ldap_password_spray.py
@@ -8,24 +8,21 @@
 
 from lib.alerttask import AlertTask
 from mozdef_util.query_models import SearchQuery, TermMatch
+import re
 
 
 class AlertLdapPasswordSpray(AlertTask):
     def main(self):
-        # TODO: evaluate what the threshold should be to detect a password spray
-        search_query = SearchQuery(minutes=60)
-
+        self.parse_config('ldap_password_spray.conf', ['threshold_count', 'search_depth_min'])
+        search_query = SearchQuery(minutes=int(self.config.search_depth_min))
         search_query.add_must([
             TermMatch('category', 'ldap'),
             TermMatch('tags', 'ldap'),
             TermMatch('details.response.error', 'LDAP_INVALID_CREDENTIALS')
         ])
-
         self.filtersManual(search_query)
         self.searchEventsAggregated('details.client', samplesLimit=10)
-
-        # TODO: evaluate what the threshold should be for this alert
-        self.walkAggregations(threshold=1)
+        self.walkAggregations(threshold=int(self.config.threshold_count))
 
     # Set alert properties
     def onAggregation(self, aggreg):
@@ -33,15 +30,37 @@ class AlertLdapPasswordSpray(AlertTask):
         tags = ['ldap']
         severity = 'WARNING'
 
-        user_dn_list = set()
+        email_list = set()
 
         for event in aggreg['allevents']:
             for request in event['_source']['details']['requests']:
-                user_dn_list.add(request['details'][0])
+                email = extractEmail(request['details'][0])
+                if email:
+                    email_list.add(extractEmail(request['details'][0]))
 
-        summary = 'Possible Password Spray Attack in Progress from {0} using the following distinguished names: {1}'.format(
+        # If no emails, don't throw alert
+        if len(email_list) == 0:
+            return None
+
+        summary = 'Password Spray Attack in Progress from {0} targeting the following accounts: {1}'.format(
             aggreg['value'],
-            ",".join(sorted(user_dn_list))
+            ",".join(sorted(email_list))
         )
 
         return self.createAlertDict(summary, category, tags, aggreg['events'], severity)
+
+    # A helper function to extract the first email in a string.
+    #
+    # Example:
+    #
+    # Input: 'dn="mail=user@example.com,o=com,dc=example"'
+    # Ouput: user@example.com
+    #
+    def extractEmail(string):
+        email_regex = r'mail=([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)'
+        match_object = re.match(email_regex, string)
+
+        if match_object:
+            return match_object.group(1)
+        else:
+            return None

From 3e206649089a9a5f46e3fef86675250a67ce6661 Mon Sep 17 00:00:00 2001
From: Jonathan Claudius <jclaudius@mozilla.com>
Date: Thu, 6 Jun 2019 08:30:45 -0400
Subject: [PATCH 03/58] Add unit-test stub for LDAP password spray alert

---
 alerts/ldap_password_spray.py            |  3 +-
 tests/alerts/test_ldap_password_spray.py | 98 ++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 2 deletions(-)
 create mode 100644 tests/alerts/test_ldap_password_spray.py

diff --git a/alerts/ldap_password_spray.py b/alerts/ldap_password_spray.py
index ce92409a..017b280b 100644
--- a/alerts/ldap_password_spray.py
+++ b/alerts/ldap_password_spray.py
@@ -17,7 +17,6 @@ class AlertLdapPasswordSpray(AlertTask):
         search_query = SearchQuery(minutes=int(self.config.search_depth_min))
         search_query.add_must([
             TermMatch('category', 'ldap'),
-            TermMatch('tags', 'ldap'),
             TermMatch('details.response.error', 'LDAP_INVALID_CREDENTIALS')
         ])
         self.filtersManual(search_query)
@@ -42,7 +41,7 @@ class AlertLdapPasswordSpray(AlertTask):
         if len(email_list) == 0:
             return None
 
-        summary = 'Password Spray Attack in Progress from {0} targeting the following accounts: {1}'.format(
+        summary = 'Password Spray Attack in Progress from {0} targeting the following account(s): {1}'.format(
             aggreg['value'],
             ",".join(sorted(email_list))
         )
diff --git a/tests/alerts/test_ldap_password_spray.py b/tests/alerts/test_ldap_password_spray.py
new file mode 100644
index 00000000..bc695082
--- /dev/null
+++ b/tests/alerts/test_ldap_password_spray.py
@@ -0,0 +1,98 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+from positive_alert_test_case import PositiveAlertTestCase
+from negative_alert_test_case import NegativeAlertTestCase
+from alert_test_suite import AlertTestSuite
+
+
+class TestAlertLdapPasswordSpray(AlertTestSuite):
+    alert_filename = "ldap_password_spray"
+    # This event is the default positive event that will cause the
+    # alert to trigger
+    default_event = {
+        "_source": {
+            "category": "ldap",
+            "details": {
+                "client": "1.2.3.4",
+                "requests": [
+                    {
+                        'verb': 'BIND',
+                        'details': [
+                            'dn="mail=jsmith@example.com,o=com,dc=example"',
+                            'method=128'
+                        ]
+                    }
+                ],
+                "response": {
+                    "error": 'LDAP_INVALID_CREDENTIALS',
+                }
+            }
+        }
+    }
+
+    # This alert is the expected result from running this task
+    default_alert = {
+        "category": "ldap",
+        "tags": ["ldap"],
+        "severity": "WARNING",
+        "summary": "Password Spray Attack in Progress from 1.2.3.4 targeting the following account(s): jsmith@example.com",
+    }
+
+    # This alert is the expected result from this task against multiple matching events
+    default_alert_aggregated = AlertTestSuite.copy(default_alert)
+    default_alert_aggregated[
+        "summary"
+    ] = "Password Spray Attack in Progress from 1.2.3.4 targeting the following account(s): jsmith@example.com"
+
+    test_cases = []
+
+    test_cases.append(
+        PositiveAlertTestCase(
+            description="Positive test with default events and default alert expected",
+            events=AlertTestSuite.create_events(default_event, 1),
+            expected_alert=default_alert,
+        )
+    )
+
+    test_cases.append(
+        PositiveAlertTestCase(
+            description="Positive test with default events and default alert expected - dedup",
+            events=AlertTestSuite.create_events(default_event, 2),
+            expected_alert=default_alert,
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 10)
+    for event in events:
+        event["_source"]["details"]["response"]["error"] = "LDAP_SUCCESS"
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test with default negative event", events=events
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 10)
+    for event in events:
+        event["_source"]["category"] = "bad"
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with incorrect category",
+            events=events,
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 10)
+    for event in events:
+        event["_source"][
+            "utctimestamp"
+        ] = AlertTestSuite.subtract_from_timestamp_lambda({"minutes": 241})
+        event["_source"][
+            "receivedtimestamp"
+        ] = AlertTestSuite.subtract_from_timestamp_lambda({"minutes": 241})
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with old timestamp", events=events
+        )
+    )

From 5022e63c1e887dbb2e0b0f8c41b9151d4dc681b4 Mon Sep 17 00:00:00 2001
From: Jonathan Claudius <jclaudius@mozilla.com>
Date: Thu, 6 Jun 2019 08:41:55 -0400
Subject: [PATCH 04/58] Minor tweaks to eliminate errors in unit-test runs

---
 alerts/ldap_password_spray.py | 29 ++++++-----------------------
 1 file changed, 6 insertions(+), 23 deletions(-)

diff --git a/alerts/ldap_password_spray.py b/alerts/ldap_password_spray.py
index 017b280b..ff835ab4 100644
--- a/alerts/ldap_password_spray.py
+++ b/alerts/ldap_password_spray.py
@@ -23,23 +23,22 @@ class AlertLdapPasswordSpray(AlertTask):
         self.searchEventsAggregated('details.client', samplesLimit=10)
         self.walkAggregations(threshold=int(self.config.threshold_count))
 
-    # Set alert properties
     def onAggregation(self, aggreg):
         category = 'ldap'
         tags = ['ldap']
         severity = 'WARNING'
-
         email_list = set()
+        email_regex = r'mail=([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)'
 
         for event in aggreg['allevents']:
             for request in event['_source']['details']['requests']:
-                email = extractEmail(request['details'][0])
-                if email:
-                    email_list.add(extractEmail(request['details'][0]))
+                match_object = re.match(email_regex, request['details'][0])
+                if match_object:
+                    email_list.add(match_object.group(1))
 
         # If no emails, don't throw alert
-        if len(email_list) == 0:
-            return None
+        # if len(email_list) == 0:
+        #     return None
 
         summary = 'Password Spray Attack in Progress from {0} targeting the following account(s): {1}'.format(
             aggreg['value'],
@@ -47,19 +46,3 @@ class AlertLdapPasswordSpray(AlertTask):
         )
 
         return self.createAlertDict(summary, category, tags, aggreg['events'], severity)
-
-    # A helper function to extract the first email in a string.
-    #
-    # Example:
-    #
-    # Input: 'dn="mail=user@example.com,o=com,dc=example"'
-    # Ouput: user@example.com
-    #
-    def extractEmail(string):
-        email_regex = r'mail=([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)'
-        match_object = re.match(email_regex, string)
-
-        if match_object:
-            return match_object.group(1)
-        else:
-            return None

From c89f5ea39847f6a29588162a4bf4a4e3be1d60b2 Mon Sep 17 00:00:00 2001
From: Jonathan Claudius <jclaudius@mozilla.com>
Date: Thu, 6 Jun 2019 08:55:15 -0400
Subject: [PATCH 05/58] Get unit-tests passing

---
 alerts/ldap_password_spray.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/alerts/ldap_password_spray.py b/alerts/ldap_password_spray.py
index ff835ab4..1a1f22af 100644
--- a/alerts/ldap_password_spray.py
+++ b/alerts/ldap_password_spray.py
@@ -28,7 +28,7 @@ class AlertLdapPasswordSpray(AlertTask):
         tags = ['ldap']
         severity = 'WARNING'
         email_list = set()
-        email_regex = r'mail=([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)'
+        email_regex = r'.*mail=([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)'
 
         for event in aggreg['allevents']:
             for request in event['_source']['details']['requests']:

From cf6ab524911254db55396fdb1310d1abdf96e477 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Mon, 8 Jul 2019 14:05:00 -0500
Subject: [PATCH 06/58] Fixup imports from python3 upgrade

---
 tests/alerts/test_ldap_password_spray.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/alerts/test_ldap_password_spray.py b/tests/alerts/test_ldap_password_spray.py
index bc695082..8f780cbc 100644
--- a/tests/alerts/test_ldap_password_spray.py
+++ b/tests/alerts/test_ldap_password_spray.py
@@ -2,9 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 # Copyright (c) 2017 Mozilla Corporation
-from positive_alert_test_case import PositiveAlertTestCase
-from negative_alert_test_case import NegativeAlertTestCase
-from alert_test_suite import AlertTestSuite
+from .positive_alert_test_case import PositiveAlertTestCase
+from .negative_alert_test_case import NegativeAlertTestCase
+
+from .alert_test_suite import AlertTestSuite
 
 
 class TestAlertLdapPasswordSpray(AlertTestSuite):

From 2b0ba950b3035c9224c714869f081d8b88b21b58 Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Wed, 10 Jul 2019 13:20:30 -0700
Subject: [PATCH 07/58] Clarify summary placeholder text

and remove trailing quote
Fixes #1307
---
 meteor/client/incidentAdd.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meteor/client/incidentAdd.html b/meteor/client/incidentAdd.html
index d47bd1ea..b5cb9862 100644
--- a/meteor/client/incidentAdd.html
+++ b/meteor/client/incidentAdd.html
@@ -13,7 +13,7 @@ Copyright (c) 2014 Mozilla Corporation
     <div class="form-group">
       <label class="col-xs-3 control-label" for="summary">Incident Summary:</label>
       <div class="col-xs-5">
-        <input id="summary" name="summary" placeholder="'Category: 'activity' on 'affected target''"
+        <input id="summary" name="summary" placeholder="e.g. Data Disclosure on Foo Product"
           class="form-control summary" required="" type="text">
       </div>
     </div>
@@ -50,4 +50,4 @@ Copyright (c) 2014 Mozilla Corporation
       </div>
     </div>
   </form>
-</template>
\ No newline at end of file
+</template>

From ef922aee902fa20a71e2dd11bd41e96d128d3a5c Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Fri, 12 Jul 2019 13:48:39 -0500
Subject: [PATCH 08/58] Updating veris html to remove data-target and css to
 ensure text visibility

---
 meteor/client/verisTags.html                  |  2 +-
 meteor/imports/themes/classic/mozdef.css      | 75 ++++++++++++++-----
 meteor/imports/themes/dark/mozdef.css         | 63 +++++++++++++---
 .../imports/themes/side_nav_dark/mozdef.css   | 73 +++++++++++++-----
 4 files changed, 165 insertions(+), 48 deletions(-)

diff --git a/meteor/client/verisTags.html b/meteor/client/verisTags.html
index 9438ae4a..955f8793 100644
--- a/meteor/client/verisTags.html
+++ b/meteor/client/verisTags.html
@@ -13,7 +13,7 @@ Copyright (c) 2014 Mozilla Corporation
                     placeholder="tag filter">
             </div>
             <div class="dropdown">
-                <a class="dropdown-toggle" data-toggle="dropdown" id="dLabel" data-target="#"><span
+                <a class="dropdown-toggle" data-toggle="dropdown" id="dLabel"><span
                         class="label label-info">common</span><b class="caret"></b></a>
                 <ul class="dropdown-menu" role="menu" aria-labelledby="dLabel">
                     <li>category</li>
diff --git a/meteor/imports/themes/classic/mozdef.css b/meteor/imports/themes/classic/mozdef.css
index 8becb163..8e68ff40 100644
--- a/meteor/imports/themes/classic/mozdef.css
+++ b/meteor/imports/themes/classic/mozdef.css
@@ -66,7 +66,6 @@ caption, legend {
   color:var(--txt-primary-color);
   font-style: normal;
   font-weight: normal;
-
 }
 
 .headercontainer {
@@ -121,7 +120,6 @@ caption, legend {
   line-height: 13px;
 }
 
-
 .attackshoverboard {
     /*width: 500px;*/
     /*height: 500px;*/
@@ -133,17 +131,53 @@ caption, legend {
     display: none;
 }
 
-.dropdown-submenu{position:relative;}
-.dropdown-submenu>.dropdown-menu{top:0;left:100%;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px;}
-.dropdown-submenu:active>.dropdown-menu, .dropdown-submenu:hover>.dropdown-menu {
+.dropdown-submenu {
+    position:relative;
+}
+
+.dropdown-submenu>.dropdown-menu {
+    top:0;
+    left:100%;
+    -webkit-border-radius:0 6px 6px 6px;
+    -moz-border-radius:0 6px 6px 6px;
+    border-radius:0 6px 6px 6px;
+}
+
+.dropdown-submenu:active>.dropdown-menu,
+.dropdown-submenu:hover>.dropdown-menu {
        display: block;
        right:162px;
 }
-.dropdown-submenu>a:after{display:block;content:" ";float:right;width:0;height:0;border-color:transparent;border-style:solid;border-width:5px 0 5px 5px;border-left-color:#cccccc;margin-top:5px;margin-right:-10px;}
-.dropdown-submenu:active>a:after{border-left-color:#ffffff;}
-.dropdown-submenu.pull-left{float:none;}.dropdown-submenu.pull-left>.dropdown-menu{left:-100%;margin-left:10px;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px;}
 
+.dropdown-submenu>a:after {
+    display:block;
+    content:" ";
+    float:right;
+    width:0;
+    height:0;
+    border-color:transparent;
+    border-style:solid;
+    border-width:5px 0 5px 5px;
+    border-left-color:#cccccc;
+    margin-top:5px;
+    margin-right:-10px;
+}
 
+.dropdown-submenu:active>a:after {
+    border-left-color:#ffffff;
+}
+
+.dropdown-submenu.pull-left {
+    float:none;
+}
+
+.dropdown-submenu.pull-left>.dropdown-menu {
+    left:-100%;
+    margin-left:10px;
+    -webkit-border-radius:6px 0 6px 6px;
+    -moz-border-radius:6px 0 6px 6px;
+    border-radius:6px 0 6px 6px;
+}
 
 .attackercallout {
     width: 120px;
@@ -172,6 +206,7 @@ caption, legend {
     text-transform: uppercase;
     margin-top: 20px;
 }
+
 .attackercallout ul{
     list-style: none;
     float: left;
@@ -183,6 +218,7 @@ caption, legend {
 .attackercallout .indicator{
     color: yellow;
 }
+
 .attackercallout a{
     color: yellow;
 }
@@ -199,23 +235,27 @@ caption, legend {
 .alert.alert-NOTICE {
     --alert-bg-color: #4a6785;
     --alert-txt-color: white;
-  }
+}
+
 .alert.alert-WARNING {
     --alert-bg-color: #ffd351;
     --alert-txt-color: black;
-  }
+}
+
 .alert.alert-CRITICAL {
     --alert-bg-color: #d04437;
     --alert-txt-color: white;
-  }
+}
+
 .alert.alert-INFO {
     --alert-bg-color: #cccccc;
     --alert-txt-color: black;
-  }
+}
+
 .alert.alert-ERROR {
     --alert-bg-color: #d04437;
     --alert-txt-color: white;
-  }
+}
 
 .alert  {
     color: var(--alert-txt-color);
@@ -223,7 +263,7 @@ caption, legend {
     text-transform: uppercase;
     display: table-cell;
     font-weight: bold;
-  }
+}
 
 .alert-row a{
     color: var(--a-link-color);
@@ -267,7 +307,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
     border-radius: 4px;
     color: var(--txt-primary-color);
     background-color: var(--arm-color);
-  }
+}
   
   .btn-warning.active,
   .btn-warning:active,
@@ -276,7 +316,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
     color: var(--txt-secondary-color);
     background-color: var(--arm-focus-color);
     border-color: var(--arm-color);
-  }
+}
 
   .btnAlertAcked,
   .btnAlertAcked.active,
@@ -285,8 +325,7 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
     color: var(--txt-disabled-color);
     background-color: var(--arm-focus-color);
     border-color: var(--arm-color);
-  }
-
+}
 
 input[type="search"] {
   border-radius: 15px;
diff --git a/meteor/imports/themes/dark/mozdef.css b/meteor/imports/themes/dark/mozdef.css
index cee6db3c..e779c864 100644
--- a/meteor/imports/themes/dark/mozdef.css
+++ b/meteor/imports/themes/dark/mozdef.css
@@ -112,6 +112,57 @@ caption, legend {
   line-height: 13px;
 }
 
+.dropdown-menu li {
+    color: #000;
+}
+
+.dropdown-submenu {
+    position:relative;
+}
+
+.dropdown-submenu>.dropdown-menu {
+    top:0;
+    left:100%;
+    -webkit-border-radius:0 6px 6px 6px;
+    -moz-border-radius:0 6px 6px 6px;
+    border-radius:0 6px 6px 6px;
+}
+
+.dropdown-submenu:active>.dropdown-menu,
+.dropdown-submenu:hover>.dropdown-menu {
+       display: block;
+       right:162px;
+}
+
+.dropdown-submenu>a:after {
+    display:block;
+    content:" ";
+    float:right;
+    width:0;
+    height:0;
+    border-color:transparent;
+    border-style:solid;
+    border-width:5px 0 5px 5px;
+    border-left-color:#cccccc;
+    margin-top:5px;
+    margin-right:-10px;
+}
+
+.dropdown-submenu:active>a:after {
+    border-left-color:#ffffff;
+}
+
+.dropdown-submenu.pull-left {
+    float:none;
+}
+
+.dropdown-submenu.pull-left>.dropdown-menu {
+    left:-100%;
+    margin-left:10px;
+    -webkit-border-radius:6px 0 6px 6px;
+    -moz-border-radius:6px 0 6px 6px;
+    border-radius:6px 0 6px 6px;
+}
 
 .attackshoverboard {
     /*width: 500px;*/
@@ -124,18 +175,6 @@ caption, legend {
     display: none;
 }
 
-.dropdown-submenu{position:relative;}
-.dropdown-submenu>.dropdown-menu{top:0;left:100%;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px;}
-.dropdown-submenu:active>.dropdown-menu, .dropdown-submenu:hover>.dropdown-menu {
-       display: block;
-       right:162px;
-}
-.dropdown-submenu>a:after{display:block;content:" ";float:right;width:0;height:0;border-color:transparent;border-style:solid;border-width:5px 0 5px 5px;border-left-color:#cccccc;margin-top:5px;margin-right:-10px;}
-.dropdown-submenu:active>a:after{border-left-color:#ffffff;}
-.dropdown-submenu.pull-left{float:none;}.dropdown-submenu.pull-left>.dropdown-menu{left:-100%;margin-left:10px;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px;}
-
-
-
 .attackercallout {
     width: 120px;
     height: 160px;
diff --git a/meteor/imports/themes/side_nav_dark/mozdef.css b/meteor/imports/themes/side_nav_dark/mozdef.css
index c498f6bb..28c03d50 100644
--- a/meteor/imports/themes/side_nav_dark/mozdef.css
+++ b/meteor/imports/themes/side_nav_dark/mozdef.css
@@ -115,29 +115,66 @@ caption, legend {
   line-height: 13px;
 }
 
-
-.attackshoverboard {
-    /*width: 500px;*/
-    /*height: 500px;*/
-    /*background-color: green;*/
-    -moz-transform: scaleY(-1);
-    -webkit-transform: scaleY(-1);
-    -o-transform: scaleY(-1);
-    transform: scaleY(-1);
-    display: none;
+.dropdown-menu li {
+  color: #000;
 }
 
-.dropdown-submenu{position:relative;}
-.dropdown-submenu>.dropdown-menu{top:0;left:100%;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px;}
-.dropdown-submenu:active>.dropdown-menu, .dropdown-submenu:hover>.dropdown-menu {
+.dropdown-submenu {
+  position:relative;
+}
+
+.dropdown-submenu>.dropdown-menu {
+  top:0;left:100%;
+  -webkit-border-radius:0 6px 6px 6px;
+  -moz-border-radius:0 6px 6px 6px;
+  border-radius:0 6px 6px 6px;
+}
+
+.dropdown-submenu:active>.dropdown-menu,
+.dropdown-submenu:hover>.dropdown-menu {
        display: block;
        right:162px;
 }
-.dropdown-submenu>a:after{display:block;content:" ";float:right;width:0;height:0;border-color:transparent;border-style:solid;border-width:5px 0 5px 5px;border-left-color:#cccccc;margin-top:5px;margin-right:-10px;}
-.dropdown-submenu:active>a:after{border-left-color:#ffffff;}
-.dropdown-submenu.pull-left{float:none;}.dropdown-submenu.pull-left>.dropdown-menu{left:-100%;margin-left:10px;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px;}
 
+.dropdown-submenu>a:after {
+  display:block;
+  content:" ";
+  float:right;
+  width:0;
+  height:0;
+  border-color:transparent;
+  border-style:solid;
+  border-width:5px 0 5px 5px;
+  border-left-color:#cccccc;
+  margin-top:5px;
+  margin-right:-10px;
+}
 
+.dropdown-submenu:active>a:after {
+  border-left-color:#ffffff;
+}
+
+.dropdown-submenu.pull-left {
+  float:none;
+}
+.dropdown-submenu.pull-left>.dropdown-menu {
+  left:-100%;
+  margin-left:10px;
+  -webkit-border-radius:6px 0 6px 6px;
+  -moz-border-radius:6px 0 6px 6px;
+  border-radius:6px 0 6px 6px;
+}
+
+.attackshoverboard {
+  /*width: 500px;*/
+  /*height: 500px;*/
+  /*background-color: green;*/
+  -moz-transform: scaleY(-1);
+  -webkit-transform: scaleY(-1);
+  -o-transform: scaleY(-1);
+  transform: scaleY(-1);
+  display: none;
+}
 
 .attackercallout {
     width: 120px;
@@ -170,7 +207,8 @@ caption, legend {
     text-transform: uppercase;
     margin-top: 20px;
 }
-.attackercallout ul{
+
+.attackercallout ul {
     list-style: none;
     float: left;
     left: auto;
@@ -181,6 +219,7 @@ caption, legend {
 .attackercallout .indicator{
     color: yellow;
 }
+
 .attackercallout a{
     color: yellow;
 }

From 10bf9b708547960a8e1f21142722c0f46f67e2d2 Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Fri, 12 Jul 2019 14:04:12 -0500
Subject: [PATCH 09/58] formatting css.

---
 meteor/imports/themes/classic/mozdef.css      | 64 +++++++--------
 meteor/imports/themes/dark/mozdef.css         | 60 +++++++-------
 .../imports/themes/side_nav_dark/mozdef.css   | 81 ++++++++++---------
 3 files changed, 103 insertions(+), 102 deletions(-)

diff --git a/meteor/imports/themes/classic/mozdef.css b/meteor/imports/themes/classic/mozdef.css
index 8e68ff40..8cec9396 100644
--- a/meteor/imports/themes/classic/mozdef.css
+++ b/meteor/imports/themes/classic/mozdef.css
@@ -132,51 +132,51 @@ caption, legend {
 }
 
 .dropdown-submenu {
-    position:relative;
+    position: relative;
 }
 
-.dropdown-submenu>.dropdown-menu {
-    top:0;
-    left:100%;
-    -webkit-border-radius:0 6px 6px 6px;
-    -moz-border-radius:0 6px 6px 6px;
-    border-radius:0 6px 6px 6px;
+.dropdown-submenu > .dropdown-menu {
+    top: 0;
+    left: 100%;
+    -webkit-border-radius: 0 6px 6px 6px;
+    -moz-border-radius: 0 6px 6px 6px;
+    border-radius: 0 6px 6px 6px;
 }
 
-.dropdown-submenu:active>.dropdown-menu,
-.dropdown-submenu:hover>.dropdown-menu {
+.dropdown-submenu:active > .dropdown-menu,
+.dropdown-submenu:hover > .dropdown-menu {
        display: block;
-       right:162px;
+       right: 162px;
 }
 
-.dropdown-submenu>a:after {
-    display:block;
-    content:" ";
-    float:right;
-    width:0;
-    height:0;
-    border-color:transparent;
-    border-style:solid;
-    border-width:5px 0 5px 5px;
-    border-left-color:#cccccc;
-    margin-top:5px;
-    margin-right:-10px;
+.dropdown-submenu > a:after {
+    display: block;
+    content: " ";
+    float: right;
+    width: 0;
+    height: 0;
+    border-color: transparent;
+    border-style: solid;
+    border-width: 5px 0 5px 5px;
+    border-left-color: #cccccc;
+    margin-top: 5px;
+    margin-right: -10px;
 }
 
-.dropdown-submenu:active>a:after {
-    border-left-color:#ffffff;
+.dropdown-submenu:active > a:after {
+    border-left-color: #ffffff;
 }
 
 .dropdown-submenu.pull-left {
-    float:none;
+    float: none;
 }
 
-.dropdown-submenu.pull-left>.dropdown-menu {
-    left:-100%;
-    margin-left:10px;
-    -webkit-border-radius:6px 0 6px 6px;
-    -moz-border-radius:6px 0 6px 6px;
-    border-radius:6px 0 6px 6px;
+.dropdown-submenu.pull-left > .dropdown-menu {
+    left: -100%;
+    margin-left: 10px;
+    -webkit-border-radius: 6px 0 6px 6px;
+    -moz-border-radius: 6px 0 6px 6px;
+    border-radius: 6px 0 6px 6px;
 }
 
 .attackercallout {
@@ -685,7 +685,7 @@ sidenav {
     border-width: 0px 1px;
     z-index: 100;
     position:relative;
-    float:left;
+    float: left;
     border-image: none;
     text-decoration: none;
 }
diff --git a/meteor/imports/themes/dark/mozdef.css b/meteor/imports/themes/dark/mozdef.css
index e779c864..7b169cba 100644
--- a/meteor/imports/themes/dark/mozdef.css
+++ b/meteor/imports/themes/dark/mozdef.css
@@ -120,48 +120,48 @@ caption, legend {
     position:relative;
 }
 
-.dropdown-submenu>.dropdown-menu {
-    top:0;
-    left:100%;
-    -webkit-border-radius:0 6px 6px 6px;
-    -moz-border-radius:0 6px 6px 6px;
-    border-radius:0 6px 6px 6px;
+.dropdown-submenu > .dropdown-menu {
+    top: 0;
+    left: 100%;
+    -webkit-border-radius: 0 6px 6px 6px;
+    -moz-border-radius: 0 6px 6px 6px;
+    border-radius: 0 6px 6px 6px;
 }
 
-.dropdown-submenu:active>.dropdown-menu,
-.dropdown-submenu:hover>.dropdown-menu {
+.dropdown-submenu:active > .dropdown-menu,
+.dropdown-submenu:hover > .dropdown-menu {
        display: block;
-       right:162px;
+       right: 162px;
 }
 
-.dropdown-submenu>a:after {
-    display:block;
-    content:" ";
-    float:right;
-    width:0;
-    height:0;
-    border-color:transparent;
-    border-style:solid;
-    border-width:5px 0 5px 5px;
-    border-left-color:#cccccc;
-    margin-top:5px;
-    margin-right:-10px;
+.dropdown-submenu > a:after {
+    display: block;
+    content: " ";
+    float: right;
+    width: 0;
+    height: 0;
+    border-color: transparent;
+    border-style: solid;
+    border-width: 5px 0 5px 5px;
+    border-left-color: #cccccc;
+    margin-top: 5px;
+    margin-right: -10px;
 }
 
-.dropdown-submenu:active>a:after {
-    border-left-color:#ffffff;
+.dropdown-submenu:active > a:after {
+    border-left-color: #ffffff;
 }
 
 .dropdown-submenu.pull-left {
-    float:none;
+    float: none;
 }
 
-.dropdown-submenu.pull-left>.dropdown-menu {
-    left:-100%;
-    margin-left:10px;
-    -webkit-border-radius:6px 0 6px 6px;
-    -moz-border-radius:6px 0 6px 6px;
-    border-radius:6px 0 6px 6px;
+.dropdown-submenu.pull-left > .dropdown-menu {
+    left: -100%;
+    margin-left: 10px;
+    -webkit-border-radius: 6px 0 6px 6px;
+    -moz-border-radius: 6px 0 6px 6px;
+    border-radius: 6px 0 6px 6px;
 }
 
 .attackshoverboard {
diff --git a/meteor/imports/themes/side_nav_dark/mozdef.css b/meteor/imports/themes/side_nav_dark/mozdef.css
index 28c03d50..fde59084 100644
--- a/meteor/imports/themes/side_nav_dark/mozdef.css
+++ b/meteor/imports/themes/side_nav_dark/mozdef.css
@@ -120,49 +120,50 @@ caption, legend {
 }
 
 .dropdown-submenu {
-  position:relative;
+  position: relative;
 }
 
-.dropdown-submenu>.dropdown-menu {
-  top:0;left:100%;
-  -webkit-border-radius:0 6px 6px 6px;
-  -moz-border-radius:0 6px 6px 6px;
-  border-radius:0 6px 6px 6px;
+.dropdown-submenu > .dropdown-menu {
+  top: 0;
+  left: 100%;
+  -webkit-border-radius: 0 6px 6px 6px;
+  -moz-border-radius: 0 6px 6px 6px;
+  border-radius: 0 6px 6px 6px;
 }
 
-.dropdown-submenu:active>.dropdown-menu,
-.dropdown-submenu:hover>.dropdown-menu {
+.dropdown-submenu:active > .dropdown-menu,
+.dropdown-submenu:hover > .dropdown-menu {
        display: block;
-       right:162px;
+       right: 162px;
 }
 
-.dropdown-submenu>a:after {
-  display:block;
-  content:" ";
-  float:right;
-  width:0;
-  height:0;
-  border-color:transparent;
-  border-style:solid;
-  border-width:5px 0 5px 5px;
-  border-left-color:#cccccc;
-  margin-top:5px;
-  margin-right:-10px;
+.dropdown-submenu > a:after {
+  display: block;
+  content: " ";
+  float: right;
+  width: 0;
+  height: 0;
+  border-color: transparent;
+  border-style: solid;
+  border-width: 5px 0 5px 5px;
+  border-left-color: #cccccc;
+  margin-top: 5px;
+  margin-right: -10px;
 }
 
-.dropdown-submenu:active>a:after {
-  border-left-color:#ffffff;
+.dropdown-submenu:active > a:after {
+  border-left-color: #ffffff;
 }
 
 .dropdown-submenu.pull-left {
-  float:none;
+  float: none;
 }
-.dropdown-submenu.pull-left>.dropdown-menu {
-  left:-100%;
-  margin-left:10px;
-  -webkit-border-radius:6px 0 6px 6px;
-  -moz-border-radius:6px 0 6px 6px;
-  border-radius:6px 0 6px 6px;
+.dropdown-submenu.pull-left > .dropdown-menu {
+  left: -100%;
+  margin-left: 10px;
+  -webkit-border-radius: 6px 0 6px 6px;
+  -moz-border-radius: 6px 0 6px 6px;
+  border-radius: 6px 0 6px 6px;
 }
 
 .attackshoverboard {
@@ -754,7 +755,7 @@ nav.main-menu.expanded {
   z-index: 1000;
 }
 
-.main-menu>ul {
+.main-menu > ul {
   margin: 7px 0;
 }
 
@@ -764,13 +765,13 @@ nav.main-menu.expanded {
   width: 225px;
 }
 
-.main-menu li:hover>a,
-nav.main-menu li.active>a,
-.dropdown-menu>li>a:hover,
-.dropdown-menu>li>a:focus,
-.dropdown-menu>.active>a,
-.dropdown-menu>.active>a:hover,
-.dropdown-menu>.active>a:focus,
+.main-menu li:hover > a,
+nav.main-menu li.active > a,
+.dropdown-menu > li > a:hover,
+.dropdown-menu > li > a:focus,
+.dropdown-menu >.active > a,
+.dropdown-menu >.active > a:hover,
+.dropdown-menu >.active > a:focus,
 .no-touch .dashboard-page nav.dashboard-menu ul li:hover a,
 .dashboard-page nav.dashboard-menu ul li.active a {
   color: rgb(0, 0, 0);
@@ -876,7 +877,7 @@ nav.main-menu li.active>a,
   left: 200px;
 }
 
-.main-menu li>a {
+.main-menu li > a {
   position: relative;
   display: table;
   border-collapse: collapse;
@@ -908,7 +909,7 @@ nav.main-menu li.active>a,
   font-family: 'Zilla Slab', serif;
 }
 
-.main-menu>ul.logout {
+.main-menu > ul.logout {
   position: absolute;
   left: 0;
   bottom: 0;

From adc8f48e680604d453b4b91de1b554875426b1ea Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Fri, 12 Jul 2019 14:23:32 -0500
Subject: [PATCH 10/58] adding pointer and highlight to veris dropdown

---
 meteor/imports/themes/dark/mozdef.css          | 5 +++++
 meteor/imports/themes/side_nav_dark/mozdef.css | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/meteor/imports/themes/dark/mozdef.css b/meteor/imports/themes/dark/mozdef.css
index 7b169cba..bbc5e961 100644
--- a/meteor/imports/themes/dark/mozdef.css
+++ b/meteor/imports/themes/dark/mozdef.css
@@ -116,6 +116,11 @@ caption, legend {
     color: #000;
 }
 
+.dropdown-menu li:hover {
+    background: #ccc;
+    cursor: pointer;
+}
+
 .dropdown-submenu {
     position:relative;
 }
diff --git a/meteor/imports/themes/side_nav_dark/mozdef.css b/meteor/imports/themes/side_nav_dark/mozdef.css
index fde59084..52aa45ad 100644
--- a/meteor/imports/themes/side_nav_dark/mozdef.css
+++ b/meteor/imports/themes/side_nav_dark/mozdef.css
@@ -119,6 +119,11 @@ caption, legend {
   color: #000;
 }
 
+.dropdown-menu li:hover {
+  background: #ccc;
+  cursor: pointer;
+}
+
 .dropdown-submenu {
   position: relative;
 }

From 16fae7bc39a8a7ea62f7f37fded46fed9eb560a3 Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Fri, 12 Jul 2019 15:31:07 -0500
Subject: [PATCH 11/58] adding contrast and pointers to veris menu in all
 themes.

---
 meteor/imports/themes/classic/mozdef.css       | 10 ++++++++++
 meteor/imports/themes/dark/mozdef.css          | 15 ++++++++++-----
 meteor/imports/themes/light/mozdef.css         | 10 ++++++++++
 meteor/imports/themes/side_nav_dark/mozdef.css |  5 +++++
 4 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/meteor/imports/themes/classic/mozdef.css b/meteor/imports/themes/classic/mozdef.css
index 8cec9396..42015c90 100644
--- a/meteor/imports/themes/classic/mozdef.css
+++ b/meteor/imports/themes/classic/mozdef.css
@@ -131,6 +131,16 @@ caption, legend {
     display: none;
 }
 
+.dropdown-menu li:hover {
+    background: rgb(163, 163, 163);
+    cursor: pointer;
+}
+
+.tag:hover {
+    background: #5bc0de;
+    cursor: pointer;
+}
+
 .dropdown-submenu {
     position: relative;
 }
diff --git a/meteor/imports/themes/dark/mozdef.css b/meteor/imports/themes/dark/mozdef.css
index bbc5e961..64376c2a 100644
--- a/meteor/imports/themes/dark/mozdef.css
+++ b/meteor/imports/themes/dark/mozdef.css
@@ -112,14 +112,19 @@ caption, legend {
   line-height: 13px;
 }
 
-.dropdown-menu li {
+.tag:hover {
+    background: #5bc0de;
+    cursor: pointer;
+  }
+  
+  .dropdown-menu li {
     color: #000;
-}
-
-.dropdown-menu li:hover {
+  }
+  
+  .dropdown-menu li:hover {
     background: #ccc;
     cursor: pointer;
-}
+  }
 
 .dropdown-submenu {
     position:relative;
diff --git a/meteor/imports/themes/light/mozdef.css b/meteor/imports/themes/light/mozdef.css
index d352ce24..3270cbd1 100644
--- a/meteor/imports/themes/light/mozdef.css
+++ b/meteor/imports/themes/light/mozdef.css
@@ -129,6 +129,16 @@ caption, legend {
     display: none;
 }
 
+.dropdown-menu li:hover {
+    background: rgb(163, 163, 163);
+    cursor: pointer;
+}
+
+.tag:hover {
+    background: #5bc0de;
+    cursor: pointer;
+}
+
 .dropdown-submenu{position:relative;}
 .dropdown-submenu>.dropdown-menu{top:0;left:100%;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px;}
 .dropdown-submenu:active>.dropdown-menu, .dropdown-submenu:hover>.dropdown-menu {
diff --git a/meteor/imports/themes/side_nav_dark/mozdef.css b/meteor/imports/themes/side_nav_dark/mozdef.css
index 52aa45ad..8757dbab 100644
--- a/meteor/imports/themes/side_nav_dark/mozdef.css
+++ b/meteor/imports/themes/side_nav_dark/mozdef.css
@@ -115,6 +115,11 @@ caption, legend {
   line-height: 13px;
 }
 
+.tag:hover {
+  background: #5bc0de;
+  cursor: pointer;
+}
+
 .dropdown-menu li {
   color: #000;
 }

From d43dba7407793ca834f6f60a34ff028dfcddff9e Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Fri, 12 Jul 2019 19:19:13 -0500
Subject: [PATCH 12/58] Clarifying form control instructions for tags

---
 meteor/client/investigationEdit.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meteor/client/investigationEdit.html b/meteor/client/investigationEdit.html
index 21f683c5..162203f0 100644
--- a/meteor/client/investigationEdit.html
+++ b/meteor/client/investigationEdit.html
@@ -112,7 +112,7 @@ Copyright (c) 2014 Mozilla Corporation
                                     title="Drag and Drop the veris tags from the upper right to the area below"></i>
                             </label>
                             <div class="tags col-xs-10">
-                                <div class="form-control">drag here to add a tag</div>
+                                <div class="form-control">To add a tag: drag tags from the tag filter menu to here</div>
                                 <ul class="pull-left list-unstyled">
                                     {{#each tags}}
                                     <li class="list-unstyled pull-left">

From 9e142101144da8ef1a91e4e73e8ce2714552738f Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Fri, 12 Jul 2019 19:20:21 -0500
Subject: [PATCH 13/58] Clarifying form control instructions for tags

---
 meteor/client/investigationEdit.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meteor/client/investigationEdit.html b/meteor/client/investigationEdit.html
index 162203f0..d6f49c92 100644
--- a/meteor/client/investigationEdit.html
+++ b/meteor/client/investigationEdit.html
@@ -112,7 +112,7 @@ Copyright (c) 2014 Mozilla Corporation
                                     title="Drag and Drop the veris tags from the upper right to the area below"></i>
                             </label>
                             <div class="tags col-xs-10">
-                                <div class="form-control">To add a tag: drag tags from the tag filter menu to here</div>
+                                <div class="form-control">To add tags: drag tags from the tag filter menu to here</div>
                                 <ul class="pull-left list-unstyled">
                                     {{#each tags}}
                                     <li class="list-unstyled pull-left">

From b994f9baa4d735508728d886ed4db85d4e7bc0cb Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Mon, 15 Jul 2019 10:10:43 -0500
Subject: [PATCH 14/58] Update configlib version

---
 docker/compose/mozdef_base/Dockerfile | 4 ----
 requirements.txt                      | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/docker/compose/mozdef_base/Dockerfile b/docker/compose/mozdef_base/Dockerfile
index 624150ac..55b56761 100644
--- a/docker/compose/mozdef_base/Dockerfile
+++ b/docker/compose/mozdef_base/Dockerfile
@@ -62,7 +62,3 @@ VOLUME /opt/mozdef/envs/mozdef/data
 ENV PATH=/opt/mozdef/envs/python/bin:$PATH
 
 USER root
-
-# Remove once https://github.com/jeffbryner/configlib/pull/9 is mergeg
-# and a new version of configlib is in place
-RUN sed -i 's/from configlib import getConfig/from .configlib import getConfig/g' /opt/mozdef/envs/python/lib/python3.6/site-packages/configlib/__init__.py
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 6e19f847..6d328299 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ bottle==0.12.4
 celery==4.1.0
 celery[sqs]==4.1.0
 cffi==1.9.1
-configlib==2.0.3
+configlib==2.0.4
 configparser==3.5.0b2
 cryptography==2.3.1
 dnspython==1.15.0

From e83e2af603b92c0d509c8acf5626771751e12f99 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Tue, 16 Jul 2019 10:02:07 -0500
Subject: [PATCH 15/58] Add config option to suppress slack bot welcome message

---
 bot/slack/mozdefbot.py | 5 ++++-
 bot/slack/slack_bot.py | 6 ++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/bot/slack/mozdefbot.py b/bot/slack/mozdefbot.py
index b442d557..c75b7561 100644
--- a/bot/slack/mozdefbot.py
+++ b/bot/slack/mozdefbot.py
@@ -160,6 +160,9 @@ def init_config():
     # mqack=True sets persistant delivery, False sets transient delivery
     options.mq_ack = get_config('mqack', True, options.configfile)
 
+    # wether or not the bot should send a welcome message upon connecting
+    options.notify_welcome = get_config('notify_welcome', True, options.configfile)
+
 
 if __name__ == "__main__":
     parser = OptionParser()
@@ -170,7 +173,7 @@ if __name__ == "__main__":
     (options, args) = parser.parse_args()
     init_config()
 
-    bot = SlackBot(options.slack_token, options.channels, options.name)
+    bot = SlackBot(options.slack_token, options.channels, options.name, options.notify_welcome)
     monitor_alerts_thread = Thread(target=consume_alerts, args=[bot])
     monitor_alerts_thread.daemon = True
     monitor_alerts_thread.start()
diff --git a/bot/slack/slack_bot.py b/bot/slack/slack_bot.py
index ed9cc539..76df8ad7 100644
--- a/bot/slack/slack_bot.py
+++ b/bot/slack/slack_bot.py
@@ -20,10 +20,11 @@ greetings = [
 
 
 class SlackBot():
-    def __init__(self, api_key, channels, bot_name):
+    def __init__(self, api_key, channels, bot_name, notify_welcome):
         self.slack_client = SlackClient(api_key)
         self.channels = channels
         self.bot_name = bot_name
+        self.notify_welcome = notify_welcome
         self.load_commands()
 
     def load_commands(self):
@@ -36,7 +37,8 @@ class SlackBot():
     def run(self):
         if self.slack_client.rtm_connect():
             logger.info("Bot connected to slack")
-            self.post_welcome_message(random.choice(greetings))
+            if self.notify_welcome:
+                self.post_welcome_message(random.choice(greetings))
             self.listen_for_messages()
         else:
             logger.error("Unable to connect to slack")

From a2931d49bd27f1e90b77354ff7276edd5f2c573a Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Tue, 16 Jul 2019 10:01:34 -0700
Subject: [PATCH 16/58] Fix Makefile typo

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index fe8c1870..8059c289 100644
--- a/Makefile
+++ b/Makefile
@@ -53,7 +53,7 @@ run-tests-resources:  ## Just run the external resources required for tests
 .PHONY: run-test
 run-test: run-tests
 
-.PHONY: run-test
+.PHONY: run-tests
 run-tests: run-tests-resources  ## Just run the tests (no build/get). Use `make TEST_CASE=tests/...` for specific tests only
 	docker run -it --rm mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && flake8 --config .flake8 ./"
 	docker run -it --rm --network=test-mozdef_default mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && py.test --delete_indexes --delete_queues $(TEST_CASE)"

From 5a3561936bec5b129a00b5a84d42eb7573d06fec Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Tue, 16 Jul 2019 10:01:42 -0700
Subject: [PATCH 17/58] Improve Dockerfiles

* Change pattern of a final recursive chown to instead intentionally setting
  the owner and group in the COPY commands. This should avoid a layer that
  touches all previously copied files
* Add checks for yum repo signing key fingerprint
* Expand some command line arguments to long form so it's easily
  understandably by the reader without having to lookup the man page
* Add removal of yum cache to reduce docker image size
* Change kibana WORKDIR to a fixed and simple value of /kibana
* Cleanup long lines to be more readable
* Use symbolic links instead of internal file copies to reduce docker image size
* Standardize on indents of two spaces
---
 docker/compose/elasticsearch/Dockerfile       | 12 ++-
 docker/compose/kibana/Dockerfile              |  9 ++-
 docker/compose/mongodb/Dockerfile             | 10 ++-
 docker/compose/mozdef_alertactions/Dockerfile |  7 +-
 docker/compose/mozdef_alerts/Dockerfile       |  8 +-
 docker/compose/mozdef_base/Dockerfile         | 57 +++++++-------
 docker/compose/mozdef_bootstrap/Dockerfile    | 16 ++--
 docker/compose/mozdef_bot/Dockerfile          |  6 +-
 .../compose/mozdef_cognito_proxy/Dockerfile   |  2 +-
 docker/compose/mozdef_cron/Dockerfile         | 19 +++--
 docker/compose/mozdef_loginput/Dockerfile     |  6 +-
 docker/compose/mozdef_meteor/Dockerfile       | 76 ++++++++++---------
 docker/compose/mozdef_mq_worker/Dockerfile    |  6 +-
 docker/compose/mozdef_rest/Dockerfile         |  6 +-
 docker/compose/mozdef_sampledata/Dockerfile   |  6 +-
 docker/compose/mozdef_syslog/Dockerfile       |  3 +-
 docker/compose/rabbitmq/Dockerfile            |  9 ++-
 17 files changed, 134 insertions(+), 124 deletions(-)

diff --git a/docker/compose/elasticsearch/Dockerfile b/docker/compose/elasticsearch/Dockerfile
index bd5b759d..986d151e 100644
--- a/docker/compose/elasticsearch/Dockerfile
+++ b/docker/compose/elasticsearch/Dockerfile
@@ -7,11 +7,17 @@ ENV ES_JAVA_VERSION 1.8.0
 
 
 RUN \
+  gpg="gpg --no-default-keyring --secret-keyring /dev/null --keyring /dev/null --no-option --keyid-format 0xlong" && \
+  rpmkeys --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \
+  rpm -qi gpg-pubkey-f4a80eb5 | $gpg | grep 0x24C6A8A7F4A80EB5 && \
+  rpmkeys --import https://packages.elastic.co/GPG-KEY-elasticsearch && \
+  rpm -qi gpg-pubkey-d88e42b4-52371eca | $gpg | grep 0xD27D666CD88E42B4 && \
   yum install -y java-$ES_JAVA_VERSION && \
   mkdir -p /opt/mozdef/envs && \
-  curl -s -L https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.rpm -o elasticsearch.rpm && \
-  rpm -i elasticsearch.rpm && \
-  yum clean all
+  curl --silent --location https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.rpm -o elasticsearch.rpm && \
+  rpm --install elasticsearch.rpm && \
+  yum clean all && \
+  rm -rf /var/cache/yum
 
 USER elasticsearch
 
diff --git a/docker/compose/kibana/Dockerfile b/docker/compose/kibana/Dockerfile
index 6ea2b6c4..238c3ab2 100644
--- a/docker/compose/kibana/Dockerfile
+++ b/docker/compose/kibana/Dockerfile
@@ -6,11 +6,12 @@ LABEL maintainer="mozdef@mozilla.com"
 ENV KIBANA_VERSION 6.8.0
 
 RUN \
-    curl -s -L https://artifacts.elastic.co/downloads/kibana/kibana-$KIBANA_VERSION-linux-x86_64.tar.gz | tar -C / -xz && \
-    cd /kibana-$KIBANA_VERSION-linux-x86_64
+  mkdir /kibana && \
+  curl --silent --location https://artifacts.elastic.co/downloads/kibana/kibana-$KIBANA_VERSION-linux-x86_64.tar.gz \
+    | tar --extract --gzip --strip 1 --directory /kibana
 
-COPY docker/compose/kibana/files/kibana.yml /kibana-$KIBANA_VERSION-linux-x86_64/config/kibana.yml
+COPY docker/compose/kibana/files/kibana.yml /kibana/config/kibana.yml
 
-WORKDIR /kibana-$KIBANA_VERSION-linux-x86_64
+WORKDIR /kibana
 
 EXPOSE 5601
diff --git a/docker/compose/mongodb/Dockerfile b/docker/compose/mongodb/Dockerfile
index 18815e6e..61a036cf 100644
--- a/docker/compose/mongodb/Dockerfile
+++ b/docker/compose/mongodb/Dockerfile
@@ -5,14 +5,20 @@ LABEL maintainer="mozdef@mozilla.com"
 ENV MONGO_VERSION 3.4
 
 RUN \
-  echo -e "[mongodb-org-$MONGO_VERSION]\nname=MongoDB Repository\nbaseurl=https://repo.mongodb.org/yum/redhat/\$releasever/mongodb-org/$MONGO_VERSION/x86_64/\ngpgcheck=1\nenabled=1\ngpgkey=https://www.mongodb.org/static/pgp/server-$MONGO_VERSION.asc" > /etc/yum.repos.d/mongodb.repo && \
+  echo -e "[mongodb-org-$MONGO_VERSION]\n\
+name=MongoDB Repository\n\
+baseurl=https://repo.mongodb.org/yum/redhat/\$releasever/mongodb-org/$MONGO_VERSION/x86_64/\n\
+gpgcheck=1\n\
+enabled=1\n\
+gpgkey=https://www.mongodb.org/static/pgp/server-$MONGO_VERSION.asc" > /etc/yum.repos.d/mongodb.repo && \
   gpg="gpg --no-default-keyring --secret-keyring /dev/null --keyring /dev/null --no-option --keyid-format 0xlong" && \
   rpmkeys --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \
   rpm -qi gpg-pubkey-f4a80eb5 | $gpg | grep 0x24C6A8A7F4A80EB5 && \
   rpmkeys --import https://www.mongodb.org/static/pgp/server-3.4.asc && \
   rpm -qi gpg-pubkey-a15703c6 | $gpg | grep 0xBC711F9BA15703C6 && \
   yum install -y mongodb-org && \
-  yum clean all
+  yum clean all && \
+  rm -rf /var/cache/yum
 
 COPY docker/compose/mongodb/files/mongod.conf /etc/mongod.conf
 
diff --git a/docker/compose/mozdef_alertactions/Dockerfile b/docker/compose/mozdef_alertactions/Dockerfile
index 72734cb5..b46644a5 100644
--- a/docker/compose/mozdef_alertactions/Dockerfile
+++ b/docker/compose/mozdef_alertactions/Dockerfile
@@ -2,10 +2,9 @@ FROM mozdef/mozdef_base
 
 LABEL maintainer="mozdef@mozilla.com"
 
-COPY alerts /opt/mozdef/envs/mozdef/alerts
-COPY docker/compose/mozdef_alertactions/files/alert_actions_worker.conf /opt/mozdef/envs/mozdef/alerts/alert_actions_worker.conf
-COPY docker/compose/mozdef_alerts/files/config.py /opt/mozdef/envs/mozdef/alerts/lib/config.py
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/alerts
+COPY --chown=mozdef:mozdef alerts /opt/mozdef/envs/mozdef/alerts
+COPY --chown=mozdef:mozdef docker/compose/mozdef_alertactions/files/alert_actions_worker.conf /opt/mozdef/envs/mozdef/alerts/alert_actions_worker.conf
+COPY --chown=mozdef:mozdef docker/compose/mozdef_alerts/files/config.py /opt/mozdef/envs/mozdef/alerts/lib/config.py
 
 WORKDIR /opt/mozdef/envs/mozdef/alerts
 
diff --git a/docker/compose/mozdef_alerts/Dockerfile b/docker/compose/mozdef_alerts/Dockerfile
index 5adc5723..95ee41c3 100644
--- a/docker/compose/mozdef_alerts/Dockerfile
+++ b/docker/compose/mozdef_alerts/Dockerfile
@@ -2,11 +2,9 @@ FROM mozdef/mozdef_base
 
 LABEL maintainer="mozdef@mozilla.com"
 
-COPY alerts /opt/mozdef/envs/mozdef/alerts
-COPY docker/compose/mozdef_alerts/files/config.py /opt/mozdef/envs/mozdef/alerts/lib/
-COPY docker/compose/mozdef_alerts/files/get_watchlist.conf /opt/mozdef/envs/mozdef/alerts/get_watchlist.conf
-
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/alerts
+COPY --chown=mozdef:mozdef alerts /opt/mozdef/envs/mozdef/alerts
+COPY --chown=mozdef:mozdef docker/compose/mozdef_alerts/files/config.py /opt/mozdef/envs/mozdef/alerts/lib/
+COPY --chown=mozdef:mozdef docker/compose/mozdef_alerts/files/get_watchlist.conf /opt/mozdef/envs/mozdef/alerts/get_watchlist.conf
 
 WORKDIR /opt/mozdef/envs/mozdef/alerts
 
diff --git a/docker/compose/mozdef_base/Dockerfile b/docker/compose/mozdef_base/Dockerfile
index 55b56761..430d33f6 100644
--- a/docker/compose/mozdef_base/Dockerfile
+++ b/docker/compose/mozdef_base/Dockerfile
@@ -5,54 +5,55 @@ LABEL maintainer="mozdef@mozilla.com"
 ENV TZ UTC
 
 RUN \
+  gpg="gpg --no-default-keyring --secret-keyring /dev/null --keyring /dev/null --no-option --keyid-format 0xlong" && \
+  rpmkeys --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \
+  rpm -qi gpg-pubkey-f4a80eb5 | $gpg | grep 0x24C6A8A7F4A80EB5 && \
+  rpmkeys --import https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 && \
+  rpm -qi gpg-pubkey-352c64e5 | $gpg | grep 0x6A2FAEA2352C64E5 && \
   yum makecache fast && \
   yum install -y epel-release && \
   yum install -y \
-                  glibc-devel \
-                  gcc \
-                  libstdc++ \
-                  libffi-devel \
-                  zlib-devel \
-                  libcurl-devel \
-                  openssl \
-                  openssl-devel \
-                  git \
-                  make && \
-  useradd -ms /bin/bash -d /opt/mozdef -m mozdef && \
-  mkdir /opt/mozdef/envs && \
-  cd /opt/mozdef && \
-  yum install -y python36 \
-                    python36-devel \
-                    python36-pip && \
+     glibc-devel \
+     gcc \
+     libstdc++ \
+     libffi-devel \
+     zlib-devel \
+     libcurl-devel \
+     openssl \
+     openssl-devel \
+     git \
+     make \
+     python36 \
+     python36-devel \
+     python36-pip && \
   yum clean all && \
+  rm -rf /var/cache/yum && \
+  useradd --create-home --shell /bin/bash --home-dir /opt/mozdef mozdef && \
   pip3 install virtualenv && \
-  mkdir /opt/mozdef/envs/mozdef && \
-  mkdir /opt/mozdef/envs/mozdef/cron
+  install --owner mozdef --group mozdef --directory /opt/mozdef/envs /opt/mozdef/envs/mozdef /opt/mozdef/envs/mozdef/cron
 
 # Force pycurl to understand we prefer nss backend
 # Pycurl with ssl support is required by kombu in order to use SQS
 ENV PYCURL_SSL_LIBRARY=nss
 
 # Create python virtual environment and install dependencies
-COPY requirements.txt /opt/mozdef/envs/mozdef/requirements.txt
+COPY --chown=mozdef:mozdef requirements.txt /opt/mozdef/envs/mozdef/requirements.txt
 
-COPY cron/update_geolite_db.py /opt/mozdef/envs/mozdef/cron/update_geolite_db.py
-COPY cron/update_geolite_db.conf /opt/mozdef/envs/mozdef/cron/update_geolite_db.conf
-COPY cron/update_geolite_db.sh /opt/mozdef/envs/mozdef/cron/update_geolite_db.sh
+COPY --chown=mozdef:mozdef cron/update_geolite_db.py /opt/mozdef/envs/mozdef/cron/update_geolite_db.py
+COPY --chown=mozdef:mozdef cron/update_geolite_db.conf /opt/mozdef/envs/mozdef/cron/update_geolite_db.conf
+COPY --chown=mozdef:mozdef cron/update_geolite_db.sh /opt/mozdef/envs/mozdef/cron/update_geolite_db.sh
 
-COPY mozdef_util /opt/mozdef/envs/mozdef/mozdef_util
-
-RUN chown -R mozdef:mozdef /opt/mozdef/
+COPY --chown=mozdef:mozdef mozdef_util /opt/mozdef/envs/mozdef/mozdef_util
 
 USER mozdef
 RUN \
   virtualenv -p /usr/bin/python3.6 /opt/mozdef/envs/python && \
   source /opt/mozdef/envs/python/bin/activate && \
-  pip install -r /opt/mozdef/envs/mozdef/requirements.txt && \
+  pip install --requirement /opt/mozdef/envs/mozdef/requirements.txt && \
   cd /opt/mozdef/envs/mozdef/mozdef_util && \
-  pip install -e .
+  pip install --editable . && \
+  mkdir /opt/mozdef/envs/mozdef/data
 
-RUN mkdir /opt/mozdef/envs/mozdef/data
 
 WORKDIR /opt/mozdef/envs/mozdef
 
diff --git a/docker/compose/mozdef_bootstrap/Dockerfile b/docker/compose/mozdef_bootstrap/Dockerfile
index d4a1284c..8cbfaa9b 100644
--- a/docker/compose/mozdef_bootstrap/Dockerfile
+++ b/docker/compose/mozdef_bootstrap/Dockerfile
@@ -2,16 +2,14 @@ FROM mozdef/mozdef_base
 
 LABEL maintainer="mozdef@mozilla.com"
 
-RUN mkdir -p /opt/mozdef/envs/mozdef/docker/conf
+RUN install --owner mozdef --group mozdef --directory /opt/mozdef/envs/mozdef/docker /opt/mozdef/envs/mozdef/docker/conf
 
-COPY cron/mozdefStateDefaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/mozdefStateDefaultMappingTemplate.json
-COPY cron/defaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/defaultMappingTemplate.json
-COPY docker/compose/mozdef_cron/files/backup.conf /opt/mozdef/envs/mozdef/cron/backup.conf
-COPY docker/compose/mozdef_bootstrap/files/initial_setup.py /opt/mozdef/envs/mozdef/initial_setup.py
-COPY docker/compose/mozdef_bootstrap/files/index_mappings /opt/mozdef/envs/mozdef/index_mappings
-COPY docker/compose/mozdef_bootstrap/files/resources /opt/mozdef/envs/mozdef/resources
-
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/
+COPY --chown=mozdef:mozdef cron/mozdefStateDefaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/mozdefStateDefaultMappingTemplate.json
+COPY --chown=mozdef:mozdef cron/defaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/defaultMappingTemplate.json
+COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/backup.conf /opt/mozdef/envs/mozdef/cron/backup.conf
+COPY --chown=mozdef:mozdef docker/compose/mozdef_bootstrap/files/initial_setup.py /opt/mozdef/envs/mozdef/initial_setup.py
+COPY --chown=mozdef:mozdef docker/compose/mozdef_bootstrap/files/index_mappings /opt/mozdef/envs/mozdef/index_mappings
+COPY --chown=mozdef:mozdef docker/compose/mozdef_bootstrap/files/resources /opt/mozdef/envs/mozdef/resources
 
 WORKDIR /opt/mozdef/envs/mozdef
 
diff --git a/docker/compose/mozdef_bot/Dockerfile b/docker/compose/mozdef_bot/Dockerfile
index 04020161..fbabd774 100644
--- a/docker/compose/mozdef_bot/Dockerfile
+++ b/docker/compose/mozdef_bot/Dockerfile
@@ -4,10 +4,10 @@ LABEL maintainer="mozdef@mozilla.com"
 
 ARG BOT_TYPE
 
-COPY bot/$BOT_TYPE /opt/mozdef/envs/mozdef/bot/$BOT_TYPE
-COPY docker/compose/mozdef_bot/files/mozdefbot.conf /opt/mozdef/envs/mozdef/bot/$BOT_TYPE/mozdefbot.conf
+RUN install --owner mozdef --group mozdef --directory /opt/mozdef/envs/mozdef/bot
 
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/bot/$BOT_TYPE
+COPY --chown=mozdef:mozdef bot/$BOT_TYPE /opt/mozdef/envs/mozdef/bot/$BOT_TYPE
+COPY --chown=mozdef:mozdef docker/compose/mozdef_bot/files/mozdefbot.conf /opt/mozdef/envs/mozdef/bot/$BOT_TYPE/mozdefbot.conf
 
 WORKDIR /opt/mozdef/envs/mozdef/bot/$BOT_TYPE
 
diff --git a/docker/compose/mozdef_cognito_proxy/Dockerfile b/docker/compose/mozdef_cognito_proxy/Dockerfile
index df583870..da946ec6 100644
--- a/docker/compose/mozdef_cognito_proxy/Dockerfile
+++ b/docker/compose/mozdef_cognito_proxy/Dockerfile
@@ -3,5 +3,5 @@ ADD docker/compose/mozdef_cognito_proxy/files/default.conf /etc/nginx/conf.d/def
 ADD docker/compose/mozdef_cognito_proxy/files/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
 RUN touch /etc/nginx/htpasswd
 RUN /usr/local/openresty/luajit/bin/luarocks install lua-resty-jwt
-RUN yum install -y httpd-tools && yum clean all
+RUN yum install -y httpd-tools && yum clean all && rm -rf /var/cache/yum
 CMD bash -c "/usr/bin/htpasswd  -bc /etc/nginx/htpasswd mozdef $basic_auth_secret 2> /dev/null; /usr/bin/openresty -g 'daemon off;'"
diff --git a/docker/compose/mozdef_cron/Dockerfile b/docker/compose/mozdef_cron/Dockerfile
index 0fd25398..c322d4d9 100644
--- a/docker/compose/mozdef_cron/Dockerfile
+++ b/docker/compose/mozdef_cron/Dockerfile
@@ -5,20 +5,19 @@ LABEL maintainer="mozdef@mozilla.com"
 RUN \
   yum makecache fast && \
   yum install -y cronie && \
-  yum clean all
+  yum clean all && \
+  rm -rf /var/cache/yum
 
-COPY cron /opt/mozdef/envs/mozdef/cron
+COPY --chown=mozdef:mozdef cron /opt/mozdef/envs/mozdef/cron
 COPY docker/compose/mozdef_cron/files/cron_entries.txt /cron_entries.txt
 
 # Copy config files for crons
-COPY docker/compose/mozdef_cron/files/backup.conf /opt/mozdef/envs/mozdef/cron/backup.conf
-COPY docker/compose/mozdef_cron/files/collectAttackers.conf /opt/mozdef/envs/mozdef/cron/collectAttackers.conf
-COPY docker/compose/mozdef_cron/files/eventStats.conf /opt/mozdef/envs/mozdef/cron/eventStats.conf
-COPY docker/compose/mozdef_cron/files/healthAndStatus.conf /opt/mozdef/envs/mozdef/cron/healthAndStatus.conf
-COPY docker/compose/mozdef_cron/files/healthToMongo.conf /opt/mozdef/envs/mozdef/cron/healthToMongo.conf
-COPY docker/compose/mozdef_cron/files/syncAlertsToMongo.conf /opt/mozdef/envs/mozdef/cron/syncAlertsToMongo.conf
-
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/cron
+COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/backup.conf /opt/mozdef/envs/mozdef/cron/backup.conf
+COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/collectAttackers.conf /opt/mozdef/envs/mozdef/cron/collectAttackers.conf
+COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/eventStats.conf /opt/mozdef/envs/mozdef/cron/eventStats.conf
+COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/healthAndStatus.conf /opt/mozdef/envs/mozdef/cron/healthAndStatus.conf
+COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/healthToMongo.conf /opt/mozdef/envs/mozdef/cron/healthToMongo.conf
+COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/syncAlertsToMongo.conf /opt/mozdef/envs/mozdef/cron/syncAlertsToMongo.conf
 
 # https://stackoverflow.com/a/48651061/168874
 COPY docker/compose/mozdef_cron/files/launch_cron /launch_cron
diff --git a/docker/compose/mozdef_loginput/Dockerfile b/docker/compose/mozdef_loginput/Dockerfile
index 8f508ab6..8122b1ee 100644
--- a/docker/compose/mozdef_loginput/Dockerfile
+++ b/docker/compose/mozdef_loginput/Dockerfile
@@ -2,10 +2,8 @@ FROM mozdef/mozdef_base
 
 LABEL maintainer="mozdef@mozilla.com"
 
-COPY loginput /opt/mozdef/envs/mozdef/loginput
-COPY docker/compose/mozdef_loginput/files/index.conf /opt/mozdef/envs/mozdef/loginput/index.conf
-
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/loginput
+COPY --chown=mozdef:mozdef loginput /opt/mozdef/envs/mozdef/loginput
+COPY --chown=mozdef:mozdef docker/compose/mozdef_loginput/files/index.conf /opt/mozdef/envs/mozdef/loginput/index.conf
 
 EXPOSE 8080
 
diff --git a/docker/compose/mozdef_meteor/Dockerfile b/docker/compose/mozdef_meteor/Dockerfile
index 7fd6a673..cacdc8a6 100644
--- a/docker/compose/mozdef_meteor/Dockerfile
+++ b/docker/compose/mozdef_meteor/Dockerfile
@@ -11,46 +11,52 @@ ENV PORT=3000
 
 ARG METEOR_BUILD='YES'
 
-RUN \
-  useradd -ms /bin/bash -d /opt/mozdef -m mozdef && \
-  mkdir -p /opt/mozdef/envs/mozdef && \
-  cd /opt/mozdef && \
-  chown -R mozdef:mozdef /opt/mozdef && \
-  curl -sL https://rpm.nodesource.com/setup_8.x | bash - && \
-  yum makecache fast && \
-  yum install -y \
-                wget \
-                make \
-                glibc-devel \
-                gcc \
-                gcc-c++ \
-                libstdc++ \
-                libffi-devel \
-                zlib-devel \
-                nodejs && \
-  yum clean all && \
-  mkdir /opt/mozdef/meteor && \
-  curl -sL -o /opt/mozdef/meteor.tar.gz https://static-meteor.netdna-ssl.com/packages-bootstrap/$METEOR_VERSION/meteor-bootstrap-os.linux.x86_64.tar.gz && \
-  tar -xzf /opt/mozdef/meteor.tar.gz -C /opt/mozdef/meteor && \
-  mv /opt/mozdef/meteor/.meteor /opt/mozdef && \
-  rm -r /opt/mozdef/meteor && \
-  cp /opt/mozdef/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/scripts/admin/launch-meteor /usr/bin/meteor
+# Ignore warnings like 'No such file or directory for /usr/share/info/*.info.gz"
+# https://bugzilla.redhat.com/show_bug.cgi?id=516757
 
-COPY meteor /opt/mozdef/envs/mozdef/meteor
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/meteor
+RUN \
+  useradd --create-home --shell /bin/bash --home-dir /opt/mozdef mozdef && \
+  cd /opt/mozdef && \
+  gpg="gpg --no-default-keyring --secret-keyring /dev/null --keyring /dev/null --no-option --keyid-format 0xlong" && \
+  rpmkeys --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \
+  rpm -qi gpg-pubkey-f4a80eb5 | $gpg | grep 0x24C6A8A7F4A80EB5 && \
+  yum makecache fast && \
+  yum install -y which && \
+  curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \
+  rpmkeys --import /etc/pki/rpm-gpg/NODESOURCE-GPG-SIGNING-KEY-EL && \
+  rpm -qi gpg-pubkey-34fa74dd | $gpg | grep 0x5DDBE8D434FA74DD && \
+  yum install -y \
+     make \
+     glibc-devel \
+     gcc \
+     gcc-c++ \
+     libstdc++ \
+     libffi-devel \
+     zlib-devel \
+     nodejs && \
+  yum clean all && \
+  rm -rf /var/cache/yum && \
+  echo "Downloading meteor" && \
+  curl --silent --location https://static-meteor.netdna-ssl.com/packages-bootstrap/$METEOR_VERSION/meteor-bootstrap-os.linux.x86_64.tar.gz \
+    | tar --extract --gzip --directory /opt/mozdef .meteor && \
+  ln --symbolic /opt/mozdef/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/scripts/admin/launch-meteor /usr/bin/meteor && \
+  install --owner mozdef --group mozdef --directory /opt/mozdef/envs /opt/mozdef/envs/mozdef
+
+COPY --chown=mozdef:mozdef meteor /opt/mozdef/envs/mozdef/meteor
 
 USER mozdef
-RUN mkdir -p /opt/mozdef/envs/meteor/mozdef
 
 # build meteor runtime if asked, if set to NO, only create the dir created above to mount to do live development
-RUN if [ "${METEOR_BUILD}" = "YES" ]; then \
-        cd /opt/mozdef/envs/mozdef/meteor && \
-        meteor npm install && \
-        echo "Starting meteor build" && \
-        time meteor build --server localhost:3002 --directory /opt/mozdef/envs/meteor/mozdef && \
-        cp -r /opt/mozdef/envs/mozdef/meteor/node_modules /opt/mozdef/envs/meteor/mozdef/node_modules &&\
-        cd /opt/mozdef/envs/meteor/mozdef/bundle/programs/server && \
-        npm install ;\
+RUN \
+  if [ "${METEOR_BUILD}" = "YES" ]; then \
+    mkdir -p /opt/mozdef/envs/meteor/mozdef && \
+    cd /opt/mozdef/envs/mozdef/meteor && \
+    meteor npm install && \
+    echo "Starting meteor build" && \
+    time meteor build --server localhost:3002 --directory /opt/mozdef/envs/meteor/mozdef && \
+    ln --symbolic /opt/mozdef/envs/meteor/mozdef/node_modules /opt/mozdef/envs/mozdef/meteor/node_modules && \
+    cd /opt/mozdef/envs/meteor/mozdef/bundle/programs/server && \
+    npm install ;\
   fi
 
 WORKDIR /opt/mozdef/envs/meteor/mozdef
diff --git a/docker/compose/mozdef_mq_worker/Dockerfile b/docker/compose/mozdef_mq_worker/Dockerfile
index 902c242c..94bb2334 100644
--- a/docker/compose/mozdef_mq_worker/Dockerfile
+++ b/docker/compose/mozdef_mq_worker/Dockerfile
@@ -2,10 +2,8 @@ FROM mozdef/mozdef_base
 
 LABEL maintainer="mozdef@mozilla.com"
 
-COPY mq /opt/mozdef/envs/mozdef/mq
-COPY docker/compose/mozdef_mq_worker/files/*.conf /opt/mozdef/envs/mozdef/mq/
-
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/mq
+COPY --chown=mozdef:mozdef mq /opt/mozdef/envs/mozdef/mq
+COPY --chown=mozdef:mozdef docker/compose/mozdef_mq_worker/files/*.conf /opt/mozdef/envs/mozdef/mq/
 
 WORKDIR /opt/mozdef/envs/mozdef/mq
 
diff --git a/docker/compose/mozdef_rest/Dockerfile b/docker/compose/mozdef_rest/Dockerfile
index 6d4a8cfd..1c47f5d3 100644
--- a/docker/compose/mozdef_rest/Dockerfile
+++ b/docker/compose/mozdef_rest/Dockerfile
@@ -2,10 +2,8 @@ FROM mozdef/mozdef_base
 
 LABEL maintainer="mozdef@mozilla.com"
 
-COPY rest /opt/mozdef/envs/mozdef/rest
-COPY docker/compose/mozdef_rest/files/index.conf /opt/mozdef/envs/mozdef/rest/index.conf
-
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/rest
+COPY --chown=mozdef:mozdef rest /opt/mozdef/envs/mozdef/rest
+COPY --chown=mozdef:mozdef docker/compose/mozdef_rest/files/index.conf /opt/mozdef/envs/mozdef/rest/index.conf
 
 EXPOSE 8081
 
diff --git a/docker/compose/mozdef_sampledata/Dockerfile b/docker/compose/mozdef_sampledata/Dockerfile
index 8b8d69ea..2aff281b 100644
--- a/docker/compose/mozdef_sampledata/Dockerfile
+++ b/docker/compose/mozdef_sampledata/Dockerfile
@@ -2,11 +2,9 @@ FROM mozdef/mozdef_base
 
 LABEL maintainer="mozdef@mozilla.com"
 
-RUN mkdir -p /opt/mozdef/envs/mozdef/examples
-COPY ./examples /opt/mozdef/envs/mozdef/examples
+COPY --chown=mozdef:mozdef ./examples /opt/mozdef/envs/mozdef/examples
 
-COPY docker/compose/mozdef_sampledata/files/sampleData2MozDef.conf /opt/mozdef/envs/mozdef/examples/demo/sampleData2MozDef.conf
-RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/examples
+COPY --chown=mozdef:mozdef docker/compose/mozdef_sampledata/files/sampleData2MozDef.conf /opt/mozdef/envs/mozdef/examples/demo/sampleData2MozDef.conf
 RUN chmod u+rwx /opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh
 
 WORKDIR /opt/mozdef/envs/mozdef/examples/demo
diff --git a/docker/compose/mozdef_syslog/Dockerfile b/docker/compose/mozdef_syslog/Dockerfile
index f37e40e7..371c6134 100644
--- a/docker/compose/mozdef_syslog/Dockerfile
+++ b/docker/compose/mozdef_syslog/Dockerfile
@@ -14,7 +14,8 @@ RUN \
   rpm -qi gpg-pubkey-352c64e5 | $gpg | grep 0x6A2FAEA2352C64E5 && \
   yum install -y epel-release && \
   yum install -y syslog-ng.x86_64 syslog-ng-json && \
-  yum clean all
+  yum clean all && \
+  rm -rf /var/cache/yum
 
 COPY docker/compose/mozdef_syslog/files/syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 
diff --git a/docker/compose/rabbitmq/Dockerfile b/docker/compose/rabbitmq/Dockerfile
index a5f91f62..89fb9b1c 100644
--- a/docker/compose/rabbitmq/Dockerfile
+++ b/docker/compose/rabbitmq/Dockerfile
@@ -8,10 +8,13 @@ COPY docker/compose/rabbitmq/files/rabbitmq-server.repo /etc/yum.repos.d/rabbitm
 COPY docker/compose/rabbitmq/files/erlang.repo /etc/yum.repos.d/erlang.repo
 
 RUN \
-  yum -q makecache -y fast && \
-  yum install -y epel-release && \
+  gpg="gpg --no-default-keyring --secret-keyring /dev/null --keyring /dev/null --no-option --keyid-format 0xlong" && \
+  rpmkeys --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \
+  rpm -qi gpg-pubkey-f4a80eb5 | $gpg | grep 0x24C6A8A7F4A80EB5 && \
+  yum --quiet makecache -y fast && \
   yum install -y rabbitmq-server-$RABBITMQ_VERSION && \
-  yum clean all
+  yum clean all && \
+  rm -rf /var/cache/yum
 
 COPY docker/compose/rabbitmq/files/rabbitmq.config /etc/rabbitmq/
 COPY docker/compose/rabbitmq/files/enabled_plugins /etc/rabbitmq/

From fc3bd5e77031defa2a5ad49dda39b0caa989e33c Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Tue, 16 Jul 2019 12:23:50 -0500
Subject: [PATCH 18/58] Add registration to alert plugins

---
 alerts/plugins/ip_source_enrichment.py | 2 ++
 alerts/plugins/port_scan_enrichment.py | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/alerts/plugins/ip_source_enrichment.py b/alerts/plugins/ip_source_enrichment.py
index 84d18312..fd817ea8 100644
--- a/alerts/plugins/ip_source_enrichment.py
+++ b/alerts/plugins/ip_source_enrichment.py
@@ -140,6 +140,8 @@ class message(object):
     '''
 
     def __init__(self):
+        # Run plugin on all alerts
+        self.registration = '*'
         self._config = _load_config(CONFIG_FILE)
 
     def onMessage(self, message):
diff --git a/alerts/plugins/port_scan_enrichment.py b/alerts/plugins/port_scan_enrichment.py
index a7f97772..d6f940ea 100644
--- a/alerts/plugins/port_scan_enrichment.py
+++ b/alerts/plugins/port_scan_enrichment.py
@@ -88,6 +88,9 @@ class message(object):
     '''
 
     def __init__(self):
+        # Run plugin on all alerts
+        self.registration = '*'
+
         config = _load_config(CONFIG_FILE)
 
         try:

From 9d7ea147a9769809af1e3535505dbd6411c30fc5 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Tue, 16 Jul 2019 15:19:02 -0400
Subject: [PATCH 19/58] Add check for details on alert in plugin

---
 alerts/plugins/ip_source_enrichment.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/alerts/plugins/ip_source_enrichment.py b/alerts/plugins/ip_source_enrichment.py
index fd817ea8..81bf172a 100644
--- a/alerts/plugins/ip_source_enrichment.py
+++ b/alerts/plugins/ip_source_enrichment.py
@@ -59,6 +59,8 @@ def enrich(alert, known_ips):
 
     alert = alert.copy()
 
+    if 'details' not in alert:
+        alert['details'] = {}
     alert['details']['sites'] = []
 
     for ip in set(ips):

From bfa8640b576e2d6529dad239c6c2a7bfd84d84f1 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Tue, 16 Jul 2019 16:12:56 -0400
Subject: [PATCH 20/58] Modify registration for port scan alert plugin

---
 alerts/plugins/port_scan_enrichment.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/alerts/plugins/port_scan_enrichment.py b/alerts/plugins/port_scan_enrichment.py
index d6f940ea..bd180d60 100644
--- a/alerts/plugins/port_scan_enrichment.py
+++ b/alerts/plugins/port_scan_enrichment.py
@@ -88,8 +88,8 @@ class message(object):
     '''
 
     def __init__(self):
-        # Run plugin on all alerts
-        self.registration = '*'
+        # Run plugin on portscan alerts
+        self.registration = 'portscan'
 
         config = _load_config(CONFIG_FILE)
 

From 2b00efdbb1a32988bd199d1a8a6c147ef22ec4a7 Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Wed, 17 Jul 2019 08:39:54 -0700
Subject: [PATCH 21/58] Remove virtualenv requirement from MozDef and tox from
 mozdef_util

virtualenv isn't actually used within MozDef and so isn't needed
tox should only be part of the dev requirements for mozdef_util as it's used in testing, not the setup.py requirements
Fixes #1370
---
 mozdef_util/setup.py | 1 -
 requirements.txt     | 1 -
 2 files changed, 2 deletions(-)

diff --git a/mozdef_util/setup.py b/mozdef_util/setup.py
index 1ccafb9d..d2659135 100644
--- a/mozdef_util/setup.py
+++ b/mozdef_util/setup.py
@@ -20,7 +20,6 @@ requirements = [
     'wheel>=0.32.1',
     'watchdog>=0.9.0',
     'flake8>=3.5.0',
-    'tox>=3.5.2',
     'coverage>=4.5.1',
     'Sphinx>=1.8.1',
     'twine>=1.12.1',
diff --git a/requirements.txt b/requirements.txt
index 6d328299..7f2787c1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -58,6 +58,5 @@ tzlocal==1.4
 uritemplate==0.6
 urllib3==1.24.3
 uwsgi==2.0.17.1
-virtualenv==1.11.4
 tldextract==2.2.0
 websocket-client==0.44.0

From 4f8f20993a72e388c3c7e4e5ef175d2b18fd1f6a Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Wed, 17 Jul 2019 12:31:25 -0400
Subject: [PATCH 22/58] Update get_indices to include closed indices

---
 mozdef_util/mozdef_util/elasticsearch_client.py | 3 ++-
 tests/mozdef_util/test_elasticsearch_client.py  | 8 ++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/mozdef_util/mozdef_util/elasticsearch_client.py b/mozdef_util/mozdef_util/elasticsearch_client.py
index 44e9a3a2..79bf4e5a 100644
--- a/mozdef_util/mozdef_util/elasticsearch_client.py
+++ b/mozdef_util/mozdef_util/elasticsearch_client.py
@@ -53,7 +53,8 @@ class ElasticsearchClient():
         self.es_connection.indices.delete(index=index_name, ignore=ignore_codes)
 
     def get_indices(self):
-        return list(self.es_connection.indices.stats()['indices'].keys())
+        # Includes open and closed indices
+        return list(self.es_connection.indices.get_alias('*', params=dict(expand_wildcards='all')).keys())
 
     def index_exists(self, index_name):
         return self.es_connection.indices.exists(index_name)
diff --git a/tests/mozdef_util/test_elasticsearch_client.py b/tests/mozdef_util/test_elasticsearch_client.py
index 4a62b5f2..2fb75568 100644
--- a/tests/mozdef_util/test_elasticsearch_client.py
+++ b/tests/mozdef_util/test_elasticsearch_client.py
@@ -415,6 +415,14 @@ class TestGetIndices(ElasticsearchClientTest):
         indices.sort()
         assert indices == [self.alert_index_name, self.previous_event_index_name, self.event_index_name, 'test_index']
 
+    def test_closed_get_indices(self):
+        if self.config_delete_indexes:
+            self.es_client.create_index('test_index')
+        time.sleep(1)
+        self.es_client.close_index('test_index')
+        all_indices = self.es_client.get_indices()
+        assert 'test_index' in all_indices
+
 
 class TestIndexExists(ElasticsearchClientTest):
 

From 9233acb5628d6ccccbdd157d8fd87e72dbf83b45 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Wed, 17 Jul 2019 12:32:03 -0400
Subject: [PATCH 23/58] Update mozdef_util to version 3.0.2

---
 mozdef_util/HISTORY.rst | 6 ++++++
 mozdef_util/setup.py    | 2 +-
 requirements.txt        | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/mozdef_util/HISTORY.rst b/mozdef_util/HISTORY.rst
index 05c26617..e627ab78 100644
--- a/mozdef_util/HISTORY.rst
+++ b/mozdef_util/HISTORY.rst
@@ -92,3 +92,9 @@ Add is_ip utility function
 ------------------
 
 * Updated bulk queue to acquire lock before saving events
+
+
+3.0.2 (2019-07-17)
+------------------
+
+* Updated ElasticsearchClient.get_indices() to include closed indices
diff --git a/mozdef_util/setup.py b/mozdef_util/setup.py
index 1ccafb9d..d14af210 100644
--- a/mozdef_util/setup.py
+++ b/mozdef_util/setup.py
@@ -59,6 +59,6 @@ setup(
     test_suite='tests',
     tests_require=[],
     url='https://github.com/mozilla/MozDef/tree/master/lib',
-    version='3.0.1',
+    version='3.0.2',
     zip_safe=False,
 )
diff --git a/requirements.txt b/requirements.txt
index 6d328299..a1e2529f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -30,7 +30,7 @@ jmespath==0.9.3
 kombu==4.1.0
 meld3==1.0.2
 mozdef-client==1.0.11
-mozdef-util==3.0.1
+mozdef-util==3.0.2
 netaddr==0.7.19
 nose==1.3.7
 oauth2client==1.4.12

From 5373a3b320297e6844e6e9c958ca0be40a0797f3 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Wed, 17 Jul 2019 14:16:34 -0400
Subject: [PATCH 24/58] Update healthToMongo to include 2 digits for disk usage

---
 cron/healthToMongo.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/cron/healthToMongo.py b/cron/healthToMongo.py
index d3c748fe..db49c344 100755
--- a/cron/healthToMongo.py
+++ b/cron/healthToMongo.py
@@ -77,11 +77,14 @@ def getEsNodesStats():
         load_str = "{0},{1},{2}".format(load_average['1m'], load_average['5m'], load_average['15m'])
         hostname = nodeid
         if 'host' in jsonobj['nodes'][nodeid]:
-            hostname=jsonobj['nodes'][nodeid]['host']
+            hostname = jsonobj['nodes'][nodeid]['host']
+
+        disk_free = "{0:.2f}".format(jsonobj['nodes'][nodeid]['fs']['total']['free_in_bytes'] / (1024 * 1024 * 1024))
+        disk_total = "{0:.2f}".format(jsonobj['nodes'][nodeid]['fs']['total']['total_in_bytes'] / (1024 * 1024 * 1024))
         results.append({
             'hostname': hostname,
-            'disk_free': jsonobj['nodes'][nodeid]['fs']['total']['free_in_bytes'] / (1024 * 1024 * 1024),
-            'disk_total': jsonobj['nodes'][nodeid]['fs']['total']['total_in_bytes'] / (1024 * 1024 * 1024),
+            'disk_free': disk_free,
+            'disk_total': disk_total,
             'mem_heap_per': jsonobj['nodes'][nodeid]['jvm']['mem']['heap_used_percent'],
             'gc_old': jsonobj['nodes'][nodeid]['jvm']['gc']['collectors']['old']['collection_time_in_millis'] / 1000,
             'cpu_usage': jsonobj['nodes'][nodeid]['os']['cpu']['percent'],

From 28bb27b7b51758b272c5f0310103a35ae41cd645 Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Wed, 17 Jul 2019 17:18:35 -0700
Subject: [PATCH 25/58] Add check to prevent CodeBuild rebuilds

---
 cloudy_mozdef/ci/deploy | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cloudy_mozdef/ci/deploy b/cloudy_mozdef/ci/deploy
index c50ff0e2..ceefa7f8 100755
--- a/cloudy_mozdef/ci/deploy
+++ b/cloudy_mozdef/ci/deploy
@@ -21,6 +21,10 @@ echo "  Event : ${CODEBUILD_WEBHOOK_EVENT}"
 echo "  Head Ref : ${CODEBUILD_WEBHOOK_HEAD_REF}"
 echo "  Trigger : ${CODEBUILD_WEBHOOK_TRIGGER}"
 
+if [ -z "${CODEBUILD_WEBHOOK_TRIGGER}" ]; then
+  echo "CODEBUILD_WEBHOOK_TRIGGER is unset, likely because this build was retried. Retry build isn't supported. Exiting"
+  exit 1
+fi
 echo "Codebuild is ubuntu 14.04. Installing packer in order to compensate. Someone should build a CI docker container \;)."
 wget -nv https://releases.hashicorp.com/packer/1.3.5/packer_1.3.5_linux_amd64.zip
 unzip packer_1.3.5_linux_amd64.zip -d /usr/bin

From 7cb752ceeb5124167df263196e8c3d45f565af34 Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Wed, 17 Jul 2019 17:21:36 -0700
Subject: [PATCH 26/58] Bugfix AWS CodeBuild settings

* Allow setting the CodeBuild project name (as project names can't be changed)
* Fix typo in IAM Policy S3 resource ARN
* Fix IAM Policy CloudWatch log policy statements
* Clarify CodeBuild project description
* Add PrivilegedMode true to fix Docker errors
* Fix GitHub location URL
* Fix FilterGroup for tagged commits
---
 .../cloudformation/mozdef-cicd-codebuild.yml  | 50 +++++++++----------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/cloudy_mozdef/cloudformation/mozdef-cicd-codebuild.yml b/cloudy_mozdef/cloudformation/mozdef-cicd-codebuild.yml
index aacaa102..07a891bf 100644
--- a/cloudy_mozdef/cloudformation/mozdef-cicd-codebuild.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-cicd-codebuild.yml
@@ -1,5 +1,9 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: CodeBuild CI/CD Job to build on commit
+Description: MozDef CodeBuild CI/CD Job and IAM role
+Parameters:
+  CodeBuildProjectName:
+    Type: String
+    Description: The name of the CodeBuild project to create. Project names can't be modified once the project is created
 Mappings:
   VariableMap:
     Variables:
@@ -42,12 +46,12 @@ Resources:
                 Action:
                   - s3:PutObject*
                   - s3:GetObject*
-                Resource: !Join [ '', [ 'arn:aws:s3::', !FindInMap [ 'VariableMap', 'Variables', 'S3BucketToPublishCloudFormationTemplatesTo' ], '/*' ] ]
+                Resource: !Join [ '', [ 'arn:aws:s3:::', !FindInMap [ 'VariableMap', 'Variables', 'S3BucketToPublishCloudFormationTemplatesTo' ], '/*' ] ]
               - Sid: ListS3BucketContents
                 Effect: Allow
                 Action:
                   - s3:ListBucket*
-                Resource: !Join [ '', [ 'arn:aws:s3::', !FindInMap [ 'VariableMap', 'Variables', 'S3BucketToPublishCloudFormationTemplatesTo' ] ] ]
+                Resource: !Join [ '', [ 'arn:aws:s3:::', !FindInMap [ 'VariableMap', 'Variables', 'S3BucketToPublishCloudFormationTemplatesTo' ] ] ]
               - Sid: CreatePackerEC2Instance
                 Effect: Allow
                 Action:
@@ -79,24 +83,21 @@ Resources:
                   - ec2:DescribeImageAttribute
                   - ec2:DescribeSubnets
                 Resource: '*'
-              - Sid: ReadSSMParameters
+              - Sid: ReadSecrets
                 Effect: Allow
                 Action: ssm:GetParameter
                 Resource: arn:aws:ssm:*:*:parameter/mozdef/ci/*
-              # I think these are vestigial, created by the CodeBuild UI.
-              # Also they're not even the right resource path since they contain "/aws/codebuild/" but the actual LogGroup that CodeBuild writes to doesn't
-              # e.g. arn:aws:logs:us-west-2:371522382791:log-group:MozDefCI:*
-#              - Sid: NotSure1
-#                Effect: Allow
-#                Action:
-#                  - logs:CreateLogGroup
-#                  - logs:CreateLogStream
-#                  - logs:PutLogEvents
-#                Resource:
-#                  - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/mozdef' ] ]
-#                  - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/mozdef:*' ] ]
-#                  - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/MozDefCI' ] ]
-#                  - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/MozDefCI:*' ] ]
+              - Sid: CloudWatchLogGroup
+                Effect: Allow
+                Action:
+                  - logs:CreateLogGroup
+                Resource: !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group', !FindInMap [ 'VariableMap', 'Variables', 'CloudWatchLogGroupName' ] ] ]
+              - Sid: CloudWatchLogStream
+                Effect: Allow
+                Action:
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                Resource: !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group', !FindInMap [ 'VariableMap', 'Variables', 'CloudWatchLogGroupName' ], 'log-stream:*' ] ]
               - Sid: NotSure2
                 Action:
                   - s3:PutObject
@@ -110,8 +111,8 @@ Resources:
   CodeBuildProject:
     Type: AWS::CodeBuild::Project
     Properties:
-      Name: mozdef
-      Description: Builds MozDef AMI, dockers containers,  and runs test suite. Owner is Andrew Krug.
+      Name: !Ref CodeBuildProjectName
+      Description: Builds the MozDef AMI, the MozDef Docker containers and shares the AMIs with AWS Marketplace.
       BadgeEnabled: True
       ServiceRole: !GetAtt CodeBuildServiceRole.Arn
       Artifacts:
@@ -120,22 +121,21 @@ Resources:
         Type: LINUX_CONTAINER
         ComputeType: BUILD_GENERAL1_MEDIUM
         Image: aws/codebuild/docker:18.09.0-1.7.0
+        PrivilegedMode: true  # Required for docker
       Source:
         Type: GITHUB
-        # Auth:  # This information is for the AWS CodeBuild console's use only. Your code should not get or set Auth directly.
-        # SourceIdentifier:  # Not sure what this should be yet
         BuildSpec: cloudy_mozdef/buildspec.yml
-        Location: https://github.com/mozilla/MozDef
+        Location: https://github.com/mozilla/MozDef.git
         ReportBuildStatus: True
       Triggers:
         Webhook: true
         FilterGroups:
           - - Type: EVENT
               Pattern: PUSH
-            - Type: HEAD_REF  # Build on commits to branch reinforce2019
-              Pattern: '^refs/heads/reinforce2019'
             - Type: HEAD_REF  # Build on commits to branch master
               Pattern: '^refs/heads/master'
+          - - Type: EVENT
+              Pattern: PUSH
             - Type: HEAD_REF  # Build on tags like v1.2.3 and v1.2.3-testing
               Pattern: '^refs/tags\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$'
       Tags:

From 1aeb8da8095e41c3a33ed084c34824696d9429fe Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Wed, 17 Jul 2019 17:22:01 -0700
Subject: [PATCH 27/58] Update documentation to conform to new AWS CodeBuild
 provisioning method

---
 docs/source/cicd.rst | 72 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 54 insertions(+), 18 deletions(-)

diff --git a/docs/source/cicd.rst b/docs/source/cicd.rst
index 8bd09d45..356b9fb3 100644
--- a/docs/source/cicd.rst
+++ b/docs/source/cicd.rst
@@ -54,26 +54,58 @@ AWS CodeBuild
 Enabling GitHub AWS CodeBuild Integration
 _________________________________________
 
-* Request that a github.com/mozilla GitHub Organization owner temporarily
-  `approve / whitelist
-  <https://help.github.com/en/articles/approving-oauth-apps-for-your-organization>`_
-  the `AWS CodeBuild integration <https://bugzilla.mozilla.org/show_bug.cgi?id=1506740>`_
-  in the github.com/mozilla GitHub Organization
-* Manually configure the GitHub integration in AWS CodeBuild which will create
-  the GitHub webhooks needed using the dedicated, AWS account specific, GitHub
-  service user. A service user is needed as AWS CodeBuild can only integrate
-  with GitHub from one AWS account in one region with a single GitHub user.
-  Technically we could use different users for each region in a single AWS
-  account, but for simplicity we're limiting to only one GitHub user per AWS
-  account (instead of one GitHub user per AWS account per region)
+Onetime Manual Step
+*******************
 
-  * For the `infosec-prod` AWS account use the `infosec-prod-371522382791-codebuild`
-    GitHub user
-  * For the `infosec-dev` AWS account use the `infosec-dev-656532927350-codebuild`
-    GitHub user
+The steps to establish a GitHub CodeBuild integration unfortunately
+require a onetime manual step be done before using CloudFormation to
+configure the integration. This onetime manual step **need only happen a
+single time for a given AWS Account + Region**. It need **not be
+performed with each new CodeBuild project or each new GitHub repo**
 
-* Request that a GitHub Organization owner, re-deny the integration for
-  github.com/mozilla
+1. Manually enable the GitHub integration in AWS CodeBuild using the
+   dedicated, AWS account specific, GitHub service user.
+
+   1. A service user is needed as AWS CodeBuild can only integrate with
+      GitHub from one AWS account in one region with a single GitHub
+      user. Technically you could use different users for each region in
+      a single AWS account, but for simplicity limit yourself to only
+      one GitHub user per AWS account (instead of one GitHub user per
+      AWS account per region)
+
+   2. To do the one time step of integrating the entire AWS account in
+      that region with the GitHub service user
+
+      1. Browse to `CodeBuild`_\  in AWS and click Create Project
+      2. Navigate down to ``Source`` and set ``Source Provider`` to
+         ``GitHub``
+      3. For ``Repository`` select
+         ``Connect with a GitHub personal access token``
+      4. Enter the persona access token for the GitHub service user. If
+         you haven't created one do so and grant it ``repo`` and
+         ``admin:repo_hook``
+      5. Click ``Save Token``
+      6. Abort the project setup process by clicking the
+         ``Build Projects`` breadcrumb at the top. This “Save Token”
+         step was the only thing you needed to do in that process
+
+Grant the GitHub service user access to the GitHub repository
+*************************************************************
+
+1. As an admin of the GitHub repository go to that repositories
+   settings, select Collaborators and Teams, and add the GitHub
+   service user to the repository
+2. Set their access level to ``Admin``
+3. Copy the invite link, login as the service user and accept the
+   invitation
+
+Deploy CloudFormation stack creating CodeBuild project
+******************************************************
+
+Deploy the ``mozdef-cicd-codebuild.yml`` CloudFormation template
+to create the CodeBuild project and IAM Role
+
+.. _CodeBuild: https://us-west-2.console.aws.amazon.com/codesuite/codebuild/
 
 The Build Sequence
 __________________
@@ -81,6 +113,10 @@ __________________
 * A branch is merged into `master` in the GitHub repo or a version git tag is
   applied to a commit
 * GitHub emits a webhook event to AWS CodeBuild indicating this
+* AWS CodeBuild considers the Filter Groups configured to decide if the tag
+  or branch warrants triggering a build. These Filter Groups are defined in
+  the ``mozdef-cicd-codebuild.yml`` CloudFormation template. Assuming the tag
+  or branch are acceptable, CodeBuild continues.
 * AWS CodeBuild reads the
   `buildspec.yml <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/buildspec.yml>`_
   file to know what to do

From 0e180cdc5c253ff49a9ff8949d34ce3cc6c61793 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 18 Jul 2019 10:40:47 -0400
Subject: [PATCH 28/58] Add get_open_indices on mozdef_util elasticsearch
 client

---
 mozdef_util/mozdef_util/elasticsearch_client.py |  3 +++
 tests/mozdef_util/test_elasticsearch_client.py  | 12 +++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/mozdef_util/mozdef_util/elasticsearch_client.py b/mozdef_util/mozdef_util/elasticsearch_client.py
index 79bf4e5a..f05c9217 100644
--- a/mozdef_util/mozdef_util/elasticsearch_client.py
+++ b/mozdef_util/mozdef_util/elasticsearch_client.py
@@ -56,6 +56,9 @@ class ElasticsearchClient():
         # Includes open and closed indices
         return list(self.es_connection.indices.get_alias('*', params=dict(expand_wildcards='all')).keys())
 
+    def get_open_indices(self):
+        return list(self.es_connection.indices.get_alias('*', params=dict(expand_wildcards='open')).keys())
+
     def index_exists(self, index_name):
         return self.es_connection.indices.exists(index_name)
 
diff --git a/tests/mozdef_util/test_elasticsearch_client.py b/tests/mozdef_util/test_elasticsearch_client.py
index 2fb75568..f2a344c5 100644
--- a/tests/mozdef_util/test_elasticsearch_client.py
+++ b/tests/mozdef_util/test_elasticsearch_client.py
@@ -411,9 +411,13 @@ class TestGetIndices(ElasticsearchClientTest):
         if self.config_delete_indexes:
             self.es_client.create_index('test_index')
         time.sleep(1)
-        indices = self.es_client.get_indices()
-        indices.sort()
-        assert indices == [self.alert_index_name, self.previous_event_index_name, self.event_index_name, 'test_index']
+        all_indices = self.es_client.get_indices()
+        all_indices.sort()
+        open_indices = self.es_client.get_open_indices()
+        open_indices.sort()
+        expected_indices = [self.alert_index_name, self.previous_event_index_name, self.event_index_name, 'test_index']
+        assert all_indices == expected_indices
+        assert open_indices == expected_indices
 
     def test_closed_get_indices(self):
         if self.config_delete_indexes:
@@ -421,7 +425,9 @@ class TestGetIndices(ElasticsearchClientTest):
         time.sleep(1)
         self.es_client.close_index('test_index')
         all_indices = self.es_client.get_indices()
+        open_indices = self.es_client.get_open_indices()
         assert 'test_index' in all_indices
+        assert 'test_index' not in open_indices
 
 
 class TestIndexExists(ElasticsearchClientTest):

From 799b4315ee4827a9ef833dd0b85557ee844f3fb9 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 18 Jul 2019 10:41:03 -0400
Subject: [PATCH 29/58] Modify cron scripts to use get_open_indices

---
 cron/closeIndices.py | 2 +-
 cron/esCacheMaint.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cron/closeIndices.py b/cron/closeIndices.py
index 99e5bd1b..6442ca31 100644
--- a/cron/closeIndices.py
+++ b/cron/closeIndices.py
@@ -24,7 +24,7 @@ def esCloseIndices():
     logger.debug('started')
     try:
         es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
-        indices = es.get_indices()
+        indices = es.get_open_indices()
     except Exception as e:
         logger.error("Unhandled exception while connecting to ES, terminating: %r" % (e))
 
diff --git a/cron/esCacheMaint.py b/cron/esCacheMaint.py
index ca45ddbc..7a499865 100755
--- a/cron/esCacheMaint.py
+++ b/cron/esCacheMaint.py
@@ -46,7 +46,7 @@ def isJVMMemoryHigh():
 
 def clearESCache():
     es = esConnect(None)
-    indexes = es.get_indices()
+    indexes = es.get_open_indices()
     # assums index names  like events-YYYYMMDD etc.
     # used to avoid operating on current indexes
     dtNow = datetime.utcnow()

From f3ef773ceccad738d91156216f66113a1891c685 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 18 Jul 2019 10:41:15 -0400
Subject: [PATCH 30/58] Create mozdef_util version 3.0.3

---
 mozdef_util/HISTORY.rst | 6 ++++++
 mozdef_util/setup.py    | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/mozdef_util/HISTORY.rst b/mozdef_util/HISTORY.rst
index e627ab78..2d5d084e 100644
--- a/mozdef_util/HISTORY.rst
+++ b/mozdef_util/HISTORY.rst
@@ -98,3 +98,9 @@ Add is_ip utility function
 ------------------
 
 * Updated ElasticsearchClient.get_indices() to include closed indices
+
+
+3.0.3 (2019-07-18)
+------------------
+
+* Added ElasticsearchClient.get_open_indices()
diff --git a/mozdef_util/setup.py b/mozdef_util/setup.py
index 922c6443..0fa783f1 100644
--- a/mozdef_util/setup.py
+++ b/mozdef_util/setup.py
@@ -58,6 +58,6 @@ setup(
     test_suite='tests',
     tests_require=[],
     url='https://github.com/mozilla/MozDef/tree/master/lib',
-    version='3.0.2',
+    version='3.0.3',
     zip_safe=False,
 )

From 51eb8be6c13a5aaf9f7518d90bf200cb866b5611 Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Thu, 18 Jul 2019 07:50:58 -0700
Subject: [PATCH 31/58] Rename CHANGELOG to conform to keepachangelog.org
 standard

---
 CHANGELOG => CHANGELOG.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename CHANGELOG => CHANGELOG.md (100%)

diff --git a/CHANGELOG b/CHANGELOG.md
similarity index 100%
rename from CHANGELOG
rename to CHANGELOG.md

From 9876c2eafc5ae1e72c6a889a4536be2d7a40fa1c Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 18 Jul 2019 11:13:24 -0400
Subject: [PATCH 32/58] Modify mozdef_util version in requirements

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 381efc26..32fb6aad 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -30,7 +30,7 @@ jmespath==0.9.3
 kombu==4.1.0
 meld3==1.0.2
 mozdef-client==1.0.11
-mozdef-util==3.0.2
+mozdef-util==3.0.3
 netaddr==0.7.19
 nose==1.3.7
 oauth2client==1.4.12

From f517be47a85f5273a4513bf5bf737a3d6474cd9d Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Thu, 18 Jul 2019 08:32:43 -0700
Subject: [PATCH 33/58] Release v3.1.0

Update changelog
Update links to launch MozDef in AWS
---
 CHANGELOG.md                     | 39 +++++++++++++++++++++++++++++++-
 README.md                        |  2 +-
 docs/source/cloud_deployment.rst |  2 +-
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 81158bdb..caabce25 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,42 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 
+## [v3.1.0] - 2019-07-18
+
+### Added
+- Captured the AWS CodeBuild CI/CD configuration in code with documentation
+- Support for HTTP Basic Auth in AWS deployment
+- Docker healthchecks to docker containers
+- Descriptions to all AWS Lambda functions
+- Support for alerts-* index in docker environment
+- Alert that detects excessive numbers of AWS API describe calls
+- Additional AWS infrastructure to support AWS re:Inforce 2019 workshop
+- Documentation specific to MozDef installation now that MozDef uses Python 3
+- Config setting for CloudTrail notification SQS queue polling time
+- Config setting for Slack bot welcome message
+
+### Changed
+- Kibana port from 9443 to 9090
+- AWS CloudFormation default values from "unset" to empty string
+- Simplify mozdef-mq logic determining AMQP endpoint URI
+- SQS to always use secure transport
+- CloudTrail alert unit tests
+- Incident summary placeholder text for greater clarity
+- Display of Veris data for easier viewing
+- All Dockerfiles to reduce image size, pin package signing keys and improve
+  clarity
+
+### Fixed
+- Workers starting before GeoIP data is available
+- Mismatched MozDefACMCertArn parameter name in CloudFormation template
+- Duplicate mozdefvpcflowlogs object
+- Hard coded AWS Availability Zone
+- httplib2 by updating to version to 0.13.0 for python3
+- mozdef_util by modifying bulk queue to acquire lock before saving events
+- Dashboard Kibana URL
+- Unnecessary and conflicting package dependencies from MozDef and mozdef_util
+- get_indices to include closed indices
+
 ## [v3.0.0] - 2019-07-08
 ### Added
 - Support for Python3
@@ -132,7 +168,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added checks on sending SQS messages to only accept intra-account messages
 - Improved docker performance and disk space requirements
 
-[Unreleased]: https://github.com/mozilla/MozDef/compare/v3.0.0...HEAD
+[Unreleased]: https://github.com/mozilla/MozDef/compare/v3.1.0...HEAD
+[v3.1.0]: https://github.com/mozilla/MozDef/compare/v3.0.0...v3.1.0
 [v3.0.0]: https://github.com/mozilla/MozDef/compare/v2.0.1...v3.0.0
 [v2.0.1]: https://github.com/mozilla/MozDef/compare/v2.0.0...v2.0.1
 [v2.0.0]: https://github.com/mozilla/MozDef/compare/v1.40.0...v2.0.0
diff --git a/README.md b/README.md
index d973bf62..096b5b61 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ The Mozilla Enterprise Defense Platform (MozDef) seeks to automate the security
 
 MozDef is in production at Mozilla where we are using it to process over 300 million events per day.
 
-[1]: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/v1.38.5/mozdef-parent.yml
+[1]: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/v3.1.0/mozdef-parent.yml
 
 ## Survey & Contacting us
 
diff --git a/docs/source/cloud_deployment.rst b/docs/source/cloud_deployment.rst
index 44122275..f69a20cd 100644
--- a/docs/source/cloud_deployment.rst
+++ b/docs/source/cloud_deployment.rst
@@ -7,7 +7,7 @@ Cloud based MozDef is an opinionated deployment of the MozDef services created i
 ingest CloudTrail, GuardDuty, and provide security services.
 
 .. image:: images/cloudformation-launch-stack.png
-   :target: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/v1.38.5/mozdef-parent.yml
+   :target: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/v3.1.0/mozdef-parent.yml
 
 
 Feedback

From 504db1e7612b3bd17da222384cd6b74e44837ac1 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 18 Jul 2019 16:37:50 -0400
Subject: [PATCH 34/58] Add instructions for installing on mac os x

---
 docs/source/installation.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/source/installation.rst b/docs/source/installation.rst
index f0ab4262..8f89794e 100644
--- a/docs/source/installation.rst
+++ b/docs/source/installation.rst
@@ -113,6 +113,13 @@ Then::
   PYCURL_SSL_LIBRARY=nss pip install -r requirements.txt
 
 
+If you're using Mac OS X::
+
+  export PYCURL_SSL_LIBRARY=openssl
+  export LDFLAGS=-L/usr/local/opt/openssl/lib;export CPPFLAGS=-I/usr/local/opt/openssl/include
+  pip install -r requirements.txt
+
+
 Copy the following into a file called .bash_profile for the mozdef user within /opt/mozdef::
 
   [mozdef@server ~]$ vim /opt/mozdef/.bash_profile

From aa03a4da57bdb445cee693a3cbf4a3a2e3f3905d Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Fri, 19 Jul 2019 13:46:23 -0400
Subject: [PATCH 35/58] Remove unused eventexchange options

---
 mq/esworker_cloudtrail.py | 1 -
 mq/esworker_sqs.py        | 1 -
 2 files changed, 2 deletions(-)

diff --git a/mq/esworker_cloudtrail.py b/mq/esworker_cloudtrail.py
index dc747756..039075d0 100755
--- a/mq/esworker_cloudtrail.py
+++ b/mq/esworker_cloudtrail.py
@@ -413,7 +413,6 @@ def initConfig():
     # rabbit message queue options
     options.mqserver = getConfig('mqserver', 'localhost', options.configfile)
     options.taskexchange = getConfig('taskexchange', 'eventtask', options.configfile)
-    options.eventexchange = getConfig('eventexchange', 'events', options.configfile)
     # rabbit: how many messages to ask for at once from the message queue
     options.prefetch = getConfig('prefetch', 10, options.configfile)
     # rabbit: user creds
diff --git a/mq/esworker_sqs.py b/mq/esworker_sqs.py
index bb55e8e2..6275bfbc 100755
--- a/mq/esworker_sqs.py
+++ b/mq/esworker_sqs.py
@@ -355,7 +355,6 @@ def initConfig():
     # rabbit message queue options
     options.mqserver = getConfig('mqserver', 'localhost', options.configfile)
     options.taskexchange = getConfig('taskexchange', 'eventtask', options.configfile)
-    options.eventexchange = getConfig('eventexchange', 'events', options.configfile)
     # rabbit: how many messages to ask for at once from the message queue
     options.prefetch = getConfig('prefetch', 10, options.configfile)
     # rabbit: user creds

From 7019a0060bb183aba14f48c38376592a378dd993 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Fri, 19 Jul 2019 16:10:52 -0400
Subject: [PATCH 36/58] Update Elasticsearch version for cloudymozdef to 6.8

---
 cloudy_mozdef/cloudformation/mozdef-es.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudy_mozdef/cloudformation/mozdef-es.yml b/cloudy_mozdef/cloudformation/mozdef-es.yml
index e975662f..96b0002b 100644
--- a/cloudy_mozdef/cloudformation/mozdef-es.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-es.yml
@@ -63,7 +63,7 @@ Resources:
         EBSEnabled: true
         VolumeType: gp2
         VolumeSize: !Ref BlockStoreSizeGB
-      ElasticsearchVersion: '5.6'
+      ElasticsearchVersion: '6.8'
       ElasticsearchClusterConfig:
         InstanceCount: !Ref ESInstanceCount
       AccessPolicies:

From fa7875f5c43055dc887aeb3c31b020b9122444de Mon Sep 17 00:00:00 2001
From: Brandon Myers <pwnbus@users.noreply.github.com>
Date: Fri, 19 Jul 2019 17:50:58 -0400
Subject: [PATCH 37/58] Fix region parameter in connect_sqs (#1383)

* Fix region parameter in connect_sqs

* Modify parameter names to get_aws_credentials

* Clean up connect_sqs function to call get_aws_credentials

* Cleanup workers to use connect_sqs call

* Fix local import in sqs file

* Fix parameter name in cloudtrail worker

* Assert AWS region of SQS queues

Since the two SQS queues are provisioned by CloudTrail in the same region as the CloudTrail
stack, let's assert to MozDef that the region for those queus is indeed the same region.

* Update region name parameter in cloudtrail worker
---
 .../cloudformation/mozdef-instance.yml        |  2 ++
 mq/esworker_cloudtrail.py                     | 22 +++++++++----------
 mq/esworker_sns_sqs.py                        | 11 +++++-----
 mq/esworker_sqs.py                            | 11 +++++-----
 mq/lib/aws.py                                 | 18 +++++++--------
 mq/lib/sqs.py                                 | 10 ++-------
 6 files changed, 33 insertions(+), 41 deletions(-)

diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml
index bb702084..73417ab6 100644
--- a/cloudy_mozdef/cloudformation/mozdef-instance.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml
@@ -340,9 +340,11 @@ Resources:
             - content: |
                 # This configures the worker that pulls in CloudTrail logs
                 OPTIONS_TASKEXCHANGE=${CloudTrailSQSNotificationQueueName}
+                OPTIONS_REGION=${AWS::Region}
               path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env
             - content: |
                 OPTIONS_TASKEXCHANGE=${MozDefSQSQueueName}
+                OPTIONS_REGION=${AWS::Region}
               path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
             - content: |
                 [Unit]
diff --git a/mq/esworker_cloudtrail.py b/mq/esworker_cloudtrail.py
index dc747756..27712466 100755
--- a/mq/esworker_cloudtrail.py
+++ b/mq/esworker_cloudtrail.py
@@ -228,10 +228,10 @@ class taskConsumer(object):
             self.flush_wait_time = (response['Credentials']['Expiration'] - current_time).seconds - 3
         else:
             role_creds = {}
+        role_creds['region_name'] = options.region
         self.s3_client = boto3.client(
             's3',
-            region_name=options.region,
-            **role_creds
+            **get_aws_credentials(**role_creds)
         )
 
     def reauth_timer(self):
@@ -284,11 +284,10 @@ class taskConsumer(object):
                 logger.info('Received network related error...reconnecting')
                 time.sleep(5)
                 self.sqs_queue = connect_sqs(
-                    task_exchange=options.taskexchange,
-                    **get_aws_credentials(
-                        options.region,
-                        options.accesskey,
-                        options.secretkey)
+                    region_name=options.region,
+                    aws_access_key_id=options.accesskey,
+                    aws_secret_access_key=options.secretkey,
+                    task_exchange=options.taskexchange
                 )
             time.sleep(options.sleep_time)
 
@@ -383,11 +382,10 @@ def main():
         sys.exit(1)
 
     sqs_queue = connect_sqs(
-        task_exchange=options.taskexchange,
-        **get_aws_credentials(
-            options.region,
-            options.accesskey,
-            options.secretkey)
+        region_name=options.region,
+        aws_access_key_id=options.accesskey,
+        aws_secret_access_key=options.secretkey,
+        task_exchange=options.taskexchange
     )
     # consume our queue
     taskConsumer(sqs_queue, es).run()
diff --git a/mq/esworker_sns_sqs.py b/mq/esworker_sns_sqs.py
index 6404b450..5b44ee0d 100755
--- a/mq/esworker_sns_sqs.py
+++ b/mq/esworker_sns_sqs.py
@@ -24,7 +24,6 @@ from mozdef_util.utilities.logger import logger, initLogger
 from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../"))
-from mq.lib.aws import get_aws_credentials
 from mq.lib.plugins import sendEventToPlugins, registerPlugins
 from mq.lib.sqs import connect_sqs
 
@@ -192,11 +191,11 @@ def main():
         sys.exit(1)
 
     sqs_queue = connect_sqs(
-        task_exchange=options.taskexchange,
-        **get_aws_credentials(
-            options.region,
-            options.accesskey,
-            options.secretkey))
+        region_name=options.region,
+        aws_access_key_id=options.accesskey,
+        aws_secret_access_key=options.secretkey,
+        task_exchange=options.taskexchange
+    )
     # consume our queue
     taskConsumer(sqs_queue, es, options).run()
 
diff --git a/mq/esworker_sqs.py b/mq/esworker_sqs.py
index bb55e8e2..9de62d19 100755
--- a/mq/esworker_sqs.py
+++ b/mq/esworker_sqs.py
@@ -29,7 +29,6 @@ from mozdef_util.utilities.logger import logger, initLogger
 from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../"))
-from mq.lib.aws import get_aws_credentials
 from mq.lib.plugins import sendEventToPlugins, registerPlugins
 from mq.lib.sqs import connect_sqs
 
@@ -331,11 +330,11 @@ def main():
         sys.exit(1)
 
     sqs_queue = connect_sqs(
-        task_exchange=options.taskexchange,
-        **get_aws_credentials(
-            options.region,
-            options.accesskey,
-            options.secretkey))
+        region_name=options.region,
+        aws_access_key_id=options.accesskey,
+        aws_secret_access_key=options.secretkey,
+        task_exchange=options.taskexchange
+    )
     # consume our queue
     taskConsumer(sqs_queue, es).run()
 
diff --git a/mq/lib/aws.py b/mq/lib/aws.py
index 834ac8c0..aa37df07 100644
--- a/mq/lib/aws.py
+++ b/mq/lib/aws.py
@@ -4,14 +4,14 @@
 # Copyright (c) 2017 Mozilla Corporation
 
 
-def get_aws_credentials(region=None, access_key=None, secret_key=None, security_token=None):
+def get_aws_credentials(region_name=None, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None):
     result = {}
-    if region and region != '<add_region>':
-        result['region_name'] = region
-    if access_key and access_key != '<add_accesskey>':
-        result['aws_access_key_id'] = access_key
-    if secret_key and secret_key != '<add_secretkey>':
-        result['aws_secret_access_key'] = secret_key
-    if security_token:
-        result['security_token'] = security_token
+    if region_name and region_name != '<add_region>':
+        result['region_name'] = region_name
+    if aws_access_key_id and aws_access_key_id != '<add_accesskey>':
+        result['aws_access_key_id'] = aws_access_key_id
+    if aws_secret_access_key and aws_secret_access_key != '<add_secretkey>':
+        result['aws_secret_access_key'] = aws_secret_access_key
+    if aws_session_token:
+        result['aws_session_token'] = aws_session_token
     return result
diff --git a/mq/lib/sqs.py b/mq/lib/sqs.py
index 22f00073..04f6900b 100644
--- a/mq/lib/sqs.py
+++ b/mq/lib/sqs.py
@@ -1,18 +1,12 @@
 import boto3
+from .aws import get_aws_credentials
 
 
 def connect_sqs(region_name=None, aws_access_key_id=None,
                 aws_secret_access_key=None, task_exchange=None):
-    credentials = {}
-    if aws_access_key_id is not None:
-        credentials['aws_access_key_id'] = aws_access_key_id
-    if aws_secret_access_key is not None:
-        credentials['aws_secret_access_key'] = aws_secret_access_key
-
     sqs = boto3.resource(
         'sqs',
-        region_name=region_name,
-        **credentials
+        **get_aws_credentials(region_name, aws_access_key_id, aws_secret_access_key)
     )
     queue = sqs.get_queue_by_name(QueueName=task_exchange)
     return queue

From 1566a453787828a6189127914fbc0e9553ede90d Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Mon, 22 Jul 2019 17:08:50 -0500
Subject: [PATCH 38/58] Adds additional fields to the cloudtrail plugin to
 prevent field conflicts

---
 mq/plugins/cloudtrail.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py
index 3ffe65f0..b62f4c43 100644
--- a/mq/plugins/cloudtrail.py
+++ b/mq/plugins/cloudtrail.py
@@ -24,6 +24,8 @@ class message(object):
             'details.serviceeventdetails',
             'details.requestparameters.attribute',
             'details.requestparameters.bucketpolicy.statement.principal',
+            'details.requestparameters.bucketpolicy.statement.principal.service',
+            'details.requestparameters.bucketpolicy.statement.principal.aws',
             'details.requestparameters.callerreference',
             'details.requestparameters.description',
             'details.requestparameters.describeflowlogsrequest.filter.value',

From 3422c5dd0cb8ec4561756c719f4f18b693a34ad1 Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Mon, 22 Jul 2019 17:10:42 -0500
Subject: [PATCH 39/58] Adds additional fields to the cloudtrail plugin to
 prevent field conflicts

---
 mq/plugins/cloudtrail.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py
index b62f4c43..968f499a 100644
--- a/mq/plugins/cloudtrail.py
+++ b/mq/plugins/cloudtrail.py
@@ -23,7 +23,6 @@ class message(object):
             'details.apiversion',
             'details.serviceeventdetails',
             'details.requestparameters.attribute',
-            'details.requestparameters.bucketpolicy.statement.principal',
             'details.requestparameters.bucketpolicy.statement.principal.service',
             'details.requestparameters.bucketpolicy.statement.principal.aws',
             'details.requestparameters.callerreference',

From 581c597c111186c198c4ec39bb142cd3135dda3e Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 12:12:05 -0500
Subject: [PATCH 40/58] Use logger instead of writing to sys.stdout and
 sys.stderr

---
 alerts/lib/alerttask.py       |  3 ++-
 cron/auth02mozdef.py          |  8 +++-----
 rest/index.py                 | 13 ++++++-------
 rest/plugins/cymon.py         |  8 ++++----
 rest/plugins/fqdnblocklist.py | 31 ++++++++++++++++---------------
 rest/plugins/ipblocklist.py   | 35 ++++++++++++++++++-----------------
 rest/plugins/logincounts.py   |  4 ++--
 rest/plugins/vpc_blackhole.py | 23 ++++++++++++-----------
 rest/plugins/watchlist.py     | 17 +++++++++--------
 9 files changed, 72 insertions(+), 70 deletions(-)

diff --git a/alerts/lib/alerttask.py b/alerts/lib/alerttask.py
index 2c42b255..aee49eb3 100644
--- a/alerts/lib/alerttask.py
+++ b/alerts/lib/alerttask.py
@@ -20,6 +20,7 @@ from celery import Task
 from celery.utils.log import get_task_logger
 
 from mozdef_util.utilities.toUTC import toUTC
+from mozdef_util.utilities.logger import logger
 from mozdef_util.elasticsearch_client import ElasticsearchClient
 from mozdef_util.query_models import TermMatch, ExistsMatch
 
@@ -545,6 +546,6 @@ class AlertTask(Task):
             try:
                 json_obj = json.load(fd)
             except ValueError:
-                sys.stderr.write("FAILED to open the configuration file\n")
+                logger.error("FAILED to open the configuration file\n")
 
         return json_obj
diff --git a/cron/auth02mozdef.py b/cron/auth02mozdef.py
index 96ad32e2..81c4c6ea 100644
--- a/cron/auth02mozdef.py
+++ b/cron/auth02mozdef.py
@@ -16,6 +16,7 @@ import traceback
 import mozdef_client as mozdef
 
 from mozdef_util.utilities.dot_dict import DotDict
+from mozdef_util.utilities.logger import logger
 
 
 def fatal(msg):
@@ -23,9 +24,6 @@ def fatal(msg):
     sys.exit(1)
 
 
-def debug(msg):
-    sys.stderr.write("+++ {}\n".format(msg))
-
 
 # This is from https://auth0.com/docs/api/management/v2#!/Logs/get_logs
 # auth0 calls these events with an acronym and description
@@ -163,7 +161,7 @@ def process_msg(mozmsg, msg):
             details.success = True
     except KeyError:
         # New message type, check https://manage-dev.mozilla.auth0.com/docs/api/management/v2#!/Logs/get_logs for ex.
-        debug("New auth0 message type, please add support: {}".format(msg.type))
+        logger.error("New auth0 message type, please add support: {}".format(msg.type))
         details["eventname"] = msg.type
 
     # determine severity level
@@ -323,7 +321,7 @@ def main():
         config = DotDict(hjson.load(fd))
 
     if config is None:
-        print("No configuration file 'auth02mozdef.json' found.")
+        logger.error("No configuration file 'auth02mozdef.json' found.")
         sys.exit(1)
 
     headers = {"Authorization": "Bearer {}".format(config.auth0.token), "Accept": "application/json"}
diff --git a/rest/index.py b/rest/index.py
index 105174ea..cff419a6 100644
--- a/rest/index.py
+++ b/rest/index.py
@@ -11,7 +11,6 @@ import pynsive
 import random
 import re
 import requests
-import sys
 import socket
 import importlib
 from bottle import route, run, response, request, default_app, post
@@ -536,10 +535,10 @@ def kibanaDashboards():
             })
 
     except ElasticsearchInvalidIndex as e:
-        sys.stderr.write('Kibana dashboard index not found: {0}\n'.format(e))
+        logger.error('Kibana dashboard index not found: {0}\n'.format(e))
 
     except Exception as e:
-        sys.stderr.write('Kibana dashboard received error: {0}\n'.format(e))
+        logger.error('Kibana dashboard received error: {0}\n'.format(e))
 
     return json.dumps(resultsList)
 
@@ -555,7 +554,7 @@ def getWatchlist():
         # Log the entries we are removing to maintain an audit log
         expired = watchlistentries.find({'dateExpiring': {"$lte": datetime.utcnow() - timedelta(hours=1)}})
         for entry in expired:
-            sys.stdout.write('Deleting entry {0} from watchlist /n'.format(entry))
+            logger.debug('Deleting entry {0} from watchlist /n'.format(entry))
 
         # delete any that expired
         watchlistentries.delete_many({'dateExpiring': {"$lte": datetime.utcnow() - timedelta(hours=1)}})
@@ -578,7 +577,7 @@ def getWatchlist():
             )
         return json.dumps(WatchList)
     except ValueError as e:
-        sys.stderr.write('Exception {0} collecting watch list\n'.format(e))
+        logger.error('Exception {0} collecting watch list\n'.format(e))
 
 
 def getWhois(ipaddress):
@@ -591,7 +590,7 @@ def getWhois(ipaddress):
         whois['fqdn']=socket.getfqdn(str(netaddr.IPNetwork(ipaddress)[0]))
         return (json.dumps(whois))
     except Exception as e:
-        sys.stderr.write('Error looking up whois for {0}: {1}\n'.format(ipaddress, e))
+        logger.error('Error looking up whois for {0}: {1}\n'.format(ipaddress, e))
 
 
 def verisSummary(verisRegex=None):
@@ -617,7 +616,7 @@ def verisSummary(verisRegex=None):
         else:
             return json.dumps(list())
     except Exception as e:
-            sys.stderr.write('Exception while aggregating veris summary: {0}\n'.format(e))
+            logger.error('Exception while aggregating veris summary: {0}\n'.format(e))
 
 
 def initConfig():
diff --git a/rest/plugins/cymon.py b/rest/plugins/cymon.py
index ce84bad3..21d20c0e 100644
--- a/rest/plugins/cymon.py
+++ b/rest/plugins/cymon.py
@@ -6,9 +6,10 @@
 import requests
 import json
 import os
-import sys
 from configlib import getConfig, OptionParser
 
+from mozdef_util.utilities.logger import logger
+
 
 class message(object):
     def __init__(self):
@@ -41,7 +42,7 @@ class message(object):
         self.configfile = './plugins/cymon.conf'
         self.options = None
         if os.path.exists(self.configfile):
-            sys.stdout.write('found conf file {0}\n'.format(self.configfile))
+            logger.debug('found conf file {0}\n'.format(self.configfile))
             self.initConfiguration()
 
     def onMessage(self, request, response):
@@ -58,9 +59,8 @@ class message(object):
         except ValueError:
             response.status = 500
 
-        print(requestDict, requestDict.keys())
         if 'ipaddress' in requestDict:
-            url="https://cymon.io/api/nexus/v1/ip/{0}/events?combined=true&format=json".format(requestDict['ipaddress'])
+            url = "https://cymon.io/api/nexus/v1/ip/{0}/events?combined=true&format=json".format(requestDict['ipaddress'])
 
             # add the cymon api key?
             if self.options is not None:
diff --git a/rest/plugins/fqdnblocklist.py b/rest/plugins/fqdnblocklist.py
index 8eccdf3a..3fbbd0a1 100644
--- a/rest/plugins/fqdnblocklist.py
+++ b/rest/plugins/fqdnblocklist.py
@@ -7,11 +7,12 @@ import os
 import random
 import requests
 import re
-import sys
 from configlib import getConfig, OptionParser
 from datetime import datetime, timedelta
 from pymongo import MongoClient
 
+from mozdef_util.utilities.logger import logger
+
 
 def isFQDN(fqdn):
     try:
@@ -59,7 +60,7 @@ class message(object):
         self.configfile = './plugins/fqdnblocklist.conf'
         self.options = None
         if os.path.exists(self.configfile):
-            sys.stdout.write('found conf file {0}\n'.format(self.configfile))
+            logger.debug('found conf file {0}\n'.format(self.configfile))
         self.initConfiguration()
 
     def parse_fqdn_whitelist(self, fqdn_whitelist_location):
@@ -146,8 +147,8 @@ class message(object):
                     fqdnblock['creator'] = userID
                     fqdnblock['reference'] = referenceID
                     ref = fqdnblocklist.insert(fqdnblock)
-                    sys.stdout.write('{0} written to db\n'.format(ref))
-                    sys.stdout.write('%s: added to the fqdnblocklist table\n' % (fqdn))
+                    logger.debug('{0} written to db\n'.format(ref))
+                    logger.debug('%s: added to the fqdnblocklist table\n' % (fqdn))
 
                     # send to statuspage.io?
                     if len(self.options.statuspage_api_key) > 1:
@@ -170,17 +171,17 @@ class message(object):
                                                      headers=headers,
                                                      data=post_data)
                             if response.ok:
-                                sys.stdout.write('%s: notification sent to statuspage.io\n' % (fqdn))
+                                logger.info('%s: notification sent to statuspage.io\n' % (fqdn))
                             else:
-                                sys.stderr.write('%s: statuspage.io notification failed %s\n' % (fqdn, response.json()))
+                                logger.error('%s: statuspage.io notification failed %s\n' % (fqdn, response.json()))
                         except Exception as e:
-                            sys.stderr.write('Error while notifying statuspage.io for %s: %s\n' %(fqdn, e))
+                            logger.error('Error while notifying statuspage.io for %s: %s\n' % (fqdn, e))
                 else:
-                    sys.stderr.write('%s: is already present in the fqdnblocklist table\n' % (fqdn))
+                    logger.error('%s: is already present in the fqdnblocklist table\n' % (fqdn))
             else:
-                sys.stderr.write('%s: is not a valid fqdn\n' % (fqdn))
+                logger.error('%s: is not a valid fqdn\n' % (fqdn))
         except Exception as e:
-            sys.stderr.write('Error while blocking %s: %s\n' % (fqdn, e))
+            logger.error('Error while blocking %s: %s\n' % (fqdn, e))
 
     def onMessage(self, request, response):
         '''
@@ -224,7 +225,7 @@ class message(object):
                         for whitelist_fqdn in self.options.fqdnwhitelist:
                             if fqdn == whitelist_fqdn:
                                 whitelisted = True
-                                sys.stdout.write('{0} is whitelisted as part of {1}\n'.format(fqdn, whitelist_fqdn))
+                                logger.debug('{0} is whitelisted as part of {1}\n'.format(fqdn, whitelist_fqdn))
 
                         if not whitelisted:
                             self.blockFQDN(
@@ -234,15 +235,15 @@ class message(object):
                                 referenceID,
                                 userid
                             )
-                            sys.stdout.write('added {0} to blocklist\n'.format(fqdn))
+                            logger.debug('added {0} to blocklist\n'.format(fqdn))
                         else:
-                            sys.stdout.write('not adding {0} to blocklist, it was found in whitelist\n'.format(fqdn))
+                            logger.debug('not adding {0} to blocklist, it was found in whitelist\n'.format(fqdn))
                 else:
-                    sys.stdout.write('not adding {0} to blocklist, invalid fqdn\n'.format(fqdn))
+                    logger.error('not adding {0} to blocklist, invalid fqdn\n'.format(fqdn))
                     response.status = "400 invalid FQDN"
                     response.body = "invalid FQDN"
         except Exception as e:
-            sys.stderr.write('Error handling request.json %r \n' % (e))
+            logger.error('Error handling request.json %r \n' % (e))
             response.status = "500"
 
         return (request, response)
diff --git a/rest/plugins/ipblocklist.py b/rest/plugins/ipblocklist.py
index d8c292a9..799f9458 100644
--- a/rest/plugins/ipblocklist.py
+++ b/rest/plugins/ipblocklist.py
@@ -7,11 +7,12 @@ import netaddr
 import os
 import random
 import requests
-import sys
 from configlib import getConfig, OptionParser
 from datetime import datetime, timedelta
 from pymongo import MongoClient
 
+from mozdef_util.utilities.logger import logger
+
 
 def isIPv4(ip):
     try:
@@ -70,14 +71,14 @@ class message(object):
         self.configfile = './plugins/ipblocklist.conf'
         self.options = None
         if os.path.exists(self.configfile):
-            sys.stdout.write('found conf file {0}\n'.format(self.configfile))
+            logger.debug('found conf file {0}\n'.format(self.configfile))
         self.initConfiguration()
 
     def parse_network_whitelist(self, network_whitelist_location):
         networks = []
         with open(network_whitelist_location, "r") as text_file:
             for line in text_file:
-                line=line.strip().strip("'").strip('"')
+                line = line.strip().strip("'").strip('"')
                 if isIPv4(line) or isIPv6(line):
                     networks.append(line)
         return networks
@@ -140,11 +141,11 @@ class message(object):
                 ipblock = ipblocklist.find_one({'ipaddress': str(ipcidr)})
                 if ipblock is None:
                     # insert
-                    ipblock= dict()
+                    ipblock = dict()
                     ipblock['_id'] = genMeteorID()
                     # str to get the ip/cidr rather than netblock cidr.
                     # i.e. '1.2.3.4/24' not '1.2.3.0/24'
-                    ipblock['address']= str(ipcidr)
+                    ipblock['address'] = str(ipcidr)
                     ipblock['dateAdded'] = datetime.utcnow()
                     # Compute start and end dates
                     # default
@@ -166,8 +167,8 @@ class message(object):
                     ipblock['creator'] = userID
                     ipblock['reference'] = referenceID
                     ref = ipblocklist.insert(ipblock)
-                    sys.stdout.write('{0} written to db\n'.format(ref))
-                    sys.stdout.write('%s: added to the ipblocklist table\n' % (ipaddress))
+                    logger.debug('{0} written to db\n'.format(ref))
+                    logger.debug('%s: added to the ipblocklist table\n' % (ipaddress))
 
                     # send to statuspage.io?
                     if len(self.options.statuspage_api_key) > 1:
@@ -190,17 +191,17 @@ class message(object):
                                                      headers=headers,
                                                      data=post_data)
                             if response.ok:
-                                sys.stdout.write('%s: notification sent to statuspage.io\n' % (str(ipcidr)))
+                                logger.info('%s: notification sent to statuspage.io\n' % (str(ipcidr)))
                             else:
-                                sys.stderr.write('%s: statuspage.io notification failed %s\n' % (str(ipcidr),response.json()))
+                                logger.error('%s: statuspage.io notification failed %s\n' % (str(ipcidr), response.json()))
                         except Exception as e:
-                            sys.stderr.write('Error while notifying statuspage.io for %s: %s\n' %(str(ipcidr),e))
+                            logger.error('Error while notifying statuspage.io for %s: %s\n' % (str(ipcidr), e))
                 else:
-                    sys.stderr.write('%s: is already present in the ipblocklist table\n' % (str(ipcidr)))
+                    logger.error('%s: is already present in the ipblocklist table\n' % (str(ipcidr)))
             else:
-                sys.stderr.write('%s: is not a valid ip address\n' % (ipaddress))
+                logger.error('%s: is not a valid ip address\n' % (ipaddress))
         except Exception as e:
-            sys.stderr.write('Error while blocking %s: %s\n' % (ipaddress, e))
+            logger.error('Error while blocking %s: %s\n' % (ipaddress, e))
 
     def onMessage(self, request, response):
         '''
@@ -251,7 +252,7 @@ class message(object):
                             whitelist_network = netaddr.IPNetwork(whitelist_range)
                             if ipcidr in whitelist_network:
                                 whitelisted = True
-                                sys.stdout.write('{0} is whitelisted as part of {1}\n'.format(ipcidr, whitelist_network))
+                                logger.debug('{0} is whitelisted as part of {1}\n'.format(ipcidr, whitelist_network))
 
                         if not whitelisted:
                             self.blockIP(str(ipcidr),
@@ -259,10 +260,10 @@ class message(object):
                                          duration,
                                          referenceID,
                                          userid)
-                            sys.stdout.write('added {0} to blocklist\n'.format(ipaddress))
+                            logger.info('added {0} to blocklist\n'.format(ipaddress))
                         else:
-                            sys.stdout.write('not adding {0} to blocklist, it was found in whitelist\n'.format(ipaddress))
+                            logger.info('not adding {0} to blocklist, it was found in whitelist\n'.format(ipaddress))
         except Exception as e:
-            sys.stderr.write('Error handling request.json %r \n'% (e))
+            logger.error('Error handling request.json %r \n' % (e))
 
         return (request, response)
diff --git a/rest/plugins/logincounts.py b/rest/plugins/logincounts.py
index 62be468e..3d9de71d 100644
--- a/rest/plugins/logincounts.py
+++ b/rest/plugins/logincounts.py
@@ -5,12 +5,12 @@
 
 import json
 import os
-import sys
 from configlib import getConfig, OptionParser
 from datetime import datetime, timedelta
 from mozdef_util.elasticsearch_client import ElasticsearchClient
 from mozdef_util.query_models import SearchQuery, RangeMatch, Aggregation, ExistsMatch, PhraseMatch
 from mozdef_util.utilities.toUTC import toUTC
+from mozdef_util.utilities.logger import logger
 
 
 class message(object):
@@ -44,7 +44,7 @@ class message(object):
         self.configfile = './plugins/logincounts.conf'
         self.options = None
         if os.path.exists(self.configfile):
-            sys.stdout.write('found conf file {0}\n'.format(self.configfile))
+            logger.debug('found conf file {0}\n'.format(self.configfile))
         self.initConfiguration()
 
     def onMessage(self, request, response):
diff --git a/rest/plugins/vpc_blackhole.py b/rest/plugins/vpc_blackhole.py
index ec99e102..6328bd02 100644
--- a/rest/plugins/vpc_blackhole.py
+++ b/rest/plugins/vpc_blackhole.py
@@ -4,11 +4,12 @@
 # Copyright (c) 2014 Mozilla Corporation
 
 import os
-import sys
 import configparser
 import netaddr
 from boto3.session import Session
 
+from mozdef_util.utilities.logger import logger
+
 
 def isIPv4(ip):
     try:
@@ -63,7 +64,7 @@ class message(object):
         self.options = None
         self.multioptions = []
         if os.path.exists(self.configfile):
-            sys.stdout.write('found conf file {0}\n'.format(self.configfile))
+            logger.debug('found conf file {0}\n'.format(self.configfile))
             self.initConfiguration()
 
     def initConfiguration(self):
@@ -100,7 +101,7 @@ class message(object):
                         if len(routetable['Associations']) > 0:
                             if 'SubnetId' in routetable['Associations'][0]:
                                 subnet_id = routetable['Associations'][0]['SubnetId']
-                        sys.stdout.write('{0} {1}\n'.format(rt_id, vpc_id))
+                        logger.debug('{0} {1}\n'.format(rt_id, vpc_id))
 
                         response = client.describe_network_interfaces(
                             Filters=[
@@ -131,10 +132,10 @@ class message(object):
                             ]
                         )
 
-                        sys.stdout.write('{0}\n'.format(response))
+                        logger.debug('{0}\n'.format(response))
                         if len(response['NetworkInterfaces']) > 0:
                             bheni_id = response['NetworkInterfaces'][0]['NetworkInterfaceId']
-                            sys.stdout.write('{0} {1} {2}\n'.format(rt_id, vpc_id, bheni_id))
+                            logger.debug('{0} {1} {2}\n'.format(rt_id, vpc_id, bheni_id))
 
                             # get a handle to a route table associated with a netsec-private subnet
                             route_table = ec2.RouteTable(rt_id)
@@ -144,11 +145,11 @@ class message(object):
                                 NetworkInterfaceId=bheni_id,
                             )
                         else:
-                            sys.stdout.write('Skipping route table {0} in the VPC {1} - blackhole ENI could not be found\n'.format(rt_id, vpc_id))
+                            logger.debug('Skipping route table {0} in the VPC {1} - blackhole ENI could not be found\n'.format(rt_id, vpc_id))
                             continue
 
         except Exception as e:
-            sys.stderr.write('Error while creating a blackhole entry %s: %r\n' % (ipaddress, e))
+            logger.error('Error while creating a blackhole entry %s: %r\n' % (ipaddress, e))
 
     def onMessage(self, request, response):
         '''
@@ -172,20 +173,20 @@ class message(object):
 
             # are we configured?
             if self.multioptions is None:
-                sys.stderr.write("Customs server blockip requested but not configured\n")
+                logger.error("Customs server blockip requested but not configured\n")
                 sendToBHVPC = False
 
             if sendToBHVPC and ipaddress is not None:
                 # figure out the CIDR mask
                 if isIPv4(ipaddress) or isIPv6(ipaddress):
-                    ipcidr=netaddr.IPNetwork(ipaddress)
+                    ipcidr = netaddr.IPNetwork(ipaddress)
                     if not ipcidr.ip.is_loopback() \
                        and not ipcidr.ip.is_private() \
                        and not ipcidr.ip.is_reserved():
                         ipaddress = str(ipcidr.cidr)
                         self.addBlackholeEntry(ipaddress)
-                        sys.stdout.write('Blackholed {0}\n'.format(ipaddress))
+                        logger.info('Blackholed {0}\n'.format(ipaddress))
         except Exception as e:
-            sys.stderr.write('Error handling request.json %r \n'% (e))
+            logger.error('Error handling request.json %r \n' % (e))
 
         return (request, response)
diff --git a/rest/plugins/watchlist.py b/rest/plugins/watchlist.py
index 95555133..cf0a88ff 100644
--- a/rest/plugins/watchlist.py
+++ b/rest/plugins/watchlist.py
@@ -5,11 +5,12 @@
 
 import os
 import random
-import sys
 from configlib import getConfig, OptionParser
 from datetime import datetime, timedelta
 from pymongo import MongoClient
 
+from mozdef_util.utilities.logger import logger
+
 
 def genMeteorID():
     return('%024x' % random.randrange(16**24))
@@ -43,7 +44,7 @@ class message(object):
         self.configfile = './plugins/watchlist.conf'
         self.options = None
         if os.path.exists(self.configfile):
-            sys.stdout.write('found conf file {0}\n'.format(self.configfile))
+            logger.debug('found conf file {0}\n'.format(self.configfile))
         self.initConfiguration()
 
     def initConfiguration(self):
@@ -100,13 +101,13 @@ class message(object):
                 watched['creator']=userID
                 watched['reference']=referenceID
                 ref=watchlist.insert(watched)
-                sys.stdout.write('{0} written to db.\n'.format(ref))
-                sys.stdout.write('%s added to the watchlist table.\n' % (watchcontent))
+                logger.debug('{0} written to db.\n'.format(ref))
+                logger.debug('%s added to the watchlist table.\n' % (watchcontent))
 
             else:
-                sys.stderr.write('%s is already present in the watchlist table\n' % (str(watchcontent)))
+                logger.error('%s is already present in the watchlist table\n' % (str(watchcontent)))
         except Exception as e:
-            sys.stderr.write('Error while watching %s: %s\n' % (watchcontent, e))
+            logger.error('Error while watching %s: %s\n' % (watchcontent, e))
 
     def onMessage(self, request, response):
         '''
@@ -142,7 +143,7 @@ class message(object):
 
             if watchitem and watchcontent is not None:
                 if len(watchcontent) < 2:
-                    sys.stderr.write('{0} does not meet requirements. Not added. \n'.format(watchcontent))
+                    logger.error('{0} does not meet requirements. Not added. \n'.format(watchcontent))
 
                 else:
                     self.watchItem(str(watchcontent),
@@ -152,6 +153,6 @@ class message(object):
                                    userid)
 
         except Exception as e:
-            sys.stderr.write('Error handling request.json %r \n'% (e))
+            logger.error('Error handling request.json %r \n' % (e))
 
         return (request, response)

From ca037892eb5b0447af2c6399f0f839d34eb834aa Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 13:10:52 -0500
Subject: [PATCH 41/58] Fix parsing values for rest plugins

---
 rest/plugins/fqdnblocklist.py | 27 +++++++++++++--------------
 rest/plugins/ipblocklist.py   | 35 +++++++++++++++++------------------
 rest/plugins/vpc_blackhole.py | 11 +++++------
 rest/plugins/watchlist.py     | 28 +++++++++++++---------------
 4 files changed, 48 insertions(+), 53 deletions(-)

diff --git a/rest/plugins/fqdnblocklist.py b/rest/plugins/fqdnblocklist.py
index 3fbbd0a1..5bdff5e4 100644
--- a/rest/plugins/fqdnblocklist.py
+++ b/rest/plugins/fqdnblocklist.py
@@ -204,21 +204,20 @@ class message(object):
         # loop through the fields of the form
         # and fill in our values
         try:
-            for i in request.json:
+            for field in request.json:
                 # were we checked?
-                if self.name in i:
-                    blockfqdn = i.values()[0]
-                if 'fqdn' in i:
-                    fqdn = i.values()[0]
-                if 'duration' in i:
-                    duration = i.values()[0]
-                if 'comment' in i:
-                    comment = i.values()[0]
-                if 'referenceid' in i:
-                    referenceID = i.values()[0]
-                if 'userid' in i:
-                    userid = i.values()[0]
-
+                if self.name in field:
+                    blockfqdn = field[self.name]
+                if 'fqdn' in field:
+                    fqdn = field['fqdn']
+                if 'duration' in field:
+                    duration = field['duration']
+                if 'comment' in field:
+                    comment = field['comment']
+                if 'referenceid' in field:
+                    referenceID = field['referenceid']
+                if 'userid' in field:
+                    userid = field['userid']
             if blockfqdn and fqdn is not None:
                 if isFQDN(fqdn):
                         whitelisted = False
diff --git a/rest/plugins/ipblocklist.py b/rest/plugins/ipblocklist.py
index 799f9458..f949222f 100644
--- a/rest/plugins/ipblocklist.py
+++ b/rest/plugins/ipblocklist.py
@@ -10,7 +10,6 @@ import requests
 from configlib import getConfig, OptionParser
 from datetime import datetime, timedelta
 from pymongo import MongoClient
-
 from mozdef_util.utilities.logger import logger
 
 
@@ -191,7 +190,7 @@ class message(object):
                                                      headers=headers,
                                                      data=post_data)
                             if response.ok:
-                                logger.info('%s: notification sent to statuspage.io\n' % (str(ipcidr)))
+                                logger.debug('%s: notification sent to statuspage.io\n' % (str(ipcidr)))
                             else:
                                 logger.error('%s: statuspage.io notification failed %s\n' % (str(ipcidr), response.json()))
                         except Exception as e:
@@ -201,7 +200,7 @@ class message(object):
             else:
                 logger.error('%s: is not a valid ip address\n' % (ipaddress))
         except Exception as e:
-            logger.error('Error while blocking %s: %s\n' % (ipaddress, e))
+            logger.exception('Error while blocking %s: %s\n' % (ipaddress, e))
 
     def onMessage(self, request, response):
         '''
@@ -221,23 +220,23 @@ class message(object):
         userid = None
         blockip = False
 
-        # loop through the fields of the form
-        # and fill in our values
         try:
-            for i in request.json:
+            # loop through the fields of the form
+            # and fill in our values
+            for field in request.json:
                 # were we checked?
-                if self.name in i:
-                    blockip = i.values()[0]
-                if 'ipaddress' in i:
-                    ipaddress = i.values()[0]
-                if 'duration' in i:
-                    duration = i.values()[0]
-                if 'comment' in i:
-                    comment = i.values()[0]
-                if 'referenceid' in i:
-                    referenceID = i.values()[0]
-                if 'userid' in i:
-                    userid = i.values()[0]
+                if self.name in field:
+                    blockip = field[self.name]
+                if 'ipaddress' in field:
+                    ipaddress = field['ipaddress']
+                if 'duration' in field:
+                    duration = field['duration']
+                if 'comment' in field:
+                    comment = field['comment']
+                if 'referenceid' in field:
+                    referenceID = field['referenceid']
+                if 'userid' in field:
+                    userid = field['userid']
 
             if blockip and ipaddress is not None:
                 # figure out the CIDR mask
diff --git a/rest/plugins/vpc_blackhole.py b/rest/plugins/vpc_blackhole.py
index 6328bd02..c0f50b13 100644
--- a/rest/plugins/vpc_blackhole.py
+++ b/rest/plugins/vpc_blackhole.py
@@ -164,13 +164,12 @@ class message(object):
         # loop through the fields of the form
         # and fill in our values
         try:
-            for i in request.json:
+            for field in request.json:
                 # were we checked?
-                if self.name in i:
-                    sendToBHVPC = i.values()[0]
-                if 'ipaddress' in i:
-                    ipaddress = i.values()[0]
-
+                if self.name in field:
+                    sendToBHVPC = field[self.name]
+                if 'ipaddress' in field:
+                    ipaddress = field['ipaddress']
             # are we configured?
             if self.multioptions is None:
                 logger.error("Customs server blockip requested but not configured\n")
diff --git a/rest/plugins/watchlist.py b/rest/plugins/watchlist.py
index cf0a88ff..5a43d891 100644
--- a/rest/plugins/watchlist.py
+++ b/rest/plugins/watchlist.py
@@ -126,21 +126,19 @@ class message(object):
         # loop through the fields of the form
         # and fill in our values
         try:
-            for i in request.json:
-                # were we checked?
-                if self.name in i.keys():
-                    watchitem = i.values()[0]
-                if 'watchcontent' in i.keys():
-                    watchcontent = i.values()[0]
-                if 'duration' in i.keys():
-                    duration = i.values()[0]
-                if 'comment' in i.keys():
-                    comment = i.values()[0]
-                if 'referenceid' in i.keys():
-                    referenceID = i.values()[0]
-                if 'userid' in i.keys():
-                    userid = i.values()[0]
-
+            for field in request.json:
+                if self.name in field:
+                    watchitem = field[self.name]
+                if 'watchcontent' in field:
+                    watchcontent = field['watchcontent']
+                if 'duration' in field:
+                    duration = field['duration']
+                if 'comment' in field:
+                    comment = field['comment']
+                if 'referenceid' in field:
+                    referenceID = field['referenceid']
+                if 'userid' in field:
+                    userid = field['userid']
             if watchitem and watchcontent is not None:
                 if len(watchcontent) < 2:
                     logger.error('{0} does not meet requirements. Not added. \n'.format(watchcontent))

From 66536ed446464b997d80362dbc45f0d6b9cfc2b2 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 13:26:04 -0500
Subject: [PATCH 42/58] Fixup extra line in pep8 errors

---
 cron/auth02mozdef.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cron/auth02mozdef.py b/cron/auth02mozdef.py
index 81c4c6ea..06b9ab15 100644
--- a/cron/auth02mozdef.py
+++ b/cron/auth02mozdef.py
@@ -24,7 +24,6 @@ def fatal(msg):
     sys.exit(1)
 
 
-
 # This is from https://auth0.com/docs/api/management/v2#!/Logs/get_logs
 # auth0 calls these events with an acronym and description
 # The logs have the acronym, but not the description

From 8e9646fdd222c551fd0df8f1e933b1bc87bd22e6 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 14:46:03 -0500
Subject: [PATCH 43/58] Update ES version in cloudy mozdef to 6.7

---
 cloudy_mozdef/cloudformation/mozdef-es.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudy_mozdef/cloudformation/mozdef-es.yml b/cloudy_mozdef/cloudformation/mozdef-es.yml
index 96b0002b..b19501bb 100644
--- a/cloudy_mozdef/cloudformation/mozdef-es.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-es.yml
@@ -63,7 +63,7 @@ Resources:
         EBSEnabled: true
         VolumeType: gp2
         VolumeSize: !Ref BlockStoreSizeGB
-      ElasticsearchVersion: '6.8'
+      ElasticsearchVersion: '6.7'
       ElasticsearchClusterConfig:
         InstanceCount: !Ref ESInstanceCount
       AccessPolicies:

From b87ef1bca21331ac296f0841ff7d2a080888f1a2 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 14:52:53 -0500
Subject: [PATCH 44/58] Add aws parameters file to gitignore

---
 .gitignore | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 92392c31..c5cd705e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,8 +13,8 @@ alerts/generic_alerts
 /.project
 /data
 .vscode
-cloudy_mozdef/aws_parameters.json
-cloudy_mozdef/aws_parameters.sh
+cloudy_mozdef/aws_parameters.*.json
+cloudy_mozdef/aws_parameters.*.sh
 docs/source/_build
 docs/source/_static
 *.swp

From 570a09d737fbdad4b25578c3eb6c6fa1634be4ae Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 16:28:52 -0500
Subject: [PATCH 45/58] Add release notes for v3.1.1

---
 CHANGELOG.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index caabce25..0c096bed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 
+## [v3.1.1] - 2019-07-25
+
+### Added
+- Ability to get open indices in ElasticsearchClient
+- Documentation on installing dependencies on Mac OS X
+
+### Changed
+- AWS Managed Elasticsearch/Kibana version to 6.7
+
+### Fixed
+- Disk free/total in /about page shows at most 2 decimal places
+- Connections to SQS and S3 without access key and secret
+- Ability to block IPs and add to Watchlist
+
+
 ## [v3.1.0] - 2019-07-18
 
 ### Added
@@ -41,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Unnecessary and conflicting package dependencies from MozDef and mozdef_util
 - get_indices to include closed indices
 
+
 ## [v3.0.0] - 2019-07-08
 ### Added
 - Support for Python3

From 5b05cad40f6ca44bd8e07f7a757f5da76d5063de Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 16:32:34 -0500
Subject: [PATCH 46/58] Fix path for changelog in codeowners

---
 CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index 8ce07e26..61da2356 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -8,5 +8,5 @@
 
 # Entire set can review certain documentation files
 /README.md       @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug
-/CHANGELOG       @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug
+/CHANGELOG.md    @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug
 /docs/           @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug

From bebf3a41abcc19a9cc0bbea86bfa2a4bbed47bdf Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 16:49:58 -0500
Subject: [PATCH 47/58] Add v3.1.1 link at bottom of changelog

---
 CHANGELOG.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0c096bed..5d28d409 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -184,7 +184,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added checks on sending SQS messages to only accept intra-account messages
 - Improved docker performance and disk space requirements
 
-[Unreleased]: https://github.com/mozilla/MozDef/compare/v3.1.0...HEAD
+[Unreleased]: https://github.com/mozilla/MozDef/compare/v3.1.1...HEAD
+[v3.1.1]: https://github.com/mozilla/MozDef/compare/v3.1.0...v3.1.1
 [v3.1.0]: https://github.com/mozilla/MozDef/compare/v3.0.0...v3.1.0
 [v3.0.0]: https://github.com/mozilla/MozDef/compare/v2.0.1...v3.0.0
 [v2.0.1]: https://github.com/mozilla/MozDef/compare/v2.0.0...v2.0.1

From e8ea4927da2e248a57b36cec95810f97aecc5e18 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Thu, 25 Jul 2019 17:06:09 -0500
Subject: [PATCH 48/58] Update cloud formation url for v3.1.1

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 096b5b61..88c58289 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ The Mozilla Enterprise Defense Platform (MozDef) seeks to automate the security
 
 MozDef is in production at Mozilla where we are using it to process over 300 million events per day.
 
-[1]: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/v3.1.0/mozdef-parent.yml
+[1]: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/v3.1.1/mozdef-parent.yml
 
 ## Survey & Contacting us
 

From d6fcbec474c8794e841964f0831242de0cb3ea48 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Fri, 26 Jul 2019 23:26:31 -0500
Subject: [PATCH 49/58] Add example cloudtrail dashboard and visualizations

---
 .../resources/cloudtrail_eventname_pie_graph.json | 13 +++++++++++++
 .../resources/cloudtrail_eventname_table.json     | 13 +++++++++++++
 .../resources/cloudtrail_events_dashboard.json    | 15 +++++++++++++++
 .../resources/cloudtrail_events_line_graph.json   | 13 +++++++++++++
 .../files/resources/cloudtrail_events_map.json    | 13 +++++++++++++
 .../resources/cloudtrail_total_event_count.json   | 13 +++++++++++++
 .../resources/cloudtrail_user_identity_table.json | 13 +++++++++++++
 7 files changed, 93 insertions(+)
 create mode 100644 docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_pie_graph.json
 create mode 100644 docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_table.json
 create mode 100644 docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_dashboard.json
 create mode 100644 docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_line_graph.json
 create mode 100644 docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_map.json
 create mode 100644 docker/compose/mozdef_bootstrap/files/resources/cloudtrail_total_event_count.json
 create mode 100644 docker/compose/mozdef_bootstrap/files/resources/cloudtrail_user_identity_table.json

diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_pie_graph.json b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_pie_graph.json
new file mode 100644
index 00000000..73729295
--- /dev/null
+++ b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_pie_graph.json
@@ -0,0 +1,13 @@
+{
+  "visualization": {
+    "title": "Cloudtrail Eventname Pie-Graph",
+    "visState": "{\"title\":\"Cloudtrail Eventname Pie-Graph\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"details.sourceipaddress\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP Address\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"details.eventname\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"AWS Api Call\"}}]}",
+    "uiStateJSON": "{}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": "{\"index\":\"events-*\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+    }
+  },
+  "type": "visualization"
+}
\ No newline at end of file
diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_table.json b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_table.json
new file mode 100644
index 00000000..e9a6597f
--- /dev/null
+++ b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_table.json
@@ -0,0 +1,13 @@
+{
+  "visualization": {
+    "title": "Cloudtrail Eventname Table",
+    "visState": "{\"title\":\"Cloudtrail Eventname Table\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"details.eventname\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
+    "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": "{\"index\":\"events-*\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+    }
+  },
+  "type": "visualization"
+}
\ No newline at end of file
diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_dashboard.json b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_dashboard.json
new file mode 100644
index 00000000..cbf51464
--- /dev/null
+++ b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_dashboard.json
@@ -0,0 +1,15 @@
+{
+  "dashboard": {
+    "title": "Cloudtrail Events",
+    "hits": 0,
+    "description": "",
+    "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"gridData\":{\"x\":12,\"y\":16,\"w\":12,\"h\":18,\"i\":\"1\"},\"id\":\"cloudtrail_eventname_pie_graph\",\"panelIndex\":\"1\",\"title\":\"Event Names\",\"type\":\"visualization\",\"version\":\"6.8.0\"},{\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"gridData\":{\"x\":12,\"y\":0,\"w\":36,\"h\":16,\"i\":\"2\"},\"id\":\"cloudtrail_events_line_graph\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.8.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":12,\"h\":7,\"i\":\"3\"},\"id\":\"cloudtrail_total_event_count\",\"panelIndex\":\"3\",\"title\":\"# Events\",\"type\":\"visualization\",\"version\":\"6.8.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":16,\"w\":24,\"h\":18,\"i\":\"4\"},\"id\":\"cloudtrail_events_map\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.8.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":7,\"w\":12,\"h\":15,\"i\":\"5\"},\"id\":\"cloudtrail_user_identity_table\",\"panelIndex\":\"5\",\"type\":\"visualization\",\"version\":\"6.8.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":22,\"w\":12,\"h\":12,\"i\":\"8\"},\"id\":\"cloudtrail_eventname_table\",\"panelIndex\":\"8\",\"type\":\"visualization\",\"version\":\"6.8.0\"}]",
+    "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
+    "version": 1,
+    "timeRestore": false,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"e3d06450-9b8d-11e9-9b0e-b35568cb01e3\",\"key\":\"source\",\"negate\":false,\"params\":{\"query\":\"cloudtrail\",\"type\":\"phrase\"},\"type\":\"phrase\",\"value\":\"cloudtrail\"},\"query\":{\"match\":{\"source\":{\"query\":\"cloudtrail\",\"type\":\"phrase\"}}}}]}"
+    }
+  },
+  "type": "dashboard"
+}
\ No newline at end of file
diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_line_graph.json b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_line_graph.json
new file mode 100644
index 00000000..e142d7bc
--- /dev/null
+++ b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_line_graph.json
@@ -0,0 +1,13 @@
+{
+  "visualization": {
+    "title": "Cloudtrail Events Line Graph",
+    "visState": "{\"title\":\"Cloudtrail Events Line Graph\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"receivedtimestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\",\"mode\":\"quick\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"details.awsregion\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}}]}",
+    "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": "{\"index\":\"events-*\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+    }
+  },
+  "type": "visualization"
+}
\ No newline at end of file
diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_map.json b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_map.json
new file mode 100644
index 00000000..9a644034
--- /dev/null
+++ b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_map.json
@@ -0,0 +1,13 @@
+{
+  "visualization": {
+    "title": "Cloudtrail Events Map",
+    "visState": "{\"title\":\"Cloudtrail Events Map\",\"type\":\"tile_map\",\"params\":{\"colorSchema\":\"Yellow to Red\",\"mapType\":\"Shaded Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatClusterSize\":1.5,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[0,0],\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"selectedTmsLayer\":{\"origin\":\"elastic_maps_service\",\"id\":\"road_map\",\"minZoom\":0,\"maxZoom\":18,\"attribution\":\"<p>&#169; <a href=\\\"https://www.openstreetmap.org/copyright\\\">OpenStreetMap contributors</a>|<a href=\\\"https://openmaptiles.org\\\">OpenMapTiles</a>|<a href=\\\"https://www.maptiler.com\\\">MapTiler</a>|<a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p>&#10;\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"details.sourceipgeopoint\",\"autoPrecision\":true,\"isFilteredByCollar\":true,\"useGeocentroid\":true,\"mapZoom\":2,\"mapCenter\":{\"lon\":0,\"lat\":-0.17578097424708533},\"mapBounds\":{\"bottom_right\":{\"lat\":-83.94227191521858,\"lon\":282.30468750000006},\"top_left\":{\"lat\":83.9050579559856,\"lon\":-282.30468750000006}},\"precision\":2}}]}",
+    "uiStateJSON": "{}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": "{\"index\":\"events-*\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+    }
+  },
+  "type": "visualization"
+}
\ No newline at end of file
diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_total_event_count.json b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_total_event_count.json
new file mode 100644
index 00000000..48233222
--- /dev/null
+++ b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_total_event_count.json
@@ -0,0 +1,13 @@
+{
+  "visualization": {
+    "title": "Cloudtrail Total Event Count",
+    "visState": "{\"title\":\"Cloudtrail Total Event Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total\"}}]}",
+    "uiStateJSON": "{}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": "{\"index\":\"events-*\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+    }
+  },
+  "type": "visualization"
+}
diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_user_identity_table.json b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_user_identity_table.json
new file mode 100644
index 00000000..731e0b94
--- /dev/null
+++ b/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_user_identity_table.json
@@ -0,0 +1,13 @@
+{
+  "visualization": {
+    "title": "Cloudtrail User Identify Table",
+    "visState": "{\"title\":\"Cloudtrail User Identify Table\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"details.useridentity.arn\",\"size\":2,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\"}}]}",
+    "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": "{\"index\":\"events-*\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+    }
+  },
+  "type": "visualization"
+}
\ No newline at end of file

From bc7bafa52e33cbfcd71e0b9b8a91d96dadf5f618 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Fri, 26 Jul 2019 23:35:36 -0500
Subject: [PATCH 50/58] Increase wait time for kibana to start up

---
 docker/compose/mozdef_bootstrap/files/initial_setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/compose/mozdef_bootstrap/files/initial_setup.py b/docker/compose/mozdef_bootstrap/files/initial_setup.py
index 3b54d8d1..c080509f 100644
--- a/docker/compose/mozdef_bootstrap/files/initial_setup.py
+++ b/docker/compose/mozdef_bootstrap/files/initial_setup.py
@@ -125,7 +125,7 @@ if state_index_name not in all_indices:
     client.create_index(state_index_name, index_config=state_index_settings)
 
 # Wait for kibana service to get ready
-total_num_tries = 10
+total_num_tries = 40
 for attempt in range(total_num_tries):
     try:
         if requests.get(kibana_url).ok:
@@ -133,7 +133,7 @@ for attempt in range(total_num_tries):
     except Exception:
         pass
     print('Unable to connect to Elasticsearch...retrying')
-    sleep(5)
+    sleep(10)
 else:
     print('Cannot connect to Kibana after ' + str(total_num_tries) + ' tries, exiting script.')
     exit(1)

From 2579ea5f0461a775298568f6637dc598c337722a Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Fri, 26 Jul 2019 23:51:03 -0500
Subject: [PATCH 51/58] Allow kibana index creation

---
 docker/compose/elasticsearch/files/elasticsearch.yml   | 2 +-
 docker/compose/mozdef_bootstrap/files/initial_setup.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/compose/elasticsearch/files/elasticsearch.yml b/docker/compose/elasticsearch/files/elasticsearch.yml
index 104801f4..5f5fc56d 100644
--- a/docker/compose/elasticsearch/files/elasticsearch.yml
+++ b/docker/compose/elasticsearch/files/elasticsearch.yml
@@ -5,7 +5,7 @@ discovery.type: single-node
 action.destructive_requires_name: true
 
 # Disable auto creation unless these indexes
-action.auto_create_index:  .watches,.triggered_watches,.watcher-history-*
+action.auto_create_index:  .watches,.triggered_watches,.watcher-history-*,.kibana_*
 
 # Add these to prevent requiring a user/pass and termination of ES when looking for "ingest" assignments.
 # The watcher directive allows for the deletion of failed watcher indices as they sometimes get created with glitches.
diff --git a/docker/compose/mozdef_bootstrap/files/initial_setup.py b/docker/compose/mozdef_bootstrap/files/initial_setup.py
index c080509f..af0fea2c 100644
--- a/docker/compose/mozdef_bootstrap/files/initial_setup.py
+++ b/docker/compose/mozdef_bootstrap/files/initial_setup.py
@@ -125,15 +125,15 @@ if state_index_name not in all_indices:
     client.create_index(state_index_name, index_config=state_index_settings)
 
 # Wait for kibana service to get ready
-total_num_tries = 40
+total_num_tries = 20
 for attempt in range(total_num_tries):
     try:
-        if requests.get(kibana_url).ok:
+        if requests.get(kibana_url, allow_redirects=True):
             break
     except Exception:
         pass
-    print('Unable to connect to Elasticsearch...retrying')
-    sleep(10)
+    print('Unable to connect to Kibana ({0})...retrying'.format(kibana_url))
+    sleep(5)
 else:
     print('Cannot connect to Kibana after ' + str(total_num_tries) + ' tries, exiting script.')
     exit(1)

From e97f1908a3d6fdd3a9a3f45ef5fbae9598e07820 Mon Sep 17 00:00:00 2001
From: Gene Wood <gene_wood@cementhorizon.com>
Date: Mon, 29 Jul 2019 07:41:48 -0700
Subject: [PATCH 52/58] Fix documentation bullet list syntax

This more closely conforms to how best to punctuate bullet lists[1].
Thanks to @Siddharth1698 for noticing these inconsistencies and suggesting
a fix in #1400

[1]: https://www.businesswritingblog.com/business_writing/2012/01/punctuating-bullet-points-.html
---
 README.md                               |  2 +-
 docs/source/alert_development_guide.rst |  2 +-
 docs/source/benchmarking.rst            |  9 +++--
 docs/source/cicd.rst                    | 52 ++++++++++++-------------
 docs/source/cloud_deployment.rst        | 13 ++++---
 docs/source/overview.rst                | 36 ++++++++---------
 docs/source/usage.rst                   | 14 ++++---
 7 files changed, 67 insertions(+), 61 deletions(-)

diff --git a/README.md b/README.md
index 096b5b61..e5cf4328 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ The Mozilla Enterprise Defense Platform (MozDef) seeks to automate the security
 
 ## Goals:
 
-* Provide a platform for use by defenders to rapidly discover and respond to security incidents.
+* Provide a platform for use by defenders to rapidly discover and respond to security incidents
 * Automate interfaces to other systems like bunker, cymon, mig
 * Provide metrics for security events and incidents
 * Facilitate real-time collaboration amongst incident handlers
diff --git a/docs/source/alert_development_guide.rst b/docs/source/alert_development_guide.rst
index 503c21cb..4df309d2 100644
--- a/docs/source/alert_development_guide.rst
+++ b/docs/source/alert_development_guide.rst
@@ -53,7 +53,7 @@ At this point, begin development and periodically run your unit-tests locally wi
 Background on concepts
 ----------------------
 
-- Logs - These are individual log entries that are typically emitted from systems, like an Apache log
+- Logs - These are individual log entries that are typically emitted from systems, like an Apache log.
 - Events - The entry point into MozDef, a log parsed into JSON by some log shipper (syslog-ng, nxlog) or a native JSON data source like GuardDuty, CloudTrail, most SaaS systems, etc.
 - Alerts - These are either a 1:1 events to alerts (this thing happens and alert) or a M:1 events to alerts (N of these things happen and alert).
 
diff --git a/docs/source/benchmarking.rst b/docs/source/benchmarking.rst
index efd2f3b2..141452b8 100644
--- a/docs/source/benchmarking.rst
+++ b/docs/source/benchmarking.rst
@@ -27,7 +27,8 @@ insert_simple.js
 Usage: `node ./insert_simple.js <processes> <totalInserts> <host1> [host2] [host3] [...]`
 
   * `processes`: Number of processes to spawn
-  * `totalInserts`: Number of inserts to perform, please note after a certain number node will slow down. You want to have a lower number if you are in this case.
+  * `totalInserts`: Number of inserts to perform
+     * Please note after a certain number node will slow down. You want to have a lower number if you are in this case.
   * `host1`, `host2`, `host3`, etc: Elasticsearch hosts to which you want to send the HTTP requests
 
 insert_bulk.js
@@ -39,7 +40,8 @@ Usage: `node ./insert_bulk.js <processes> <insertsPerQuery> <totalInserts> <host
 
   * `processes`: Number of processes to spawn
   * `insertsPerQuery`: Number of logs per request
-  * `totalInserts`: Number of inserts to perform, please note after a certain number node will slow down. You want to have a lower number if you are in this case.
+  * `totalInserts`: Number of inserts to perform
+     * Please note after a certain number node will slow down. You want to have a lower number if you are in this case.
   * `host1`, `host2`, `host3`, etc: Elasticsearch hosts to which you want to send the HTTP requests
 
 search_all_fulltext.js
@@ -50,7 +52,8 @@ search_all_fulltext.js
 Usage: `node ./search_all_fulltext.js <processes> <totalSearches> <host1> [host2] [host3] [...]`
 
   * `processes`: Number of processes to spawn
-  * `totalSearches`: Number of search requests to perform, please note after a certain number node will slow down. You want to have a lower number if you are in this case.
+  * `totalSearches`: Number of search requests to perform
+     * Please note after a certain number node will slow down. You want to have a lower number if you are in this case.
   * `host1`, `host2`, `host3`, etc: Elasticsearch hosts to which you want to send the HTTP requests
 
 
diff --git a/docs/source/cicd.rst b/docs/source/cicd.rst
index 356b9fb3..fd90601e 100644
--- a/docs/source/cicd.rst
+++ b/docs/source/cicd.rst
@@ -19,34 +19,34 @@ The Test Sequence
 _________________
 
 * Travis CI creates webhooks when first setup which allow commits to the MozDef
-  GitHub repo to trigger Travis
+  GitHub repo to trigger Travis.
 * When a commit is made to MozDef, Travis CI follows the instructions in the
   `.travis.yml <https://github.com/mozilla/MozDef/blob/master/.travis.yml>`_
-  file
-* `.travis.yml` installs `docker-compose` in the `before_install` phase
-* in the `install` phase, Travis runs the
+  file.
+* `.travis.yml` installs `docker-compose` in the `before_install` phase.
+* In the `install` phase, Travis runs the
   `build-tests <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L88-L89>`_
   make target which calls `docker-compose build` on the
   `docker/compose/docker-compose-tests.yml`_ file which builds a few docker
-  containers to use for testing
-* in the `script` phase, Travis runs the
+  containers to use for testing.
+* In the `script` phase, Travis runs the
   `tests <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L52>`_
   make target which
 
   * calls the `build-tests` make target which again runs `docker-compose build`
-    on the `docker/compose/docker-compose-tests.yml`_ file
+    on the `docker/compose/docker-compose-tests.yml`_ file.
   * calls the
     `run-tests <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L67-L69>`_
-    make target which
+    make target which.
 
     * calls the
       `run-tests-resources <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L60-L61>`_
       make target which starts the docker
-      containers listed in `docker/compose/docker-compose-tests.yml`_
+      containers listed in `docker/compose/docker-compose-tests.yml`_.
     * runs `flake8` with the
       `.flake8 <https://github.com/mozilla/MozDef/blob/master/.flake8>`_
-      config file to check code style
-    * runs `py.test tests` which runs all the test cases
+      config file to check code style.
+    * runs `py.test tests` which runs all the test cases.
 
 AWS CodeBuild
 -------------
@@ -111,24 +111,24 @@ The Build Sequence
 __________________
 
 * A branch is merged into `master` in the GitHub repo or a version git tag is
-  applied to a commit
-* GitHub emits a webhook event to AWS CodeBuild indicating this
+  applied to a commit.
+* GitHub emits a webhook event to AWS CodeBuild indicating this.
 * AWS CodeBuild considers the Filter Groups configured to decide if the tag
   or branch warrants triggering a build. These Filter Groups are defined in
   the ``mozdef-cicd-codebuild.yml`` CloudFormation template. Assuming the tag
   or branch are acceptable, CodeBuild continues.
 * AWS CodeBuild reads the
   `buildspec.yml <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/buildspec.yml>`_
-  file to know what to do
+  file to know what to do.
 * The `install` phase of the `buildspec.yml` fetches
-  `packer <https://www.packer.io/>`_ and unzips it
+  `packer <https://www.packer.io/>`_ and unzips it.
 
   * `packer` is a tool that spawns an ec2 instance, provisions it, and renders
     an AWS Machine Image (AMI) from it.
 
 * The `build` phase of the `buildspec.yml` runs the
   `cloudy_mozdef/ci/deploy <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/deploy>`_
-  script in the AWS CodeBuild Ubuntu 14.04 environment
+  script in the AWS CodeBuild Ubuntu 14.04 environment.
 * The `deploy` script calls the
   `build-from-cwd <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L78-L79>`_
   target of the `Makefile` which calls `docker-compose build` on the
@@ -153,16 +153,16 @@ __________________
 
     * Uploads the local image that was just built by AWS CodeBuild to DockerHub.
       If the branch being built is `master` then the image is uploaded both with
-      a tag of `master` as well as with a tag of `latest`
+      a tag of `master` as well as with a tag of `latest`.
     * If the branch being built is from a version tag (e.g. `v1.2.3`) then the
-      image is uploaded with only that version tag applied
+      image is uploaded with only that version tag applied.
 * The `deploy` script next calls the
   `packer-build-github <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/cloudy_mozdef/Makefile#L34-L36>`_
   make target in the
   `cloudy_mozdef/Makefile <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/Makefile>`_
   which calls the
   `ci/pack_and_copy <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/pack_and_copy>`_
-  script which does the following steps
+  script which does the following steps.
 
   * Calls packer which launches an ec2 instance, executing a bunch of steps and
     and producing an AMI
@@ -179,19 +179,19 @@ __________________
 
   * Within this ec2 instance, packer `clones the MozDef GitHub repo and checks
     out the branch that triggered this build
-    <https://github.com/mozilla/MozDef/blob/c7a166f2e29dde8e5d71853a279fb0c47a48e1b2/cloudy_mozdef/packer/packer.json#L58-L60>`_
-  * packer replaces all instances of the word `latest` in the
+    <https://github.com/mozilla/MozDef/blob/c7a166f2e29dde8e5d71853a279fb0c47a48e1b2/cloudy_mozdef/packer/packer.json#L58-L60>`_.
+  * Packer replaces all instances of the word `latest` in the
     `docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
-    file with either the branch `master` or the version tag (e.g. `v1.2.3`)
-  * packer runs `docker-compose pull` on the
+    file with either the branch `master` or the version tag (e.g. `v1.2.3`).
+  * Packer runs `docker-compose pull` on the
     `docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
     file to pull down both the docker images that were just built by AWS
     CodeBuild and uploaded to Dockerhub as well as other non MozDef docker
-    images
+    images.
 
 * After packer completes executing the steps laid out in `packer.json` inside
   the ec2 instance, it generates an AMI from that instance and continues with
-  the copying, tagging and sharing steps described above
+  the copying, tagging and sharing steps described above.
 * Now back in the AWS CodeBuild environment, the `deploy` script continues by
   calling the
   `publish-versioned-templates <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/cloudy_mozdef/Makefile#L85-L87>`_
@@ -205,7 +205,7 @@ __________________
     CloudFormation template so that the template knows the AMI IDs of that
     specific branch of code.
   * uploads the CloudFormation templates to S3 in a directory either called
-    `master` or the tag version that was built (e.g. `v1.2.3`)
+    `master` or the tag version that was built (e.g. `v1.2.3`).
 
 .. _docker/compose/docker-compose-tests.yml: https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-tests.yml
 .. _tag-images: https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L109-L110
diff --git a/docs/source/cloud_deployment.rst b/docs/source/cloud_deployment.rst
index f69a20cd..8073ed7b 100644
--- a/docs/source/cloud_deployment.rst
+++ b/docs/source/cloud_deployment.rst
@@ -32,18 +32,19 @@ MozDef requires the following:
 - An OIDC Provider with ClientID, ClientSecret, and Discovery URL
 
   - Mozilla uses Auth0 but you can use any OIDC provider you like: Shibboleth,
-    KeyCloak, AWS Cognito, Okta, Ping (etc.)
+    KeyCloak, AWS Cognito, Okta, Ping (etc.).
   - You will need to configure the redirect URI of ``/redirect_uri`` as allowed in
-    your OIDC provider configuration
+    your OIDC provider configuration.
 - An ACM Certificate in the deployment region for your DNS name
-- A VPC with three public subnets available.
+- A VPC with three public subnets available
 
   - It is advised that this VPC be dedicated to MozDef or used solely for security automation.
   - The three public subnets must all be in different `availability zones <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#using-regions-availability-zones-describe>`_
-    and have a large enough number of IP addresses to accommodate the infrastructure
+    and have a large enough number of IP addresses to accommodate the infrastructure.
   - The VPC must have an `internet gateway <https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html>`_
-    enabled on it so that MozDef can reach the internet
-- An SQS queue receiving GuardDuty events.  At the time of writing this is not required but may be required in future.
+    enabled on it so that MozDef can reach the internet.
+- An SQS queue receiving GuardDuty events
+  - At the time of writing this is not required but may be required in future.
 
 
 Supported Regions
diff --git a/docs/source/overview.rst b/docs/source/overview.rst
index 8944f887..d236712b 100644
--- a/docs/source/overview.rst
+++ b/docs/source/overview.rst
@@ -20,7 +20,7 @@ Goals
 High level
 **********
 
-* Provide a platform for use by defenders to rapidly discover and respond to security incidents.
+* Provide a platform for use by defenders to rapidly discover and respond to security incidents
 * Automate interfaces to other systems like firewalls, cloud protections and anything that has an API
 * Provide metrics for security events and incidents
 * Facilitate real-time collaboration amongst incident handlers
@@ -31,25 +31,25 @@ Technical
 *********
 
 * Offer micro services that make up an Open Source Security Information and Event Management (SIEM)
-* Scalable, should be able to handle thousands of events per second, provide fast searching, alerting, correlation and handle interactions between teams of incident handlers.
+* Scalable, should be able to handle thousands of events per second, provide fast searching, alerting, correlation and handle interactions between teams of incident handlers
 
 MozDef aims to provide traditional SIEM functionality including:
 
-* Accepting events/logs from a variety of systems
-* Storing events/logs
-* Facilitating searches
-* Facilitating alerting
-* Facilitating log management (archiving,restoration)
+* Accepting events/logs from a variety of systems.
+* Storing events/logs.
+* Facilitating searches.
+* Facilitating alerting.
+* Facilitating log management (archiving,restoration).
 
 It is non-traditional in that it:
 
-* Accepts only JSON input
-* Provides you open access to your data
+* Accepts only JSON input.
+* Provides you open access to your data.
 * Integrates with a variety of log shippers including logstash, beaver, nxlog, syslog-ng and any shipper that can send JSON to either rabbit-mq or an HTTP(s) endpoint.
-* Provides easy integration to Cloud-based data sources such as cloudtrail or guard duty
-* Provides easy python plugins to manipulate your data in transit
-* Provides extensive plug-in opportunities to customize your event enrichment stream, your alert workflow, etc
-* Provides realtime access to teams of incident responders to allow each other to see their work simultaneously
+* Provides easy integration to Cloud-based data sources such as CloudTrail or GuardDuty.
+* Provides easy python plugins to manipulate your data in transit.
+* Provides extensive plug-in opportunities to customize your event enrichment stream, your alert workflow, etc.
+* Provides realtime access to teams of incident responders to allow each other to see their work simultaneously.
 
 
 Architecture
@@ -60,7 +60,7 @@ MozDef is based on open source technologies including:
 * RabbitMQ (message queue and amqp(s)-based log input)
 * uWSGI (supervisory control of python-based workers)
 * bottle.py (simple python interface for web request handling)
-* elasticsearch (scalable indexing and searching of JSON documents)
+* Elasticsearch (scalable indexing and searching of JSON documents)
 * Meteor (responsive framework for Node.js enabling real-time data sharing)
 * MongoDB (scalable data store, tightly integrated to Meteor)
 * VERIS from verizon (open source taxonomy of security incident categorizations)
@@ -74,11 +74,11 @@ Frontend processing
 
 Frontend processing for MozDef consists of receiving an event/log (in json) over HTTP(S), AMQP(S), or SQS
 doing data transformation including normalization, adding metadata, etc. and pushing
-the data to elasticsearch.
+the data to Elasticsearch.
 
 Internally MozDef uses RabbitMQ to queue events that are still to be processed.
 The diagram below shows the interactions between the python scripts (controlled by uWSGI),
-the RabbitMQ exchanges and elasticsearch indices.
+the RabbitMQ exchanges and Elasticsearch indices.
 
 .. image:: images/frontend_processing.png
 
@@ -95,7 +95,7 @@ Initial Release:
 * Facilitate replacing base SIEM functionality including log input, event management, search, alerts, basic correlations
 * Enhance the incident workflow UI to enable realtime collaboration
 * Enable basic plug-ins to the event input stream for meta data, additional parsing, categorization and basic machine learning
-* Support as many common event/log shippers as possible with repeatable recipies
+* Support as many common event/log shippers as possible with repeatable recipes
 * Base integration into Mozilla's defense mechanisms for automation
 * 3D visualizations of threat actors
 * Fine tuning of interactions between meteor, mongo, dc.js
@@ -106,7 +106,7 @@ Recently implemented:
 * Docker containers for each service
 * Updates to support recent (breaking) versions of Elasticsearch
 
-Future (join us!): 
+Future (join us!):
 
 * Correlation through machine learning, AI
 * Enhanced search for alerts, events, attackers within the MozDef UI
diff --git a/docs/source/usage.rst b/docs/source/usage.rst
index 7e2391c4..15764c25 100644
--- a/docs/source/usage.rst
+++ b/docs/source/usage.rst
@@ -131,11 +131,11 @@ Background
 
 Mozilla used CEF as a logging standard for compatibility with Arcsight and for standardization across systems. While CEF is an admirable standard, MozDef prefers JSON logging for the following reasons:
 
-* Every development language can create a JSON structure
-* JSON is easily parsed by computers/programs which are the primary consumer of logs
-* CEF is primarily used by Arcsight and rarely seen outside that platform and doesn't offer the extensibility of JSON
+* Every development language can create a JSON structure.
+* JSON is easily parsed by computers/programs which are the primary consumer of logs.
+* CEF is primarily used by Arcsight and rarely seen outside that platform and doesn't offer the extensibility of JSON.
 * A wide variety of log shippers (heka, logstash, fluentd, nxlog, beaver) are readily available to meet almost any need to transport logs as JSON.
-* JSON is already the standard for cloud platforms like amazon's cloudtrail logging
+* JSON is already the standard for cloud platforms like amazon's cloudtrail logging.
 
 Description
 ***********
@@ -288,8 +288,10 @@ Alerts are stored in the `alerts`_ folder.
 
 There are two types of alerts:
 
-* simple alerts that consider events on at a time. For example you may want to get an alert everytime a single LDAP modification is detected.
-* aggregation alerts allow you to aggregate events on the field of your choice. For example you may want to alert when more than 3 login attempts failed for the same username.
+* simple alerts that consider events on at a time
+   * For example you may want to get an alert everytime a single LDAP modification is detected.
+* aggregation alerts that allow you to aggregate events on the field of your choice
+   * For example you may want to alert when more than 3 login attempts failed for the same username.
 
 You'll find documented examples in the `alerts`_ folder.
 

From 1e3341ec1d46a76968f7638334c611fd3bf5a14a Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Mon, 29 Jul 2019 16:22:03 -0500
Subject: [PATCH 53/58] Updates css font var to standardize across all dark
 themes

---
 meteor/imports/themes/dark/mozdef.css         |  4 +-
 .../imports/themes/side_nav_dark/mozdef.css   | 46 +++++++++----------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/meteor/imports/themes/dark/mozdef.css b/meteor/imports/themes/dark/mozdef.css
index 64376c2a..e71f67ec 100644
--- a/meteor/imports/themes/dark/mozdef.css
+++ b/meteor/imports/themes/dark/mozdef.css
@@ -274,11 +274,11 @@ caption, legend {
 }
 
 .modal-header {
-    color: var(--font-focus);
+    color: var(--txt-secondary-color);
 }
   
   .modal-body {
-    color: var(--font-focus);
+    color: var(--txt-secondary-color);
 }
   
   .modal-body .row {
diff --git a/meteor/imports/themes/side_nav_dark/mozdef.css b/meteor/imports/themes/side_nav_dark/mozdef.css
index 8757dbab..88663590 100644
--- a/meteor/imports/themes/side_nav_dark/mozdef.css
+++ b/meteor/imports/themes/side_nav_dark/mozdef.css
@@ -21,8 +21,8 @@ Copyright (c) 2014 Mozilla Corporation
   --txt-shadow-color: #576d54;
   --arm-color: #e69006;
   --arm-focus-color: #d58512;
-  --font-main: #fff;
-  --font-focus: #000;
+  --txt-primary-color: #fff;
+  --txt-secondary-color: #000;
   --a-link-color: #a2a9b2;
 }
 
@@ -45,7 +45,7 @@ body{
     /*margin: 0;*/
     /*min-width: 990px;*/
     padding: 0;
-    color: var(--font-main);
+    color: var(--txt-primary-color);
     line-height: normal;
     text-align: left;
 }
@@ -56,12 +56,12 @@ body{
 
 /*mozdef custom */
 .upperwhite {
-    color: var(--font-main);
+    color: var(--txt-primary-color);
     text-transform: uppercase;
 }
 
 caption, legend {
-    color: var(--font-main);
+    color: var(--txt-primary-color);
 }
 
 .shadow {
@@ -69,7 +69,7 @@ caption, legend {
 }
 
 .ipaddress {
-  color: var(--font-main);
+  color: var(--txt-primary-color);
   font-style: normal;
   font-weight: normal;
 
@@ -101,7 +101,7 @@ caption, legend {
   opacity: .3;
   z-index: 2;
   font-size: 13px;
-  color: var(--font-main);
+  color: var(--txt-primary-color);
 }
 
 #bottom-toolbar:hover {
@@ -282,11 +282,11 @@ caption, legend {
 }
 
 .modal-header {
-  color: var(--font-focus);
+  color: var(--txt-secondary-color);
 }
 
 .modal-body {
-  color: var(--font-focus);
+  color: var(--txt-secondary-color);
 }
 
 .modal-body .row {
@@ -297,7 +297,7 @@ caption, legend {
 .btn {
   border: 1px outset;
   border-radius: 4px;
-  color: var(--font-main);
+  color: var(--txt-primary-color);
   background-color: var(--arm-color);
 }
 
@@ -305,7 +305,7 @@ caption, legend {
 .btn-warning:active,
 .btn-warning:hover,
 .open > .dropdown-toggle.btn-warning {
-  color: var(--font-focus);
+  color: var(--txt-secondary-color);
   background-color: var(--arm-focus-color);
   border-color: var(--arm-color);
 }
@@ -322,7 +322,7 @@ caption, legend {
 .btn-notice {
   border: 1px outset;
   border-radius: 4px;
-  color: var(--font-focus);
+  color: var(--txt-secondary-color);
   background-color: var(--ack-edit-color);
 }
 
@@ -330,13 +330,13 @@ caption, legend {
 .btn-notice:active,
 .btn-notice:hover,
 .open > .dropdown-toggle.btn-notice {
-  color: var(--font-main);
+  color: var(--txt-primary-color);
   background-color: var(--ack-edit-focus-color);
   border-color: var(--ack-edit-border-color);
 }
 
 .btn-notice:disabled, button[disabled] {
-  color: var(--font-main);
+  color: var(--txt-primary-color);
   background-color: var(--ack-edit-disabled-color);
   border-color: var(--ack-edit-border-color);
 }
@@ -344,12 +344,12 @@ caption, legend {
 .btn-generic {
   border: 1px outset;
   border-radius: 4px;
-  color: var(--font-focus);
+  color: var(--txt-secondary-color);
   background-color: var(--ack-edit-color);
 }
 
 .btn-generic:focus {
-  color: var(--font-main);
+  color: var(--txt-primary-color);
   background-color: #286090;
   border-color: #204d74;
 }
@@ -358,7 +358,7 @@ caption, legend {
 .btn-generic:active,
 .btn-genric:hover,
 .open > .dropdown-toggle.btn-generic {
-  color: var(--font-focus);
+  color: var(--txt-secondary-color);
   background-color: var(--ack-edit-focus-color);
   border-color: var(--ack-edit-border-color);
 }
@@ -394,11 +394,11 @@ input[type="search"] {
 .table-hover tbody tr:hover > th,
 .table-hover > tbody > tr:hover {
   background-color: #9a9ea5;
-  color: var(--font-focus);
+  color: var(--txt-secondary-color);
 }
 
 td{
-  color: var(--font-main);
+  color: var(--txt-primary-color);
 }
 
 .welcome {
@@ -407,7 +407,7 @@ td{
     width: 600px;
     margin-left: 25%;
     text-align: center;
-    color: var(--font-focus);
+    color: var(--txt-secondary-color);
     border: none;
     vertical-align: middle;
 }
@@ -468,17 +468,17 @@ td{
 }
 
 circle:hover{
-  fill: var(--font-main);
+  fill: var(--txt-primary-color);
 }
 
 .node {
-    stroke: var(--font-focus);
+    stroke: var(--txt-secondary-color);
     stroke-width: 1.5px;
 }
 
 .textlabel{
     stroke-width: .2px;
-    stroke: var(--font-focus);
+    stroke: var(--txt-secondary-color);
 }
 
 .vtagholders {

From 8f9ce1765724361eef0d4f679278ad8b45f9194d Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Mon, 29 Jul 2019 19:52:09 -0500
Subject: [PATCH 54/58] Resolves the tab-nav issue in themes.

---
 meteor/client/themes/side_nav_dark/menu.html  |  2 +-
 meteor/imports/themes/classic/mozdef.css      | 18 ++++--------
 meteor/imports/themes/dark/mozdef.css         | 21 +++++++++-----
 meteor/imports/themes/light/mozdef.css        | 29 +++++++++----------
 .../imports/themes/side_nav_dark/mozdef.css   |  9 ++++--
 5 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/meteor/client/themes/side_nav_dark/menu.html b/meteor/client/themes/side_nav_dark/menu.html
index c7587093..3308fcdf 100644
--- a/meteor/client/themes/side_nav_dark/menu.html
+++ b/meteor/client/themes/side_nav_dark/menu.html
@@ -7,7 +7,7 @@ Copyright (c) 2014 Mozilla Corporation
 
 
 <template name="side_nav_menu">
-    <div class="container itemcontainer">
+    <div class="container headercontainer">
       <nav class="main-menu">
       {{#if true }}
         <ul>
diff --git a/meteor/imports/themes/classic/mozdef.css b/meteor/imports/themes/classic/mozdef.css
index 42015c90..6fa72f15 100644
--- a/meteor/imports/themes/classic/mozdef.css
+++ b/meteor/imports/themes/classic/mozdef.css
@@ -287,19 +287,6 @@ caption, legend {
     color: var(--txt-secondary-color);
 }
 
-/* incident/investigation styles */
-.daterangepicker, .daterangepicker td {
-    color: var(--txt-secondary-color);
-}
-
-.tabcontent {
-    display: none;
-}
-
-.tabcontent.active {
-    display: block;
-}
-
 textarea {
     overflow: auto;
     vertical-align: top;
@@ -389,9 +376,14 @@ td {
 }
 
 .tabcontent {
+    display: none;
     margin-top: 20px;
 }
 
+.tabcontent.active {
+    display: block;
+}
+
 .tabnav a {
     color: rgb(173, 216, 230);
 }
diff --git a/meteor/imports/themes/dark/mozdef.css b/meteor/imports/themes/dark/mozdef.css
index e71f67ec..28724f27 100644
--- a/meteor/imports/themes/dark/mozdef.css
+++ b/meteor/imports/themes/dark/mozdef.css
@@ -343,12 +343,12 @@ td {
 
 
 .welcome {
-    height: 180px;
-    width: 600px;
-    margin-left: 25%;
-    text-align: center;
-    color: var(--txt-primary-color);
-    vertical-align: middle;
+  height: 180px;
+  width: 600px;
+  margin-left: 25%;
+  text-align: center;
+  color: var(--txt-primary-color);
+  vertical-align: middle;
 }
 
 .mozdeflogo{
@@ -357,11 +357,16 @@ td {
 }
 
 .tabcontent {
-    margin-top: 20px;
+  display: none;
+  margin-top: 20px;
+}
+
+.tabcontent.active {
+  display: block;
 }
 
 .tabnav a {
-    color: lightblue;
+  color: lightblue;
 }
 
 /* uncomment this login ui css to hide the local account/password signup options
diff --git a/meteor/imports/themes/light/mozdef.css b/meteor/imports/themes/light/mozdef.css
index 3270cbd1..6c7eb34a 100644
--- a/meteor/imports/themes/light/mozdef.css
+++ b/meteor/imports/themes/light/mozdef.css
@@ -260,14 +260,6 @@ caption, legend {
     color: var(--txt-primary-color);
 }
 
-.tabcontent {
-    display: none;
-}
-
-.tabcontent.active {
-    display: block;
-}
-
 textarea {
     overflow: auto;
     vertical-align: top;
@@ -343,12 +335,12 @@ td {
 }
 
 .welcome {
-    height: 180px;
-    width: 600px;
-    margin-left: 25%;
-    text-align: center;
-    color: var(--txt-primary-color);
-    vertical-align: middle;
+  height: 180px;
+  width: 600px;
+  margin-left: 25%;
+  text-align: center;
+  color: var(--txt-primary-color);
+  vertical-align: middle;
 }
 
 .mozdeflogo{
@@ -357,11 +349,16 @@ td {
 }
 
 .tabcontent {
-    margin-top: 20px;
+  display: none;
+  margin-top: 20px;
+}
+
+.tabcontent.active {
+  display: block;
 }
 
 .tabnav a {
-    color: lightblue;
+  color: lightblue;
 }
 
 /* don't float the 'create account' link*/
diff --git a/meteor/imports/themes/side_nav_dark/mozdef.css b/meteor/imports/themes/side_nav_dark/mozdef.css
index 88663590..894cef8b 100644
--- a/meteor/imports/themes/side_nav_dark/mozdef.css
+++ b/meteor/imports/themes/side_nav_dark/mozdef.css
@@ -416,8 +416,13 @@ td{
   width: 500px;
 }
 
-.tabcontent{
-    margin-top: 20px;
+.tabcontent {
+  display: none;
+  margin-top: 20px;
+}
+
+.tabcontent.active {
+  display: block;
 }
 
 .tabnav a{

From 33d1cff95db7005c7d369ac238a9cf02d9fc5e45 Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Tue, 30 Jul 2019 15:06:58 -0500
Subject: [PATCH 55/58] adjusts kibana5 to kibana for logging needs

---
 config/50-mozdef-filter.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/50-mozdef-filter.conf b/config/50-mozdef-filter.conf
index beacd146..9cc5f885 100644
--- a/config/50-mozdef-filter.conf
+++ b/config/50-mozdef-filter.conf
@@ -5,5 +5,5 @@ if $programname == 'eventtask-worker' then /var/log/mozdef/eventtask.log
 if $programname == 'alertactions-worker' then /var/log/mozdef/alertactions.log
 if $programname == 'mongod.3002' then /var/log/mozdef/mongo/meteor-mongo.log
 if $programname == 'mongod' then /var/log/mozdef/mongo/mongo.log
-if $programname == 'kibana5' then /var/log/mozdef/kibana.log
+if $programname == 'kibana' then /var/log/mozdef/kibana.log
 & stop

From 35037392dc149e2bf88428a3181771e392f8c35b Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Tue, 30 Jul 2019 15:22:08 -0500
Subject: [PATCH 56/58] adding new requestparam callerreference

---
 mq/plugins/cloudtrail.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py
index 968f499a..c9b44d34 100644
--- a/mq/plugins/cloudtrail.py
+++ b/mq/plugins/cloudtrail.py
@@ -29,6 +29,7 @@ class message(object):
             'details.requestparameters.description',
             'details.requestparameters.describeflowlogsrequest.filter.value',
             'details.requestparameters.disableapitermination',
+            'details.requestparameters.distributionconfig.callerreference'
             'details.requestparameters.domainname',
             'details.requestparameters.domainnames',
             'details.requestparameters.ebsoptimized',

From 64efeaf9cd9b5c0166f0b07be322aff64600068c Mon Sep 17 00:00:00 2001
From: Phrozyn <asmith@mozilla.com>
Date: Tue, 30 Jul 2019 15:24:49 -0500
Subject: [PATCH 57/58] adding missing comma

---
 mq/plugins/cloudtrail.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py
index c9b44d34..b90a0f3d 100644
--- a/mq/plugins/cloudtrail.py
+++ b/mq/plugins/cloudtrail.py
@@ -29,7 +29,7 @@ class message(object):
             'details.requestparameters.description',
             'details.requestparameters.describeflowlogsrequest.filter.value',
             'details.requestparameters.disableapitermination',
-            'details.requestparameters.distributionconfig.callerreference'
+            'details.requestparameters.distributionconfig.callerreference',
             'details.requestparameters.domainname',
             'details.requestparameters.domainnames',
             'details.requestparameters.ebsoptimized',

From 487696b26e59d536307b923bddf2b6f96c411511 Mon Sep 17 00:00:00 2001
From: Brandon Myers <bmyers@mozilla.com>
Date: Wed, 31 Jul 2019 12:40:32 -0500
Subject: [PATCH 58/58] Update ldap password spray summary

---
 alerts/ldap_password_spray.py            | 2 +-
 tests/alerts/test_ldap_password_spray.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/alerts/ldap_password_spray.py b/alerts/ldap_password_spray.py
index 1a1f22af..46c646f7 100644
--- a/alerts/ldap_password_spray.py
+++ b/alerts/ldap_password_spray.py
@@ -40,7 +40,7 @@ class AlertLdapPasswordSpray(AlertTask):
         # if len(email_list) == 0:
         #     return None
 
-        summary = 'Password Spray Attack in Progress from {0} targeting the following account(s): {1}'.format(
+        summary = 'LDAP Password Spray Attack in Progress from {0} targeting the following account(s): {1}'.format(
             aggreg['value'],
             ",".join(sorted(email_list))
         )
diff --git a/tests/alerts/test_ldap_password_spray.py b/tests/alerts/test_ldap_password_spray.py
index 8f780cbc..a21e9e69 100644
--- a/tests/alerts/test_ldap_password_spray.py
+++ b/tests/alerts/test_ldap_password_spray.py
@@ -38,14 +38,14 @@ class TestAlertLdapPasswordSpray(AlertTestSuite):
         "category": "ldap",
         "tags": ["ldap"],
         "severity": "WARNING",
-        "summary": "Password Spray Attack in Progress from 1.2.3.4 targeting the following account(s): jsmith@example.com",
+        "summary": "LDAP Password Spray Attack in Progress from 1.2.3.4 targeting the following account(s): jsmith@example.com",
     }
 
     # This alert is the expected result from this task against multiple matching events
     default_alert_aggregated = AlertTestSuite.copy(default_alert)
     default_alert_aggregated[
         "summary"
-    ] = "Password Spray Attack in Progress from 1.2.3.4 targeting the following account(s): jsmith@example.com"
+    ] = "LDAP Password Spray Attack in Progress from 1.2.3.4 targeting the following account(s): jsmith@example.com"
 
     test_cases = []