Merge remote-tracking branch 'origin/master' into breaking_es6_changes

CHANGELOG

@@ -5,6 +5,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 
+## [v1.39.0] - 2019-05-29
+### Added
+- Pagination of Web UI tables
+- Added support for SQS in replacement of Rabbitmq for alerts
+- Support for no_auth for watchlist
+- Cron script for closing indexes
+- Documentation on AlertActions
+
+### Changed
+- Removed dependency on '_type' field in Elasticsearch
+
+### Fixed
+- Slackbot reconnects successfully during network errors
+- Relative Kibana URLs now work correctly with protocol
+
+
 ## [v1.38.5] - 2019-04-09
 ### Added
 - Support for CSS themes
@@ -76,7 +92,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added checks on sending SQS messages to only accept intra-account messages
 - Improved docker performance and disk space requirements
 
-[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.5...HEAD
+[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.39.0...HEAD
+[v1.39.0]: https://github.com/mozilla/MozDef/compare/v1.38.5...v1.39.0
 [v1.38.5]: https://github.com/mozilla/MozDef/compare/v1.38.4...v1.38.5
 [v1.38.4]: https://github.com/mozilla/MozDef/compare/v1.38.3...v1.38.4
 [v1.38.3]: https://github.com/mozilla/MozDef/compare/v1.38.2...v1.38.3

CODEOWNERS

@@ -4,10 +4,9 @@
 *                @pwnbus @mpurzynski @Phrozyn @tristanweir
 
 # Allow review by Gene or Andrew for cloudy MozDef code
-/cloudy_mozdef/  @gene1wood @andrewkrug
+/cloudy_mozdef/  @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug
 
-# Anyone in EIS can review documentation
-# https://github.com/orgs/mozilla/teams/enterprise-information-security/members
-/README.md       @mozilla/enterprise-information-security
-/CHANGELOG       @mozilla/enterprise-information-security
-/docs/           @mozilla/enterprise-information-security
+# Entire set can review certain documentation files
+/README.md       @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug
+/CHANGELOG       @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug
+/docs/           @pwnbus @mpurzynski @Phrozyn @tristanweir @gene1wood @andrewkrug

Makefile

@@ -7,12 +7,13 @@
 ROOT_DIR	:= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 DKR_IMAGES	:= mozdef_alertactions mozdef_alerts mozdef_base mozdef_bootstrap mozdef_meteor mozdef_rest \
 		   mozdef_mq_worker mozdef_loginput mozdef_cron mozdef_elasticsearch mozdef_mongodb \
-		   mozdef_syslog mozdef_nginx mozdef_tester mozdef_rabbitmq mozdef_kibana
+		   mozdef_syslog mozdef_nginx mozdef_tester mozdef_rabbitmq mozdef_kibana mozdef_cognito_proxy
 BUILD_MODE	:= build  ## Pass `pull` in order to pull images instead of building them
 NAME		:= mozdef
 VERSION		:= 0.1
 BRANCH		:= master
 NO_CACHE	:= ## Pass `--no-cache` in order to disable Docker cache
+PARALLEL	:= --parallel
 GITHASH		:= latest  ## Pass `$(git rev-parse --short HEAD`) to tag docker hub images as latest git-hash instead
 TEST_CASE	:= tests  ## Run all (`tests`) or a specific test case (ex `tests/alerts/tests/alerts/test_proxy_drop_exfil_domains.py`)
 TMPDIR      := $(shell mktemp -d )
@@ -65,7 +66,7 @@ build: build-from-cwd
 
 .PHONY: build-from-cwd
 build-from-cwd:  ## Build local MozDef images (use make NO_CACHE=--no-cache build to disable caching)
-	docker-compose -f docker/compose/docker-compose.yml -p $(NAME) $(NO_CACHE) $(BUILD_MODE)
+	docker-compose -f docker/compose/docker-compose.yml -p $(NAME) $(BUILD_MODE) $(PARALLEL) $(NO_CACHE)
 
 .PHONY: build-from-github
 build-from-github:  ## Build local MozDef images from the github branch (use make NO_CACHE=--no-cache build to disable caching).

README.md

@@ -1,7 +1,21 @@
 [![Build Status](https://travis-ci.org/mozilla/MozDef.svg?branch=master)](https://travis-ci.org/mozilla/MozDef)
 [![Documentation Status](https://readthedocs.org/projects/mozdef/badge/?version=latest)](http://mozdef.readthedocs.io/en/latest/?badge=latest)
 
-# MozDef: Mozilla Enterprise Defense Platform
+# MozDef: ![LOGO](docs/source/images/moz_defense-platform_01.png)
+
+## Documentation:
+
+http://mozdef.readthedocs.org/en/latest/
+
+
+## Give MozDef a Try in AWS:
+
+The following button will launch the Mozilla Enterprise Defense Platform in your AWS account.
+
+**Warning:** Pressing the "Launch Stack" button and following through with the deployment will incur charges to your AWS account. 
+
+[![Launch MozDef](docs/source/images/cloudformation-launch-stack.png)][1]
+
 
 ## Why?
 
@@ -22,12 +36,4 @@ The Mozilla Enterprise Defense Platform (MozDef) seeks to automate the security
 
 MozDef is in production at Mozilla where we are using it to process over 300 million events per day.
 
-## Give MozDef a Try in AWS:
-
-[![Launch MozDef](docs/source/images/cloudformation-launch-stack.png)][1]
-
-## Documentation:
-
-http://mozdef.readthedocs.org/en/latest/
-
 [1]: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/v1.38.5/mozdef-parent.yml

alerts/celeryconfig.py

@@ -1,3 +1,4 @@
+import os
 from celery import Celery
 from importlib import import_module
 from lib.config import ALERTS, LOGGING, RABBITMQ
@@ -6,42 +7,47 @@ from logging.config import dictConfig
 # Alert files to include
 alerts_include = []
 for alert in ALERTS.keys():
-    alerts_include.append('.'.join((alert).split('.')[:-1]))
+    alerts_include.append(".".join((alert).split(".")[:-1]))
 alerts_include = list(set(alerts_include))
 
-BROKER_URL = 'amqp://{0}:{1}@{2}:{3}//'.format(
-    RABBITMQ['mquser'],
-    RABBITMQ['mqpassword'],
-    RABBITMQ['mqserver'],
-    RABBITMQ['mqport']
-)
+# XXX TBD this should get wrapped into an object that provides pyconfig
+if os.getenv("OPTIONS_MQPROTOCOL", "amqp") == "sqs":
+    BROKER_URL = "sqs://@"
+    BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1]}
+    CELERY_RESULT_BACKEND = None
+    alert_queue_name = os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4]
+    CELERY_DEFAULT_QUEUE = alert_queue_name
+    CELERY_QUEUES = {
+        alert_queue_name: {"exchange": alert_queue_name, "binding_key": alert_queue_name}
+    }
+else:
+    BROKER_URL = "amqp://{0}:{1}@{2}:{3}//".format(
+        RABBITMQ["mquser"], RABBITMQ["mqpassword"], RABBITMQ["mqserver"], RABBITMQ["mqport"]
+    )
+    CELERY_QUEUES = {
+        "celery-default": {"exchange": "celery-default", "binding_key": "celery-default"}
+    }
+    CELERY_DEFAULT_QUEUE = 'celery-default'
+
 CELERY_DISABLE_RATE_LIMITS = True
 CELERYD_CONCURRENCY = 1
 CELERY_IGNORE_RESULT = True
-CELERY_ACCEPT_CONTENT = ['json']
-CELERY_TASK_SERIALIZER = 'json'
-CELERY_DEFAULT_QUEUE = 'celery-default'
-CELERY_QUEUES = {
-    'celery-default': {
-        "exchange": "celery-default",
-        "binding_key": "celery-default",
-    },
-}
-
+CELERY_ACCEPT_CONTENT = ["json"]
+CELERY_TASK_SERIALIZER = "json"
 CELERYBEAT_SCHEDULE = {}
 
 # Register frequency of the tasks in the scheduler
 for alert in ALERTS.keys():
     CELERYBEAT_SCHEDULE[alert] = {
-        'task': alert,
-        'schedule': ALERTS[alert]['schedule'],
-        'options': {'queue': 'celery-default', "exchange": "celery-default"},
+        "task": alert,
+        "schedule": ALERTS[alert]["schedule"],
+        "options": {"queue": CELERY_DEFAULT_QUEUE, "exchange": CELERY_DEFAULT_QUEUE},
     }
     # add optional parameters:
-    if 'args' in ALERTS[alert]:
-        CELERYBEAT_SCHEDULE[alert]['args']=ALERTS[alert]['args']
-    if 'kwargs' in ALERTS[alert]:
-        CELERYBEAT_SCHEDULE[alert]['kwargs']=ALERTS[alert]['kwargs']
+    if "args" in ALERTS[alert]:
+        CELERYBEAT_SCHEDULE[alert]["args"] = ALERTS[alert]["args"]
+    if "kwargs" in ALERTS[alert]:
+        CELERYBEAT_SCHEDULE[alert]["kwargs"] = ALERTS[alert]["kwargs"]
 
 # Load logging config
 dictConfig(LOGGING)
@@ -52,27 +58,26 @@ dictConfig(LOGGING)
 # app.conf.update(
 #     CELERY_TASK_RESULT_EXPIRES=3600,
 # )
-app = Celery('alerts',
-             include=alerts_include)
-app.config_from_object('celeryconfig', force=True)
+app = Celery("alerts", include=alerts_include)
+app.config_from_object("celeryconfig", force=True)
 
 # As a result of celery 3 to celery 4, we need to dynamically
 # register all of the alert tasks specifically
 for alert_namespace in CELERYBEAT_SCHEDULE:
     try:
-        alert_tokens = alert_namespace.split('.')
+        alert_tokens = alert_namespace.split(".")
         alert_module_name = alert_tokens[0]
         alert_classname = alert_tokens[1]
         alert_module = import_module(alert_module_name)
         alert_class = getattr(alert_module, alert_classname)
         app.register_task(alert_class())
     except ImportError as e:
-        print "Error importing {}".format(alert_namespace)
-        print e
+        print("Error importing {}").format(alert_namespace)
+        print(e)
         pass
     except Exception as e:
-        print "Error addding alert"
-        print e
+        print("Error addding alert")
+        print(e)
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.start()

alerts/cloudtrail_excessive_describe.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+
+from lib.alerttask import AlertTask
+from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch
+
+
+class AlertCloudtrailExcessiveDescribe(AlertTask):
+    def main(self):
+        # Create a query to look back the last 20 minutes
+        search_query = SearchQuery(minutes=20)
+
+        # Add search terms to our query
+        search_query.add_must([
+            TermMatch('source', 'cloudtrail'),
+            TermMatch('details.eventverb', 'Describe'),
+            ExistsMatch('details.source')
+        ])
+
+        self.filtersManual(search_query)
+        # We aggregate on details.hostname which is the AWS service name
+        self.searchEventsAggregated('details.source', samplesLimit=2)
+        self.walkAggregations(threshold=50)
+
+    def onAggregation(self, aggreg):
+        category = 'access'
+        tags = ['cloudtrail']
+        severity = 'WARNING'
+        summary = "Excessive Describe calls on {0} ({1})".format(aggreg['value'], aggreg['count'])
+
+        # Create the alert object based on these properties
+        return self.createAlertDict(summary, category, tags, aggreg['events'], severity)

alerts/cloudtrail_logging_disabled.py

@@ -15,7 +15,7 @@ class AlertCloudtrailLoggingDisabled(AlertTask):
 
         search_query.add_must([
             TermMatch('source', 'cloudtrail'),
-            TermMatch('eventname', 'StopLogging')
+            TermMatch('details.eventname', 'StopLogging')
         ])
 
         search_query.add_must_not(TermMatch('errorcode', 'AccessDenied'))
@@ -29,6 +29,6 @@ class AlertCloudtrailLoggingDisabled(AlertTask):
         tags = ['cloudtrail', 'aws', 'cloudtrailpagerduty']
         severity = 'CRITICAL'
 
-        summary = 'Cloudtrail Logging Disabled: ' + event['_source']['requestparameters']['name']
+        summary = 'Cloudtrail Logging Disabled: ' + event['_source']['details']['requestparameters']['name']
 
         return self.createAlertDict(summary, category, tags, [event], severity)

alerts/cloudtrail_public_bucket.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+
+from lib.alerttask import AlertTask
+from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch
+
+
+class AlertCloudtrailPublicBucket(AlertTask):
+    def main(self):
+        search_query = SearchQuery(minutes=20)
+
+        search_query.add_must([
+            TermMatch('source', 'cloudtrail'),
+            TermMatch('details.eventname', 'PutBucketPolicy'),
+            ExistsMatch('details.requestparameters.bucketpolicy.statement.principal')
+        ])
+
+        self.filtersManual(search_query)
+        self.searchEventsSimple()
+        self.walkEvents()
+
+    # Set alert properties
+    def onEvent(self, event):
+        request_parameters = event['_source']['details']['requestparameters']
+        for statement in request_parameters['bucketpolicy']['statement']:
+            if statement['principal'] != '*':
+                return
+        category = 'access'
+        tags = ['cloudtrail']
+        severity = 'INFO'
+
+        bucket_name = 'Unknown'
+        if 'bucketname' in request_parameters:
+            bucket_name = request_parameters['bucketname']
+
+        summary = "The s3 bucket {0} is listed as public".format(bucket_name)
+        return self.createAlertDict(summary, category, tags, [event], severity)

alerts/deadman_generic.py

@@ -14,6 +14,10 @@ from mozdef_util.utilities.logger import logger
 class AlertDeadmanGeneric(DeadmanAlertTask):
 
     def main(self):
+        # We override the event indices to search for
+        # because our deadman alerts might look past 48 hours
+        self.event_indices = ["events-weekly"]
+
         self._config = self.parse_json_alert_config('deadman_generic.json')
         for alert_cfg in self._config['alerts']:
             try:

alerts/get_watchlist.conf

@@ -2,3 +2,4 @@
 # set the following to your protected endpoint api_url
 api_url = http://localhost:8081/getwatchlist
 jwt_secret = secret
+use_auth = false

alerts/get_watchlist.py

@@ -17,31 +17,27 @@ from mozdef_util.query_models import SearchQuery, QueryStringMatch
 
 class AlertWatchList(AlertTask):
     def main(self):
-        self.parse_config('get_watchlist.conf', ['api_url', 'jwt_secret'])
+        self.parse_config('get_watchlist.conf', ['api_url', 'jwt_secret', 'use_auth'])
 
-        jwt_token = JWTAuth(self.config.jwt_secret)
-        jwt_token.set_header_format('Bearer %s')
+        jwt_token = None
+        if self.config.use_auth.lower() != 'false':
+            jwt_token = JWTAuth(self.config.jwt_secret)
+            jwt_token.set_header_format('Bearer %s')
 
-        # Connect to rest api and grab response
         r = requests.get(self.config.api_url, auth=jwt_token)
-        status = r.status_code
-        index = 0
-        if status == 200:
-            status = r.status_code
+        # Connect to rest api and grab response
+        if r.ok:
             response = r.text
             terms_list = json.loads(response)
-            while index < len(terms_list):
-                term = terms_list[index]
-                term = '"{}"'.format(term)
+            for term in terms_list:
                 self.watchterm = term
-                index += 1
-                self.process_alert(term)
+                self.process_alert()
         else:
-            logger.error('The watchlist request failed. Status {0}.\n'.format(status))
+            logger.error('The watchlist request failed. Status {0}.\n'.format(r))
 
-    def process_alert(self, term):
+    def process_alert(self):
         search_query = SearchQuery(minutes=20)
-        content = QueryStringMatch(str(term))
+        content = QueryStringMatch(str(self.watchterm))
         search_query.add_must(content)
         self.filtersManual(search_query)
         self.searchEventsSimple()

alerts/lib/alerttask.py

@@ -32,9 +32,9 @@ from lib.alert_plugin_set import AlertPluginSet
 # determine most common values
 # in a list of dicts
 def keypaths(nested):
-    ''' return a list of nested dict key paths
+    """ return a list of nested dict key paths
         like: [u'_source', u'details', u'program']
-    '''
+    """
     for key, value in nested.iteritems():
         if isinstance(value, collections.Mapping):
             for subkey, subvalue in keypaths(value):
@@ -44,11 +44,11 @@ def keypaths(nested):
 
 
 def dictpath(path):
-    ''' split a string representing a
+    """ split a string representing a
         nested dictionary path key.subkey.subkey
-    '''
-    for i in path.split('.'):
-        yield '{0}'.format(i)
+    """
+    for i in path.split("."):
+        yield "{0}".format(i)
 
 
 def getValueByPath(input_dict, path_string):
@@ -58,7 +58,7 @@ def getValueByPath(input_dict, path_string):
         path_string can be key.subkey.subkey.subkey
     """
     return_data = input_dict
-    for chunk in path_string.split('.'):
+    for chunk in path_string.split("."):
         return_data = return_data.get(chunk, {})
     return return_data
 
@@ -99,66 +99,93 @@ class AlertTask(Task):
         # e.g. when aggregField is email: [{value:'evil@evil.com',count:1337,events:[...]}, ...]
         self.aggregations = None
 
-        self.log.debug('starting {0}'.format(self.alert_name))
+        self.log.debug("starting {0}".format(self.alert_name))
         self.log.debug(RABBITMQ)
         self.log.debug(ES)
 
         self._configureKombu()
         self._configureES()
 
-        # We want to select all event indices
-        # and filter out the window based on timestamp
-        # from the search query
-        self.event_indices = ['events-*']
+        self.event_indices = ['events', 'events-previous']
 
     def classname(self):
         return self.__class__.__name__
 
     @property
     def log(self):
-        return get_task_logger('%s.%s' % (__name__, self.alert_name))
+        return get_task_logger("%s.%s" % (__name__, self.alert_name))
 
     def parse_config(self, config_filename, config_keys):
         myparser = OptionParser()
         self.config = None
         (self.config, args) = myparser.parse_args([])
         for config_key in config_keys:
-            temp_value = getConfig(config_key, '', config_filename)
+            temp_value = getConfig(config_key, "", config_filename)
             setattr(self.config, config_key, temp_value)
 
+    def _discover_task_exchange(self):
+        """Use configuration information to understand the message queue protocol.
+        return: amqp, sqs
+        """
+        return getConfig("mqprotocol", "amqp", None)
+
+    def __build_conn_string(self):
+        exchange_protocol = self._discover_task_exchange()
+        if exchange_protocol == "amqp":
+            connString = "amqp://{0}:{1}@{2}:{3}//".format(
+                RABBITMQ["mquser"],
+                RABBITMQ["mqpassword"],
+                RABBITMQ["mqserver"],
+                RABBITMQ["mqport"],
+            )
+            return connString
+        elif exchange_protocol == "sqs":
+            connString = "sqs://{}".format(getConfig("alertSqsQueueUrl", None, None))
+            if connString:
+                connString = connString.replace('https://','')
+            return connString
+
     def _configureKombu(self):
         """
-        Configure kombu for rabbitmq
+        Configure kombu for amqp or sqs
         """
         try:
-            connString = 'amqp://{0}:{1}@{2}:{3}//'.format(
-                RABBITMQ['mquser'],
-                RABBITMQ['mqpassword'],
-                RABBITMQ['mqserver'],
-                RABBITMQ['mqport'])
+            connString = self.__build_conn_string()
             self.mqConn = kombu.Connection(connString)
-
-            self.alertExchange = kombu.Exchange(
-                name=RABBITMQ['alertexchange'],
-                type='topic',
-                durable=True)
-            self.alertExchange(self.mqConn).declare()
-            alertQueue = kombu.Queue(RABBITMQ['alertqueue'], exchange=self.alertExchange)
+            if connString.find('sqs') == 0:
+                self.mqConn.transport_options['region'] = os.getenv('DEFAULT_AWS_REGION', 'us-west-2')
+                self.alertExchange = kombu.Exchange(
+                    name=RABBITMQ["alertexchange"], type="topic", durable=True
+                )
+                self.alertExchange(self.mqConn).declare()
+                alertQueue = kombu.Queue(
+                    os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4], exchange=self.alertExchange
+                )
+            else:
+                self.alertExchange = kombu.Exchange(
+                    name=RABBITMQ["alertexchange"], type="topic", durable=True
+                )
+                self.alertExchange(self.mqConn).declare()
+                alertQueue = kombu.Queue(
+                    RABBITMQ["alertqueue"], exchange=self.alertExchange
+                )
             alertQueue(self.mqConn).declare()
-            self.mqproducer = self.mqConn.Producer(serializer='json')
-            self.log.debug('Kombu configured')
+            self.mqproducer = self.mqConn.Producer(serializer="json")
+            self.log.debug("Kombu configured")
         except Exception as e:
-            self.log.error('Exception while configuring kombu for alerts: {0}'.format(e))
+            self.log.error(
+                "Exception while configuring kombu for alerts: {0}".format(e)
+            )
 
     def _configureES(self):
         """
         Configure elasticsearch client
         """
         try:
-            self.es = ElasticsearchClient(ES['servers'])
-            self.log.debug('ES configured')
+            self.es = ElasticsearchClient(ES["servers"])
+            self.log.debug("ES configured")
         except Exception as e:
-            self.log.error('Exception while configuring ES for alerts: {0}'.format(e))
+            self.log.error("Exception while configuring ES for alerts: {0}".format(e))
 
     def mostCommon(self, listofdicts, dictkeypath):
         """
@@ -169,8 +196,8 @@ class AlertTask(Task):
             returned as a list of tuples
             [(value,count),(value,count)]
         """
-        inspectlist=list()
-        path=list(dictpath(dictkeypath))
+        inspectlist = list()
+        path = list(dictpath(dictkeypath))
         for i in listofdicts:
             for k in list(keypaths(i)):
                 if not (set(k[0]).symmetric_difference(path)):
@@ -180,33 +207,34 @@ class AlertTask(Task):
 
     def alertToMessageQueue(self, alertDict):
         """
-        Send alert to the rabbit message queue
+        Send alert to the kombu based message queue.  The default is rabbitmq.
         """
         try:
             # cherry pick items from the alertDict to send to the alerts messageQueue
-            mqAlert = dict(severity='INFO', category='')
-            if 'severity' in alertDict:
-                mqAlert['severity'] = alertDict['severity']
-            if 'category' in alertDict:
-                mqAlert['category'] = alertDict['category']
-            if 'utctimestamp' in alertDict:
-                mqAlert['utctimestamp'] = alertDict['utctimestamp']
-            if 'eventtimestamp' in alertDict:
-                mqAlert['eventtimestamp'] = alertDict['eventtimestamp']
-            mqAlert['summary'] = alertDict['summary']
+            mqAlert = dict(severity="INFO", category="")
+            if "severity" in alertDict:
+                mqAlert["severity"] = alertDict["severity"]
+            if "category" in alertDict:
+                mqAlert["category"] = alertDict["category"]
+            if "utctimestamp" in alertDict:
+                mqAlert["utctimestamp"] = alertDict["utctimestamp"]
+            if "eventtimestamp" in alertDict:
+                mqAlert["eventtimestamp"] = alertDict["eventtimestamp"]
+            mqAlert["summary"] = alertDict["summary"]
             self.log.debug(mqAlert)
             ensurePublish = self.mqConn.ensure(
-                self.mqproducer,
-                self.mqproducer.publish,
-                max_retries=10)
+                self.mqproducer, self.mqproducer.publish, max_retries=10
+            )
             ensurePublish(
                 alertDict,
                 exchange=self.alertExchange,
-                routing_key=RABBITMQ['alertqueue']
+                routing_key=RABBITMQ["alertqueue"],
             )
-            self.log.debug('alert sent to the alert queue')
+            self.log.debug("alert sent to the alert queue")
         except Exception as e:
-            self.log.error('Exception while sending alert to message queue: {0}'.format(e))
+            self.log.error(
+                "Exception while sending alert to message queue: {0}".format(e)
+            )
 
     def alertToES(self, alertDict):
         """
@@ -214,31 +242,33 @@ class AlertTask(Task):
         """
         try:
             res = self.es.save_alert(body=alertDict)
-            self.log.debug('alert sent to ES')
+            self.log.debug("alert sent to ES")
             self.log.debug(res)
             return res
         except Exception as e:
-            self.log.error('Exception while pushing alert to ES: {0}'.format(e))
+            self.log.error("Exception while pushing alert to ES: {0}".format(e))
 
     def tagBotNotify(self, alert):
         """
             Tag alert to be excluded based on severity
             If 'ircchannel' is set in an alert, we automatically notify mozdefbot
         """
-        alert['notify_mozdefbot'] = True
-        if alert['severity'] == 'NOTICE' or alert['severity'] == 'INFO':
-            alert['notify_mozdefbot'] = False
+        alert["notify_mozdefbot"] = True
+        if alert["severity"] == "NOTICE" or alert["severity"] == "INFO":
+            alert["notify_mozdefbot"] = False
 
         # If an alert sets specific ircchannel, then we should probably always notify in mozdefbot
-        if 'ircchannel' in alert and alert['ircchannel'] != '' and alert['ircchannel'] is not None:
-            alert['notify_mozdefbot'] = True
+        if (
+            "ircchannel" in alert and alert["ircchannel"] != "" and alert["ircchannel"] is not None
+        ):
+            alert["notify_mozdefbot"] = True
         return alert
 
     def saveAlertID(self, saved_alert):
         """
         Save alert to self so we can analyze it later
         """
-        self.alert_ids.append(saved_alert['_id'])
+        self.alert_ids.append(saved_alert["_id"])
 
     def filtersManual(self, query):
         """
@@ -248,7 +278,7 @@ class AlertTask(Task):
 
         """
         # Don't fire on already alerted events
-        duplicate_matcher = TermMatch('alert_names', self.determine_alert_classname())
+        duplicate_matcher = TermMatch("alert_names", self.determine_alert_classname())
         if duplicate_matcher not in query.must_not:
             query.add_must_not(duplicate_matcher)
 
@@ -258,7 +288,7 @@ class AlertTask(Task):
         alert_name = self.classname()
         # Allow alerts like the generic alerts (one python alert but represents many 'alerts')
         # can customize the alert name
-        if hasattr(self, 'custom_alert_name'):
+        if hasattr(self, "custom_alert_name"):
             alert_name = self.custom_alert_name
         return alert_name
 
@@ -274,10 +304,10 @@ class AlertTask(Task):
         """
         try:
             results = self.executeSearchEventsSimple()
-            self.events = results['hits']
+            self.events = results["hits"]
             self.log.debug(self.events)
         except Exception as e:
-            self.log.error('Error while searching events in ES: {0}'.format(e))
+            self.log.error("Error while searching events in ES: {0}".format(e))
 
     def searchEventsAggregated(self, aggregationPath, samplesLimit=5):
         """
@@ -301,37 +331,36 @@ class AlertTask(Task):
 
         try:
             esresults = self.main_query.execute(self.es, indices=self.event_indices)
-            results = esresults['hits']
+            results = esresults["hits"]
 
             # List of aggregation values that can be counted/summarized by Counter
             # Example: ['evil@evil.com','haxoor@noob.com', 'evil@evil.com'] for an email aggregField
             aggregationValues = []
             for r in results:
-                aggregationValues.append(getValueByPath(r['_source'], aggregationPath))
+                aggregationValues.append(getValueByPath(r["_source"], aggregationPath))
 
             # [{value:'evil@evil.com',count:1337,events:[...]}, ...]
             aggregationList = []
             for i in Counter(aggregationValues).most_common():
-                idict = {
-                    'value': i[0],
-                    'count': i[1],
-                    'events': [],
-                    'allevents': []
-                }
+                idict = {"value": i[0], "count": i[1], "events": [], "allevents": []}
                 for r in results:
-                    if getValueByPath(r['_source'], aggregationPath).encode('ascii', 'ignore') == i[0]:
+                    if (
+                        getValueByPath(r["_source"], aggregationPath).encode(
+                            "ascii", "ignore"
+                        ) == i[0]
+                    ):
                         # copy events detail into this aggregation up to our samples limit
-                        if len(idict['events']) < samplesLimit:
-                            idict['events'].append(r)
+                        if len(idict["events"]) < samplesLimit:
+                            idict["events"].append(r)
                         # also copy all events to a non-sampled list
                         # so we mark all events as alerted and don't re-alert
-                        idict['allevents'].append(r)
+                        idict["allevents"].append(r)
                 aggregationList.append(idict)
 
             self.aggregations = aggregationList
             self.log.debug(self.aggregations)
         except Exception as e:
-            self.log.error('Error while searching events in ES: {0}'.format(e))
+            self.log.error("Error while searching events in ES: {0}".format(e))
 
     def walkEvents(self, **kwargs):
         """
@@ -367,8 +396,8 @@ class AlertTask(Task):
         """
         if len(self.aggregations) > 0:
             for aggregation in self.aggregations:
-                if aggregation['count'] >= threshold:
-                    aggregation['config']=config
+                if aggregation["count"] >= threshold:
+                    aggregation["config"] = config
                     alert = self.onAggregation(aggregation)
                     if alert:
                         alert = self.tagBotNotify(alert)
@@ -378,7 +407,7 @@ class AlertTask(Task):
                         # even though we only sample events in the alert
                         # tag all events as alerted to avoid re-alerting
                         # on events we've already processed.
-                        self.tagEventsAlert(aggregation['allevents'], alertResultES)
+                        self.tagEventsAlert(aggregation["allevents"], alertResultES)
                         self.alertToMessageQueue(alert)
                         self.saveAlertID(alertResultES)
 
@@ -387,33 +416,45 @@ class AlertTask(Task):
         Send alerts through a plugin system
         """
 
-        plugin_dir = os.path.join(os.path.dirname(__file__), '../plugins')
+        plugin_dir = os.path.join(os.path.dirname(__file__), "../plugins")
         plugin_set = AlertPluginSet(plugin_dir, ALERT_PLUGINS)
         alertDict = plugin_set.run_plugins(alert)[0]
 
         return alertDict
 
-    def createAlertDict(self, summary, category, tags, events, severity='NOTICE', url=None, ircchannel=None):
+    def createAlertDict(
+        self,
+        summary,
+        category,
+        tags,
+        events,
+        severity="NOTICE",
+        url=None,
+        ircchannel=None,
+    ):
         """
         Create an alert dict
         """
         alert = {
-            'utctimestamp': toUTC(datetime.now()).isoformat(),
-            'severity': severity,
-            'summary': summary,
-            'category': category,
-            'tags': tags,
-            'events': [],
-            'ircchannel': ircchannel,
+            "utctimestamp": toUTC(datetime.now()).isoformat(),
+            "severity": severity,
+            "summary": summary,
+            "category": category,
+            "tags": tags,
+            "events": [],
+            "ircchannel": ircchannel,
         }
         if url:
-            alert['url'] = url
+            alert["url"] = url
 
         for e in events:
-            alert['events'].append({
-                'documentindex': e['_index'],
-                'documentsource': e['_source'],
-                'documentid': e['_id']})
+            alert["events"].append(
+                {
+                    "documentindex": e["_index"],
+                    "documentsource": e["_source"],
+                    "documentid": e["_id"],
+                }
+            )
         self.log.debug(alert)
         return alert
 
@@ -459,21 +500,23 @@ class AlertTask(Task):
         """
         try:
             for event in events:
-                if 'alerts' not in event['_source']:
-                    event['_source']['alerts'] = []
-                event['_source']['alerts'].append({
-                    'index': alertResultES['_index'],
-                    'id': alertResultES['_id']})
+                if "alerts" not in event["_source"]:
+                    event["_source"]["alerts"] = []
+                event["_source"]["alerts"].append(
+                    {"index": alertResultES["_index"], "id": alertResultES["_id"]}
+                )
 
-                if 'alert_names' not in event['_source']:
-                    event['_source']['alert_names'] = []
-                event['_source']['alert_names'].append(self.determine_alert_classname())
+                if "alert_names" not in event["_source"]:
+                    event["_source"]["alert_names"] = []
+                event["_source"]["alert_names"].append(self.determine_alert_classname())
 
-                self.es.save_event(index=event['_index'], body=event['_source'], doc_id=event['_id'])
+                self.es.save_event(
+                    index=event["_index"], body=event["_source"], doc_id=event["_id"]
+                )
             # We refresh here to ensure our changes to the events will show up for the next search query results
-            self.es.refresh(event['_index'])
+            self.es.refresh(event["_index"])
         except Exception as e:
-            self.log.error('Error while updating events in ES: {0}'.format(e))
+            self.log.error("Error while updating events in ES: {0}".format(e))
 
     def main(self):
         """
@@ -487,15 +530,15 @@ class AlertTask(Task):
         """
         try:
             self.main(*args, **kwargs)
-            self.log.debug('finished')
+            self.log.debug("finished")
         except Exception as e:
-            self.log.exception('Exception in main() method: {0}'.format(e))
+            self.log.exception("Exception in main() method: {0}".format(e))
 
     def parse_json_alert_config(self, config_file):
         """
         Helper function to parse an alert config file
         """
-        alert_dir = os.path.join(os.path.dirname(__file__), '..')
+        alert_dir = os.path.join(os.path.dirname(__file__), "..")
         config_file_path = os.path.abspath(os.path.join(alert_dir, config_file))
         json_obj = {}
         with open(config_file_path, "r") as fd:

alerts/lib/config.py

@@ -8,6 +8,7 @@
 from celery.schedules import crontab, timedelta
 import time
 import logging
+import os
 
 ALERTS = {
     # 'pythonfile.pythonclass':{'schedule': crontab(minute='*/10')},
@@ -31,8 +32,13 @@ RABBITMQ = {
     'alertqueue': 'mozdef.alert'
 }
 
+es_server = "http://localhost:9200"
+
+if os.getenv('OPTIONS_ESSERVERS'):
+    es_server = os.getenv('OPTIONS_ESSERVERS')
+
 ES = {
-    'servers': ['http://localhost:9200']
+    'servers': [es_server]
 }
 
 LOGGING = {

cloudy_mozdef/Makefile

@@ -7,6 +7,8 @@ AMI_MAP_TEMP_FILE	:= /tmp/mozdef-ami-map.txt
 DEV_STACK_PARAMS_FILENAME	:= aws_parameters.dev.json
 # For more information on the rationale behind the code in STACK_PARAMS see https://github.com/aws/aws-cli/issues/2429#issuecomment-441133480
 DEV_STACK_PARAMS	:= $(shell test -e $(DEV_STACK_PARAMS_FILENAME) && python -c 'import json,sys;f=open(sys.argv[1]);print(" ".join([",".join(["%s=\\\"%s\\\""%(k,v) for k,v in x.items()]) for x in json.load(f)]));f.close()' $(DEV_STACK_PARAMS_FILENAME))
+OIDC_CLIENT_ID		:= $(shell test -e $(DEV_STACK_PARAMS_FILENAME) && python -c 'import json,sys;f=open(sys.argv[1]);print(next((x["ParameterValue"] for x in json.load(f) if x["ParameterKey"]=="OIDCClientId"),""));f.close()' $(DEV_STACK_PARAMS_FILENAME))
+DOMAIN_NAME			:= $(shell test -e $(DEV_STACK_PARAMS_FILENAME) && python -c 'import json,sys;f=open(sys.argv[1]);print(next((x["ParameterValue"] for x in json.load(f) if x["ParameterKey"]=="DomainName"),""));f.close()' $(DEV_STACK_PARAMS_FILENAME))
 # MozDef uses a nested CF stack, the mozdef-parent.yml will tie all child stacks together and load them from S3
 # See also mozdef.infosec.mozilla.org bucket
 S3_DEV_BUCKET_NAME	:= mozdef.infosec.allizom.org
@@ -39,6 +41,7 @@ packer-build-github: ## Build the base AMI with packer
 create-dev-stack: test ## Create everything you need for a fresh new stack!
 	@export AWS_REGION=$(AWS_REGION)
 	@echo "Make sure you have an environment variable OIDC_CLIENT_SECRET set."
+	@test -n "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -n "$(OIDC_CLIENT_ID)" -o -z "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -z "$(OIDC_CLIENT_ID)"
 	aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
 	  --capabilities CAPABILITY_IAM \
 	  --parameters $(OIDC_CLIENT_SECRET_PARAM_ARG) \
@@ -53,6 +56,7 @@ create-dev-s3-bucket:
 .PHONY: update-dev-stack
 update-dev-stack: test ## Updates the nested stack on AWS
 	@export AWS_REGION=$(AWS_REGION)
+	@test -n "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -n "$(OIDC_CLIENT_ID)" -o -z "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -z "$(OIDC_CLIENT_ID)"
 	aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
 	  --capabilities CAPABILITY_IAM \
 	  --parameters $(OIDC_CLIENT_SECRET_PARAM_ARG) \
@@ -93,3 +97,7 @@ diff-dev-templates:
 .PHONY: diff-prod-templates
 diff-prod-templates:
 	tempdir=`mktemp --directory`; aws s3 sync $(S3_PROD_BUCKET_URI) "$$tempdir" --exclude="*" --include="*.yml"; diff --recursive --unified "$$tempdir" cloudformation; rm -rf "$$tempdir"
+
+.PHONY: bind-domain-name
+bind-domain-name:
+	ci/bind_domain_name "$(DOMAIN_NAME)" "$(STACK_NAME)"

cloudy_mozdef/ci/bind_domain_name

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+
+DOMAIN_NAME="$1"
+STACK_NAME="$2"
+test -n "${DOMAIN_NAME}" -a -n "${STACK_NAME}"
+
+DOMAIN_NAME_ZONE="${DOMAIN_NAME#*.}."
+ZONE_ID="$(aws route53 list-hosted-zones-by-name --dns-name ${DOMAIN_NAME_ZONE} --query "HostedZones[?Name == '${DOMAIN_NAME_ZONE}'].Id" --output text)"
+INSTANCE_STACK_ARN="$(aws cloudformation describe-stack-resource --stack-name ${STACK_NAME} --logical-resource-id MozDefInstance --query 'StackResourceDetail.PhysicalResourceId' --output text)"
+instance_stack_name_prefix="${INSTANCE_STACK_ARN##*:stack/}"
+INSTANCE_STACK_NAME="${instance_stack_name_prefix%%/*}"
+ELB_ARN="$(aws cloudformation describe-stack-resource --stack-name ${INSTANCE_STACK_NAME} --logical-resource-id MozDefElasticLoadBalancingV2LoadBalancer --query 'StackResourceDetail.PhysicalResourceId' --output text)"
+#elb_name_prefix="${ELB_ARN##*:loadbalancer/app/}"
+#ELB_NAME="${elb_name_prefix%%/*}"
+ELB_DNS_NAME=$(aws elbv2 describe-load-balancers --load-balancer-arns ${ELB_ARN} --query 'LoadBalancers[0].DNSName' --output text)
+ELB_HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --load-balancer-arns ${ELB_ARN} --query 'LoadBalancers[0].CanonicalHostedZoneId' --output text)
+CHANGE_BATCH=$(cat <<END_HEREDOC
+{"Changes": [{"Action": "UPSERT", "ResourceRecordSet": {"Name": "${DOMAIN_NAME}", "Type": "A", "AliasTarget": {"HostedZoneId": "${ELB_HOSTED_ZONE_ID}", "DNSName": "${ELB_DNS_NAME}", "EvaluateTargetHealth": true}}}]}
+END_HEREDOC
+)
+echo "Changing Route53 ${DOMAIN_NAME} to ${ELB_DNS_NAME} in ELB Hosted Zone ${ELB_HOSTED_ZONE_ID}"
+CHANGE_ID=$(aws route53 change-resource-record-sets --hosted-zone-id ${ZONE_ID} --change-batch "${CHANGE_BATCH}" --query 'ChangeInfo.Id' --output text)
+CHANGE_STATUS=$(aws route53 get-change --id ${CHANGE_ID} --query 'ChangeInfo.Status' --output text)
+echo "DNS Change is ${CHANGE_STATUS}"
+while [ "${CHANGE_STATUS}" = "PENDING" ]; do
+    echo -n "."
+    sleep 5
+    CHANGE_STATUS=$(aws route53 get-change --id ${CHANGE_ID} --query 'ChangeInfo.Status' --output text)
+done
+echo "DNS Change is ${CHANGE_STATUS}"

cloudy_mozdef/ci/deploy

@@ -25,6 +25,7 @@ echo "  Head Ref : ${CODEBUILD_WEBHOOK_HEAD_REF}"
 echo "  Trigger : ${CODEBUILD_WEBHOOK_TRIGGER}"
 
 if [[ "branch/master" == "${CODEBUILD_WEBHOOK_TRIGGER}" \
+        || "branch/reinforce2019" == "${CODEBUILD_WEBHOOK_TRIGGER}" \
         || "${CODEBUILD_WEBHOOK_TRIGGER}" =~ ^tag\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$ ]]; then
     echo "Codebuild is ubuntu 14.04. Installing packer in order to compensate. Someone should build a CI docker container \;)."
     wget -nv https://releases.hashicorp.com/packer/1.3.5/packer_1.3.5_linux_amd64.zip

cloudy_mozdef/ci/docker_tag_or_push

@@ -3,7 +3,7 @@
 action="${1}"
 branch="${2}"
 
-for name in mozdef_meteor mozdef_base mozdef_tester mozdef_mq_worker mozdef_kibana \
+for name in mozdef_cognito_proxy mozdef_meteor mozdef_base mozdef_tester mozdef_mq_worker mozdef_kibana \
         mozdef_syslog mozdef_cron mozdef_elasticsearch mozdef_loginput mozdef_mongodb \
         mozdef_bootstrap mozdef_alerts mozdef_nginx mozdef_alertactions mozdef_rabbitmq \
         mozdef_rest mozdef_base ; do

cloudy_mozdef/ci/publish_versioned_templates

@@ -19,10 +19,16 @@ sed '/# INSERT MAPPING HERE.*/{
     r '"${AMI_MAP_TEMP_FILE}"'
 }' cloudformation/mozdef-parent.yml > ${TMPDIR}/mozdef-parent.yml
 
+echo "Injecting the region AMI mapping into the mozdef-parent.yml CloudFormation template"
+sed '/# INSERT MAPPING HERE.*/{
+    s/# INSERT MAPPING HERE.*//g
+    r '"${AMI_MAP_TEMP_FILE}"'
+}' cloudformation/mozdef-parent-reinforce.yml > ${TMPDIR}/mozdef-parent-reinforce.yml
+
 echo "Uploading CloudFormation templates to S3 directory ${VERSIONED_BUCKET_URI}/"
 # Sync all .yml files except mozdef-parent.yml
 aws s3 sync cloudformation/ ${VERSIONED_BUCKET_URI} --exclude="*" --include="*.yml" --exclude="mozdef-parent.yml"
 # cp modified mozdef-parent.yml from TMPDIR to S3
 aws s3 cp ${TMPDIR}/mozdef-parent.yml ${VERSIONED_BUCKET_URI}/
-
+aws s3 cp ${TMPDIR}/mozdef-parent-reinforce.yml ${VERSIONED_BUCKET_URI}/
 rm -rf "${TMPDIR}"

cloudy_mozdef/cloudformation/base-iam.yml

@@ -14,6 +14,9 @@ Parameters:
   MozDefSQSQueueArn:
     Type: String
     Description: The ARN of the SQS queue that receives events destined for MozDef
+  MozDefAlertSqsQueueArn:
+    Type: String
+    Description: The ARN of the SQS queue that alerttask uses for taskExchange
   ESServiceLinkedRoleExists:
     Type: String
     Description: Does the ES Service Linked Role already exist. true or false
@@ -102,6 +105,26 @@ Resources:
               - sqs:ReceiveMessage
               - sqs:DeleteMessage
             Resource: !Ref MozDefSQSQueueArn
+          - Sid: AllowReadWriteFromAlertQueue
+            Effect: Allow
+            Action:
+              - "sqs:DeleteMessage"
+              - "sqs:GetQueueUrl"
+              - "sqs:ChangeMessageVisibility"
+              - "sqs:DeleteMessageBatch"
+              - "sqs:SendMessageBatch"
+              - "sqs:ReceiveMessage"
+              - "sqs:SendMessage"
+              - "sqs:GetQueueAttributes"
+              - "sqs:ChangeMessageVisibilityBatch"
+            Resource: !Ref MozDefAlertSqsQueueArn
+          - Sid: AllowListQueuesCelery
+            Effect: Allow
+            Action:
+              - "sqs:ListQueues"
+              - "sqs:GetQueueAttributes"
+              - "sqs:GetQueueUrl"
+            Resource: "*"
   MozDefIAMRole:
     Type: AWS::IAM::Role
     Properties:

cloudy_mozdef/cloudformation/mozdef-alert-developer.yml

@@ -0,0 +1,63 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Setup an alert writers environment for use with MozDef for AWS.  Note this is PoC only.
+Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+    Description: 'The VPC ID of the VPC to deploy in (Example : vpc-abcdef12)'
+  PublicSubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+    Description: 'A comma delimited list of public subnet IDs (Example: subnet-abcdef12,subnet-bcdef123)'
+  MozDefSecurityGroup:
+    Type: AWS::EC2::SecurityGroup::Id
+    Description: The security group the MozDef instance runs in.  This is needed to access ES.
+  ESUrl:
+    Type: String
+    Description: 'The location of elasticsearch deployed in managed-es.'
+Resources:
+    MozDefLayer:
+        Type: AWS::Lambda::LayerVersion
+        Properties:
+            LayerName: MozDef
+            Description: Mozilla Enterprise Defense Platform Dependencies
+            Content:
+                S3Bucket: public.us-west-2.security.allizom.org
+                S3Key: mozdef-lambda-layer/layer-latest.zip
+            CompatibleRuntimes:
+              - python2.7
+            LicenseInfo: 'MPL 2.0'
+    LambdalertIAMRole:
+      Type: AWS::IAM::Role
+      Properties:
+        AssumeRolePolicyDocument:
+          Version: 2012-10-17
+          Statement:
+            - Effect: Allow
+              Principal:
+                Service: lambda.amazonaws.com
+              Action: sts:AssumeRole
+        ManagedPolicyArns:
+          -  arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole 
+    AlertWritersEnv: 
+      Type: "AWS::Lambda::Function"
+      Properties: 
+        Handler: "lambdalert.handle"
+        Role: 
+          Fn::GetAtt: 
+            - "LambdalertIAMRole"
+            - "Arn"
+        Code: 
+          S3Bucket: public.us-west-2.security.allizom.org
+          S3Key: mozdef-lambda-layer/function-latest.zip
+        Layers:
+          - !Ref MozDefLayer
+        Environment:
+          Variables:
+            OPTIONS_ESSERVERS: !Ref ESUrl
+            OPTIONS_MQPROTOCOL: sqs
+        VpcConfig:
+          SecurityGroupIds: 
+            - !Ref MozDefSecurityGroup
+          SubnetIds: !Ref PublicSubnetIds
+        ReservedConcurrentExecutions: 1
+        Runtime: "python2.7"
+        Timeout: 120

cloudy_mozdef/cloudformation/mozdef-credential-leak.yml

@@ -0,0 +1,45 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Template to build out users for insecure account this is only used for training and testing.
+Parameters:
+  SNSReceiverArn:
+    Type: String
+    Description: The ARN of the SNS topic to post credentials to.  Note that this leaks credentials.
+Resources:
+  IAMUser1:
+    Type: AWS::IAM::User
+    Properties:
+      Path: /
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/AdministratorAccess
+  IAMUser1Keys:
+    Type: AWS::IAM::AccessKey
+    Properties:
+      UserName: !Ref 'IAMUser1'
+  CaptureSetupData:
+    Type: Custom::DataCapture
+    Version: '1.0'
+    Properties:
+      ServiceToken: !Ref SNSReceiverArn
+      AccessKey: !Ref 'IAMUser1Keys'
+      SecretAccessKey: !GetAtt 'IAMUser1Keys.SecretAccessKey'
+      lbURL: !GetAtt 'MyLoadBalancer.DNSName'
+      AccountID: !Ref 'AWS::AccountId'
+  MyLoadBalancer:
+    Type: AWS::ElasticLoadBalancing::LoadBalancer
+    Properties:
+      AvailabilityZones:
+        - us-west-2a
+      Listeners:
+        - LoadBalancerPort: '80'
+          InstancePort: '80'
+          Protocol: HTTP
+Outputs:
+  AccessKey:
+    Description: AccessKey
+    Value: !Ref 'IAMUser1Keys'
+  SecretAccessKey:
+    Description: SecretAccessKey
+    Value: !GetAtt 'IAMUser1Keys.SecretAccessKey'
+  LBUrl:
+    Description: lburl
+    Value: !GetAtt 'MyLoadBalancer.DNSName'

cloudy_mozdef/cloudformation/mozdef-instance.yml

@@ -48,16 +48,32 @@ Parameters:
     Type: String
     Description: The AWS ES Kibana URL with domain only
     Default: https://kibana.example.com/
+  OIDCAuthorizationEndpoint:
+    Type: String
+    Default: Unset
+    ConstraintDescription: A valid URL
+    Description: "The url of the authorization endpoint found for your oidc provider generall found on (Example: https://auth.example.com/.well-known/openid-configuration)"
   OIDCClientId:
     Type: String
+    Default: Unset
     Description: The client ID that your OIDC provider issues you for your Mozdef instance.
   OIDCClientSecret:
     Type: String
+    Default: Unset
     Description: The secret that your OIDC provider issues you for your Mozdef instance.
     NoEcho: true
-  OIDCDiscoveryURL:
+  OIDCIssuer:
     Type: String
-    Description: The URL of your OIDC provider's well-known discovery URL
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
+  OIDCTokenEndpoint:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
+  OIDCUserInfoEndpoint:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
   CloudTrailSQSNotificationQueueName:
     Type: String
     Description: The name of the SQS used for CloudTrail notifications.
@@ -67,6 +83,20 @@ Parameters:
   DomainName:
     Type: String
     Description: The fully qualified domain you'll be hosting MozDef at.
+  AlertQueueUrl:
+    Type: String
+    Description: The url of the alert queue kombu should use for taskExchange.
+Conditions:
+  OIDCEnabledCondition:
+    !Not [!Equals [!Ref OIDCClientId, Unset]]
+  OIDCNotEnabledCondition:
+    !Equals [!Ref OIDCClientId, Unset]
+  SSLEnabledCondition:
+    !And [!Not [!Equals [!Ref MozDefACMCertArn, Unset]], !Equals [!Ref OIDCClientId, Unset]]
+  SSLNotEnabledCondition:
+    !And [!Equals [!Ref MozDefACMCertArn, Unset], !Equals [!Ref OIDCClientId, Unset]]
+  SSLEnabledWithOidcCondition:
+    !And [!Not [!Equals [!Ref MozDefACMCertArn, Unset]], !Not [!Equals [!Ref OIDCClientId, Unset]]]
 Resources:
   MozDefElasticLoadBalancingV2TargetGroup:
     Type: AWS::ElasticLoadBalancingV2::TargetGroup
@@ -120,17 +150,22 @@ Resources:
             - content: |
                 # Cloudy MozDef env file as imported by docker compose.
                 # Drives the configuration of variables for a variety of containers.
+                OIDC_CLIENT_ID=${OIDCClientId}
                 OPTIONS_ESSERVERS=${ESURL}
+                # Private unauthed kibana url for accessing kibana API
                 OPTIONS_KIBANAURL=${KibanaURL}
                 # The OPTIONS_METEOR_KIBANAURL uses the reserved word "relative" which triggers MozDef
                 # to use relative links to Kibana : https://github.com/mozilla/MozDef/pull/956
-                OPTIONS_METEOR_KIBANAURL=https://relative:9090/_plugin/kibana/
+                OPTIONS_METEOR_KIBANAURL=http://relative:9090/_plugin/kibana/
                 OPTIONS_METEOR_ROOTURL=https://${DomainName}
+                METEOR_BACKEND=meteor:3000
+                ESBACKEND=${KibanaDomainOnlyURL}
+                # Disable certain web ui features
+                OPTIONS_REMOVE_FEATURES=ipblocklist,fqdnblocklist,logincounts,globe
                 # See https://github.com/mozilla-iam/mozilla.oidc.accessproxy/blob/master/README.md#setup
                 # Future support will be added for cognito backed authentication.
                 client_id=${OIDCClientId}
                 client_secret=${OIDCClientSecret}
-                discovery_url=${OIDCDiscoveryURL}
                 backend=http://meteor:3000
                 redirect_uri_path=/redirect_uri
                 httpsredir=no
@@ -148,6 +183,9 @@ Resources:
                 # Set thresholds for attack dataviz lower means more ogres
                 OPTIONS_IPV4ATTACKERHITCOUNT=5
                 OPTIONS_IPV4ATTACKERPREFIXLENGTH=24
+                OPTIONS_ALERTSQSQUEUEURL=${AlertQueueUrl}
+                OPTIONS_MQPROTOCOL=sqs
+                DEFAULT_AWS_REGION=${AWS::Region}
               path: /opt/mozdef/docker/compose/cloudy_mozdef.env
             - content: |
                 #!/usr/bin/env python
@@ -172,9 +210,11 @@ Resources:
                 ALERTS = {
                     'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')},
                     'unauth_ssh.AlertUnauthSSH': {'schedule': crontab(minute='*/1')},
+                    'get_watchlist.AlertWatchList': {'schedule': timedelta(minutes=1)},
                     'guard_duty_probe.AlertGuardDutyProbe': {'schedule': crontab(minute='*/1')},
                     'cloudtrail_logging_disabled.AlertCloudtrailLoggingDisabled': {'schedule': timedelta(minutes=1)},
-                    'cloudtrail_deadman.AlertCloudtrailDeadman': {'schedule': timedelta(hours=1)}
+                    'cloudtrail_deadman.AlertCloudtrailDeadman': {'schedule': timedelta(minutes=15)},
+                    'cloudtrail_public_bucket.AlertCloudtrailPublicBucket': {'schedule': timedelta(minutes=1)},
                 }
 
                 ALERT_PLUGINS = [
@@ -235,26 +275,13 @@ Resources:
 
                 logging.Formatter.converter = time.gmtime
               path: /opt/mozdef/docker/compose/mozdef_alerts/files/config.py
-            - content: |
-                # Cloudy MozDef env file as imported by docker compose for the kibana reverse proxy
-                # Cloudy MozDef uses managed elasticsearch and proxies connections to that using the LUA proxy
-                # This will support cognito backed authentication in the future
-                client_id=${OIDCClientId}
-                client_secret=${OIDCClientSecret}
-                discovery_url=${OIDCDiscoveryURL}
-                backend=${KibanaDomainOnlyURL}
-                redirect_uri_path=/redirect_uri
-                httpsredir=no
-                cookiename=seskibana
-              path: /opt/mozdef/docker/compose/cloudy_mozdef_kibana.env
             - content: |
                 # This configures the worker that pulls in CloudTrail logs
                 OPTIONS_TASKEXCHANGE=${CloudTrailSQSNotificationQueueName}
               path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env
             - content: |
-                # This is the additional worker reserved for future use
                 OPTIONS_TASKEXCHANGE=${MozDefSQSQueueName}
-              path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sns_sqs.env
+              path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
             - content: |
                 [Unit]
                 Description=Docker Compose container starter
@@ -273,12 +300,18 @@ Resources:
                 [Install]
                 WantedBy=multi-user.target
               path: /etc/systemd/system/docker-compose.service
+            - content: |
+                [options]
+                api_url = http://rest:8081/getwatchlist
+                jwt_secret = secret
+                use_auth = false
+              path: /opt/mozdef/docker/compose/mozdef_alerts/files/get_watchlist.conf
           runcmd:
             - echo RABBITMQ_PASSWORD=`python3 -c 'import secrets; s = secrets.token_hex(); print(s)'` > /opt/mozdef/docker/compose/rabbitmq.env
             - chmod --verbose 600 /opt/mozdef/docker/compose/rabbitmq.env
             - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef.env
             - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_kibana.env
-            - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_mq_sns_sqs.env
+            - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
             - mkdir --verbose --parents ${EFSMountPoint}
             - echo '*.* @@127.0.0.1:514' >> /etc/rsyslog.conf
             - systemctl enable rsyslog
@@ -289,6 +322,7 @@ Resources:
             - systemctl daemon-reload
             - systemctl enable docker-compose
             - systemctl start docker-compose
+            - sleep 10
             - cd /opt/mozdef && cat /opt/mozdef/docker/compose/rabbitmq.env | cut -d '=' -f2 | xargs docker-compose -f docker/compose/docker-compose.yml -p mozdef exec -T rabbitmq rabbitmqctl add_user mozdef
             - cd /opt/mozdef && docker-compose -f docker/compose/docker-compose.yml -p mozdef exec -T rabbitmq rabbitmqctl set_user_tags mozdef administrator
             - cd /opt/mozdef && docker-compose -f docker/compose/docker-compose.yml -p mozdef exec -T rabbitmq rabbitmqctl set_permissions -p / mozdef ".*" ".*" ".*"
@@ -331,8 +365,9 @@ Resources:
           Value: mozdef
         - Key: stack
           Value: !Ref AWS::StackName
-  MozDefElasticLoadBalancingV2ListenerKibanaHttps:
+  MozDefElasticLoadBalancingV2ListenerKibanaSSL:
     Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLEnabledCondition
     Properties:
       Certificates:
         - CertificateArn: !Ref MozDefACMCertArn
@@ -342,11 +377,24 @@ Resources:
           Ref: MozDefElasticLoadBalancingV2TargetGroupKibana
       LoadBalancerArn:
         Ref: MozDefElasticLoadBalancingV2LoadBalancer
-      Port: 9090
+      Port: 9443
       Protocol: HTTPS
       SslPolicy: ELBSecurityPolicy-2016-08
-  MozDefElasticLoadBalancingV2ListenerHttps:
+  MozDefElasticLoadBalancingV2ListenerKibana:
     Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLNotEnabledCondition
+    Properties:
+      DefaultActions:
+      - Type: forward
+        TargetGroupArn:
+          Ref: MozDefElasticLoadBalancingV2TargetGroupKibana
+      LoadBalancerArn:
+        Ref: MozDefElasticLoadBalancingV2LoadBalancer
+      Port: 9090
+      Protocol: HTTP
+  MozDefElasticLoadBalancingV2ListenerSSL:
+    Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLEnabledCondition
     Properties:
       Certificates:
         - CertificateArn: !Ref MozDefACMCertArn
@@ -359,8 +407,21 @@ Resources:
       Port: 443
       Protocol: HTTPS
       SslPolicy: ELBSecurityPolicy-2016-08
-  MozDefElasticLoadBalancingV2ListenerHttp:
+  MozDefElasticLoadBalancingV2ListenerNoSSL:
     Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLNotEnabledCondition
+    Properties:
+      DefaultActions:
+      - Type: forward
+        TargetGroupArn:
+          Ref: MozDefElasticLoadBalancingV2TargetGroup
+      LoadBalancerArn:
+        Ref: MozDefElasticLoadBalancingV2LoadBalancer
+      Port: 80
+      Protocol: HTTP
+  MozDefElasticLoadBalancingV2ListenerRedirectSSL:
+    Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLEnabledCondition
     Properties:
       DefaultActions:
       - Type: redirect
@@ -370,8 +431,105 @@ Resources:
           Protocol: HTTPS
           Query: '#{query}'
           Port: '443'
-          StatusCode: HTTP_301
+          StatusCode: HTTP_302
       LoadBalancerArn:
         Ref: MozDefElasticLoadBalancingV2LoadBalancer
       Port: 80
       Protocol: HTTP
+  MozDefElasticLoadBalancingV2ListenerRedirectSSLOIDC:
+    Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLEnabledWithOidcCondition
+    Properties:
+      DefaultActions:
+      - Type: redirect
+        RedirectConfig:
+          Host: '#{host}'
+          Path: '/#{path}'
+          Protocol: HTTPS
+          Query: '#{query}'
+          Port: '443'
+          StatusCode: HTTP_302
+      LoadBalancerArn:
+        Ref: MozDefElasticLoadBalancingV2LoadBalancer
+      Port: 80
+      Protocol: HTTP
+  MozDefElasticLoadBalancingV2ListenerSSLOIDC:
+    Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLEnabledWithOidcCondition
+    Properties:
+      Certificates:
+        - CertificateArn: !Ref MozDefACMCertArn
+      DefaultActions:
+      - Type: authenticate-oidc
+        AuthenticateOidcConfig:
+          AuthorizationEndpoint: !Ref OIDCAuthorizationEndpoint
+          ClientId: !Ref OIDCClientId
+          ClientSecret: !Ref OIDCClientSecret
+          Issuer: !Ref OIDCIssuer
+          OnUnauthenticatedRequest: authenticate
+          SessionCookieName: mozdefMeteor
+          TokenEndpoint: !Ref OIDCTokenEndpoint
+          UserInfoEndpoint: !Ref OIDCUserInfoEndpoint
+          Scope: "openid email"
+        Order: 1
+      - Type: forward
+        Order: 2
+        TargetGroupArn:
+          Ref: MozDefElasticLoadBalancingV2TargetGroup
+      LoadBalancerArn:
+        Ref: MozDefElasticLoadBalancingV2LoadBalancer
+      Port: 443
+      Protocol: HTTPS
+      SslPolicy: ELBSecurityPolicy-2016-08
+  MozDefElasticLoadBalancingV2ListenerRedirectKibanaSSLOIDC:
+    Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLEnabledWithOidcCondition
+    Properties:
+      Certificates:
+        - CertificateArn: !Ref MozDefACMCertArn
+      DefaultActions:
+      - Type: redirect
+        RedirectConfig:
+          Host: '#{host}'
+          Path: '/#{path}'
+          Protocol: HTTPS
+          Query: '#{query}'
+          Port: '9443'
+          StatusCode: HTTP_302
+      LoadBalancerArn:
+        Ref: MozDefElasticLoadBalancingV2LoadBalancer
+      Port: 9090
+      Protocol: HTTPS
+  MozDefElasticLoadBalancingV2ListenerSSLKibanaOIDC:
+    Type : AWS::ElasticLoadBalancingV2::Listener
+    Condition: SSLEnabledWithOidcCondition
+    Properties:
+      Certificates:
+        - CertificateArn: !Ref MozDefACMCertArn
+      DefaultActions:
+      - Type: authenticate-oidc
+        AuthenticateOidcConfig:
+          AuthorizationEndpoint: !Ref OIDCAuthorizationEndpoint
+          ClientId: !Ref OIDCClientId
+          ClientSecret: !Ref OIDCClientSecret
+          Issuer: !Ref OIDCIssuer
+          OnUnauthenticatedRequest: authenticate
+          Scope: openid profile email
+          SessionCookieName: mozdefMeteor
+          TokenEndpoint: !Ref OIDCTokenEndpoint
+          UserInfoEndpoint: !Ref OIDCUserInfoEndpoint
+        Order: 1
+      - Type: forward
+        Order: 2
+        TargetGroupArn:
+          Ref: MozDefElasticLoadBalancingV2TargetGroupKibana
+      LoadBalancerArn:
+        Ref: MozDefElasticLoadBalancingV2LoadBalancer
+      Port: 9443
+      Protocol: HTTPS
+      SslPolicy: ELBSecurityPolicy-2016-08
+Outputs:
+  LoadBalancerDNSName:
+    Description: The DNS name of the ALB hosting MozDef.  If using OIDC or SSL point your DNS at this.
+    Value:
+      Fn::GetAtt: [ MozDefElasticLoadBalancingV2LoadBalancer, DNSName ]

cloudy_mozdef/cloudformation/mozdef-parent-reinforce.yml

@@ -0,0 +1,366 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Deploy MozDef into AWS
+Metadata:
+  'AWS::CloudFormation::Interface':
+    ParameterGroups:
+    - Label:
+        default: EC2 Instance
+      Parameters:
+      - InstanceType
+      - KeyName
+      - SSHIngressCIDR
+    - Label:
+        default: Certificate
+      Parameters:
+        - ACMCertArn
+    - Label:
+        default: OIDC Configuration (optional)  If not set this will use basic auth.
+      Parameters:
+        - OIDCAuthorizationEndpoint
+        - OIDCClientId
+        - OIDCClientSecret
+        - OIDCIssuer
+        - OIDCTokenEndpoint
+        - OIDCUserInfoEndpoint
+    - Label:
+        default: Experimental Features
+      Parameters:
+        - LeakCredentialSNSArn
+    ParameterLabels:
+      InstanceType:
+        default: EC2 Instance Type
+      KeyName:
+        default: EC2 SSH Key Name
+      SSHIngressCIDR:
+        default: Inbound SSH allowed IP address CIDR
+      DomainName:
+        default: FQDN to host MozDef at
+      ACMCertArn:
+        default: ACM Certificate ARN
+      OIDCAuthorizationEndpoint:
+        default: OIDC authorization endpoint.
+      OIDCClientId:
+        default: OIDC Client ID.
+      OIDCClientSecret:
+        default: OIDC Client Secret.
+      OIDCIssuer:
+        default: OIDC issuer.
+      OIDCTokenEndpoint:
+        default: OIDC oauth token endpoint.
+      OIDCUserInfoEndpoint:
+        default: OIDC user info endpoint.
+      LeakCredentialSNSArn: Arn of the SNS topic to post admin creds to.
+Parameters:
+  InstanceType:
+    Type: String
+    Description: EC2 instance type, e.g. m1.small, m1.large, etc.
+    Default: m5.large
+  KeyName:
+    Type: AWS::EC2::KeyPair::KeyName
+    Description: Name of an existing EC2 KeyPair to enable SSH access to the web server
+  SSHIngressCIDR:
+    Type: String
+    AllowedPattern: '^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$'
+    ConstraintDescription: A valid CIDR (e.g. 203.0.113.0/24)
+    Description: The CIDR of IP addresses from which to allow inbound SSH connections
+  DomainName:
+    Type: String
+    Description: The fully qualified DNS name you will host CloudyMozDef at.
+    Default: cloudymozdef.security.allizom.org
+  ACMCertArn:
+    Type: String
+    Default: Unset
+    Description: "The ARN of your pre-issued ACM cert. (Example: arn:aws:acm:us-west-2:123456789012:certificate/abcdef01-2345-6789-abcd-ef0123456789)"
+  OIDCAuthorizationEndpoint:
+    Type: String
+    Default: Unset
+    ConstraintDescription: A valid URL
+    Description: "The url of the authorization endpoint found for your oidc provider generall found on (Example: https://auth.example.com/.well-known/openid-configuration)"
+  OIDCClientId:
+    Type: String
+    Default: Unset
+    Description: The client ID that your OIDC provider issues you for your Mozdef instance.
+  OIDCClientSecret:
+    Type: String
+    Default: Unset
+    Description: The secret that your OIDC provider issues you for your Mozdef instance.
+    NoEcho: true
+  OIDCIssuer:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
+  OIDCTokenEndpoint:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
+  OIDCUserInfoEndpoint:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
+  LeakCredentialSNSArn:
+    Type: String
+    Description: The arn of the sns topic to post a credential back to from the account.  Do not use unless you are deploying this for reinforce workshop.  This will attack the MozDef account.
+# A RegionMap of AMI IDs is required by AWS Marketplace  https://docs.aws.amazon.com/marketplace/latest/userguide/cloudformation.html#aws-cloudformation-template-preparation
+# INSERT MAPPING HERE : This template does not work in this state. The mapping is replaced with a working AWS region to AMI ID mapping as well as a variable map with the S3TemplateLocationPrefix by cloudy_mozdef/ci/publish_versioned_templates. The resulting functioning CloudFormation template is uploaded to S3 for the version being built.
+Conditions: 
+  LeakACredential: !Not [!Equals [!Ref LeakCredentialSNSArn, ""]]
+Resources:
+  LeakedCredentials:
+    Condition: LeakACredential
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        SNSReceiverArn: !Ref LeakCredentialSNSArn
+      Tags:
+        - Key: application
+          Value: mozdef
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ] , mozdef-credential-leak.yml ] ]
+  MozDefVPC:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Tags:
+        - Key: application
+          Value: mozdef
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ] , mozdef-vpc.yml ] ]
+  MozDefSecurityGroups:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !GetAtt MozDefVPC.Outputs.VpcId
+        SSHIngressCIDR: !Ref SSHIngressCIDR
+      Tags:
+        - Key: application
+          Value: mozdef
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ] , mozdef-security-group.yml ] ]
+  MozDefIAMRoleAndInstanceProfile:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        CloudTrailS3BucketName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailS3BucketName
+        CloudTrailSQSQueueArn: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueArn
+        MozDefSQSQueueArn: !GetAtt MozDefSQS.Outputs.SQSQueueArn
+        MozDefAlertSqsQueueArn: !GetAtt MozDefSQS.Outputs.AlertTaskSQSQueueArn
+        # CloudTrailS3BucketIAMRoleArn we leave empty as we will consume CloudTrail logs from our own account
+        ESServiceLinkedRoleExists: !GetAtt ESServiceLinkedRoleExists.RoleExists
+      Tags:
+        - Key: application
+          Value: mozdef
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], base-iam.yml ] ]
+  MozDefInstance:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !GetAtt MozDefVPC.Outputs.VpcId
+        InstanceType: !Ref InstanceType
+        KeyName: !Ref KeyName
+        IamInstanceProfile: !GetAtt MozDefIAMRoleAndInstanceProfile.Outputs.InstanceProfileArn
+        AutoScaleGroupSubnetIds: !Join [ ',', [!GetAtt MozDefVPC.Outputs.Subnet1, !GetAtt MozDefVPC.Outputs.Subnet2, !GetAtt MozDefVPC.Outputs.Subnet3 ]]
+        AMIImageId: !FindInMap [ RegionMap, !Ref 'AWS::Region', HVM64 ]
+        EFSID: !GetAtt MozDefEFS.Outputs.EFSID
+        MozDefSecurityGroupId: !GetAtt MozDefSecurityGroups.Outputs.MozDefSecurityGroupId
+        MozDefLoadBalancerSecurityGroupId: !GetAtt MozDefSecurityGroups.Outputs.MozDefLoadBalancerSecurityGroupId
+        MozDefACMCertArn: !Ref ACMCertArn
+        ESURL: !GetAtt MozDefES.Outputs.ElasticsearchURL
+        KibanaURL: !GetAtt MozDefES.Outputs.ElasticsearchKibanaURL
+        KibanaDomainOnlyURL: !GetAtt MozDefES.Outputs.ElasticsearchDomainOnlyURL
+        OIDCClientId: !Ref OIDCClientId
+        OIDCClientSecret: !Ref OIDCClientSecret
+        OIDCAuthorizationEndpoint: !Ref OIDCAuthorizationEndpoint
+        OIDCIssuer: !Ref OIDCIssuer
+        OIDCTokenEndpoint: !Ref OIDCTokenEndpoint
+        OIDCUserInfoEndpoint: !Ref OIDCUserInfoEndpoint
+        CloudTrailSQSNotificationQueueName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueName
+        MozDefSQSQueueName: !GetAtt MozDefSQS.Outputs.SQSQueueName
+        DomainName: !Ref DomainName
+        AlertQueueUrl: !GetAtt MozDefSQS.Outputs.AlertTaskSQSQueueUrl
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-instance.yml ] ]
+  MozDefES:
+    Type: AWS::CloudFormation::Stack
+    DependsOn: MozDefIAMRoleAndInstanceProfile
+    Properties:
+      Parameters:
+        SubnetIds: !Join [ ',', [!GetAtt MozDefVPC.Outputs.Subnet1, !GetAtt MozDefVPC.Outputs.Subnet2, !GetAtt MozDefVPC.Outputs.Subnet3 ]]
+        BlockStoreSizeGB: '100'
+        VpcId: !GetAtt MozDefVPC.Outputs.VpcId
+        MozDefInstanceSecurityGroup: !GetAtt MozDefSecurityGroups.Outputs.MozDefSecurityGroupId
+        ESInstanceCount: '1'
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-es.yml ] ]
+  MozDefEFS:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !GetAtt MozDefVPC.Outputs.VpcId
+        SubnetList: !Join [ ',', [!GetAtt MozDefVPC.Outputs.Subnet1, !GetAtt MozDefVPC.Outputs.Subnet2, !GetAtt MozDefVPC.Outputs.Subnet3 ]]
+        NumberOfSubnets: !GetAtt NumberOfSubnets.Length
+        MozDefSecurityGroup: !GetAtt MozDefSecurityGroups.Outputs.MozDefSecurityGroupId
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-efs.yml ] ]
+  MozDefSQS:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Tags:
+      - Key: application
+        Value: mozdef
+      - Key: stack
+        Value: !Ref AWS::StackName
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-sqs.yml ] ]
+  MozDefCloudTrail:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Tags:
+      - Key: application
+        Value: mozdef
+      - Key: stack
+        Value: !Ref AWS::StackName
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-cloudtrail.yml ] ]
+  MozDefVPCFlowLogs:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !GetAtt MozDefVPC.Outputs.VpcId
+        MozDefSQSQueueArn: !GetAtt MozDefSQS.Outputs.SQSQueueArn
+        MozDefSQSQueueUrl: !GetAtt MozDefSQS.Outputs.SQSQueueUrl
+      Tags:
+      - Key: application
+        Value: mozdef
+      - Key: stack
+        Value: !Ref AWS::StackName
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-vpc-flow-logs.yml ] ]
+  CloudFormationLambdaIAMRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+        - Effect: Allow
+          Principal:
+            Service:
+            - lambda.amazonaws.com
+          Action:
+          - sts:AssumeRole
+      Policies:
+        -
+          PolicyName: AllowLambdaLogging
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              -
+                Effect: Allow
+                Action:
+                  - logs:*
+                  - iam:ListRoles
+                Resource: '*'
+  GetArrayLengthLambdaFunction:
+    Type: AWS::Lambda::Function
+    DependsOn: CloudFormationLambdaIAMRole
+    # This DependsOn shouldn't be needed because the "Role" value is set to
+    # "!GetAtt CloudFormationLambdaIAMRole.Arn" but without DependsOn the error
+    # "Template error: IAM role mozdef-aws-nested-CloudFormationLambdaIAMRole-108UCUPESC6WG doesn't exist"
+    # occurs on stack creation for this Lambda Function resource. The DependsOn
+    # prevents the error.
+    Properties:
+      Code:
+        ZipFile: |
+          import cfnresponse
+          import secrets, string
+          def handler(event, context):
+            length = len(event['ResourceProperties']['Array'])
+            physical_id = ''.join(secrets.choice(string.ascii_uppercase + string.digits) for i in range(13))
+            cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Length': length}, "GetArrayLength-%s" % physical_id)
+      Handler: index.handler
+      Runtime: python3.6
+      Role: !GetAtt CloudFormationLambdaIAMRole.Arn
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+      Timeout: 20
+  NumberOfSubnets:
+    Type: AWS::CloudFormation::CustomResource
+    Properties:
+      Array: 
+        - !GetAtt MozDefVPC.Outputs.Subnet1
+        - !GetAtt MozDefVPC.Outputs.Subnet2
+        - !GetAtt MozDefVPC.Outputs.Subnet3
+      ServiceToken: !GetAtt GetArrayLengthLambdaFunction.Arn
+  DoesRoleExistLambdaFunction:
+    Type: AWS::Lambda::Function
+    DependsOn: CloudFormationLambdaIAMRole
+    # This DependsOn shouldn't be needed because the "Role" value is set to
+    # "!GetAtt CloudFormationLambdaIAMRole.Arn" but without DependsOn the error
+    # "Template error: IAM role mozdef-aws-nested-CloudFormationLambdaIAMRole-108UCUPESC6WG doesn't exist"
+    # occurs on stack creation for this Lambda Function resource. The DependsOn
+    # prevents the error.
+    Properties:
+      Code:
+        ZipFile: |
+          import cfnresponse
+          import boto3, secrets, string
+          def handler(event, context):
+              paginator = boto3.client('iam').get_paginator('list_roles')
+              args = {'PathPrefix': event['ResourceProperties']['PathPrefix']} if 'PathPrefix' in event['ResourceProperties'] else {}
+              iterator = paginator.paginate(**args).search(
+                  "Roles[?RoleName == '%s'][]" % event['ResourceProperties']['RoleName'])
+              response = {'RoleExists': len([x for x in iterator]) > 0}
+              physical_id = ''.join(
+                  secrets.choice(string.ascii_uppercase + string.digits) for i in
+                  range(13))
+              cfnresponse.send(event, context, cfnresponse.SUCCESS, response,
+                               "DoesRoleExist-%s" % physical_id)
+      Handler: index.handler
+      Runtime: python3.6
+      Role: !GetAtt CloudFormationLambdaIAMRole.Arn
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+      Timeout: 20
+  ESServiceLinkedRoleExists:
+    Type: AWS::CloudFormation::CustomResource
+    Properties:
+      RoleName: AWSServiceRoleForAmazonElasticsearchService
+      PathPrefix: '/aws-service-role/es.amazonaws.com/'
+      ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn
+  MozDefAlertWriterEnv:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !GetAtt MozDefVPC.Outputs.VpcId
+        PublicSubnetIds: !Join [ ',', [!GetAtt MozDefVPC.Outputs.Subnet1, !GetAtt MozDefVPC.Outputs.Subnet2, !GetAtt MozDefVPC.Outputs.Subnet3 ]]
+        MozDefSecurityGroup: !GetAtt MozDefSecurityGroups.Outputs.MozDefSecurityGroupId
+        ESUrl: !GetAtt MozDefES.Outputs.ElasticsearchURL
+      Tags:
+        - Key: application
+          Value: mozdef
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ] , mozdef-alert-developer.yml ] ]
+  MozDefVPCFlowLogs:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !GetAtt MozDefVPC.Outputs.VpcId
+        MozDefSQSQueueArn: !GetAtt MozDefSQS.Outputs.SQSQueueArn
+        MozDefSQSQueueUrl: !GetAtt MozDefSQS.Outputs.SQSQueueUrl
+      Tags:
+        - Key: application
+          Value: mozdef
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ] , mozdef-vpc-flow-logs.yml ] ]
+Outputs:
+  LoadBalancerDNSName:
+    Description: The DNS name of the ALB hosting MozDef.  If using OIDC or SSL point your DNS at this.  If using basic auth no DNS is necessary.
+    Value: !GetAtt MozDefInstance.Outputs.LoadBalancerDNSName

cloudy_mozdef/cloudformation/mozdef-parent.yml

@@ -19,11 +19,14 @@ Metadata:
       Parameters:
         - ACMCertArn
     - Label:
-        default: OIDC Configuration
+        default: OIDC Configuration (optional)  If not set this will use basic auth.
       Parameters:
-        - OIDCDiscoveryURL
+        - OIDCAuthorizationEndpoint
         - OIDCClientId
         - OIDCClientSecret
+        - OIDCIssuer
+        - OIDCTokenEndpoint
+        - OIDCUserInfoEndpoint
     ParameterLabels:
       VpcId:
         default: VPC ID
@@ -39,12 +42,18 @@ Metadata:
         default: FQDN to host MozDef at
       ACMCertArn:
         default: ACM Certificate ARN
-      OIDCDiscoveryURL:
-        default: OIDC Discovery URL
+      OIDCAuthorizationEndpoint:
+        default: OIDC authorization endpoint.
       OIDCClientId:
-        default: OIDC Client ID
+        default: OIDC Client ID.
       OIDCClientSecret:
-        default: OIDC Client Secret
+        default: OIDC Client Secret.
+      OIDCIssuer:
+        default: OIDC issuer.
+      OIDCTokenEndpoint:
+        default: OIDC oauth token endpoint.
+      OIDCUserInfoEndpoint:
+        default: OIDC user info endpoint.
 Parameters:
   VpcId:
     Type: AWS::EC2::VPC::Id
@@ -70,19 +79,34 @@ Parameters:
     Default: cloudymozdef.security.allizom.org
   ACMCertArn:
     Type: String
+    Default: Unset
     Description: "The ARN of your pre-issued ACM cert. (Example: arn:aws:acm:us-west-2:123456789012:certificate/abcdef01-2345-6789-abcd-ef0123456789)"
-  OIDCDiscoveryURL:
+  OIDCAuthorizationEndpoint:
     Type: String
-    AllowedPattern: '^https?:\/\/.*'
+    Default: Unset
     ConstraintDescription: A valid URL
-    Description: "The URL of your OIDC provider's well-known discovery URL (Example: https://auth.example.com/.well-known/openid-configuration)"
+    Description: "The url of the authorization endpoint found for your oidc provider generall found on (Example: https://auth.example.com/.well-known/openid-configuration)"
   OIDCClientId:
     Type: String
+    Default: Unset
     Description: The client ID that your OIDC provider issues you for your Mozdef instance.
   OIDCClientSecret:
     Type: String
+    Default: Unset
     Description: The secret that your OIDC provider issues you for your Mozdef instance.
     NoEcho: true
+  OIDCIssuer:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
+  OIDCTokenEndpoint:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
+  OIDCUserInfoEndpoint:
+    Type: String
+    Default: Unset
+    Description: Generally can be found at the .well-known endpoint for your provider.
 # A RegionMap of AMI IDs is required by AWS Marketplace  https://docs.aws.amazon.com/marketplace/latest/userguide/cloudformation.html#aws-cloudformation-template-preparation
 # INSERT MAPPING HERE : This template does not work in this state. The mapping is replaced with a working AWS region to AMI ID mapping as well as a variable map with the S3TemplateLocationPrefix by cloudy_mozdef/ci/publish_versioned_templates. The resulting functioning CloudFormation template is uploaded to S3 for the version being built.
 Resources:
@@ -103,6 +127,7 @@ Resources:
         CloudTrailS3BucketName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailS3BucketName
         CloudTrailSQSQueueArn: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueArn
         MozDefSQSQueueArn: !GetAtt MozDefSQS.Outputs.SQSQueueArn
+        MozDefAlertSqsQueueArn: !GetAtt MozDefSQS.Outputs.AlertTaskSQSQueueArn
         # CloudTrailS3BucketIAMRoleArn we leave empty as we will consume CloudTrail logs from our own account
         ESServiceLinkedRoleExists: !GetAtt ESServiceLinkedRoleExists.RoleExists
       Tags:
@@ -128,10 +153,14 @@ Resources:
         KibanaDomainOnlyURL: !GetAtt MozDefES.Outputs.ElasticsearchDomainOnlyURL
         OIDCClientId: !Ref OIDCClientId
         OIDCClientSecret: !Ref OIDCClientSecret
-        OIDCDiscoveryURL: !Ref OIDCDiscoveryURL
+        OIDCAuthorizationEndpoint: !Ref OIDCAuthorizationEndpoint
+        OIDCIssuer: !Ref OIDCIssuer
+        OIDCTokenEndpoint: !Ref OIDCTokenEndpoint
+        OIDCUserInfoEndpoint: !Ref OIDCUserInfoEndpoint
         CloudTrailSQSNotificationQueueName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueName
         MozDefSQSQueueName: !GetAtt MozDefSQS.Outputs.SQSQueueName
         DomainName: !Ref DomainName
+        AlertQueueUrl: !GetAtt MozDefSQS.Outputs.AlertTaskSQSQueueUrl
       Tags:
         - Key: application
           Value: mozdef
@@ -186,6 +215,19 @@ Resources:
       - Key: stack
         Value: !Ref AWS::StackName
       TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-cloudtrail.yml ] ]
+  MozDefVPCFlowLogs:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !Ref VpcId
+        MozDefSQSQueueArn: !GetAtt MozDefSQS.Outputs.SQSQueueArn
+        MozDefSQSQueueUrl: !GetAtt MozDefSQS.Outputs.SQSQueueUrl
+      Tags:
+      - Key: application
+        Value: mozdef
+      - Key: stack
+        Value: !Ref AWS::StackName
+      TemplateURL: !Join [ '', [ !FindInMap [ VariableMap, Variables, S3TemplateLocation ], mozdef-vpc-flow-logs.yml ] ]
   CloudFormationLambdaIAMRole:
     Type: AWS::IAM::Role
     Properties:
@@ -280,3 +322,7 @@ Resources:
       RoleName: AWSServiceRoleForAmazonElasticsearchService
       PathPrefix: '/aws-service-role/es.amazonaws.com/'
       ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn
+Outputs:
+  LoadBalancerDNSName:
+    Description: The DNS name of the ALB hosting MozDef.  If using OIDC or SSL point your DNS at this.  If using basic auth no DNS is necessary.
+    Value: !GetAtt MozDefInstance.Outputs.LoadBalancerDNSName

cloudy_mozdef/cloudformation/mozdef-security-group.yml

@@ -26,11 +26,11 @@ Resources:
         FromPort: 80
         ToPort: 80
         SourceSecurityGroupId: !Ref MozDefLoadBalancerSecurityGroup
-      - Description: Allow 9090 inbound from everywhere
+      - Description: Allow 9090 inbound from loadbalancer
         IpProtocol: tcp
         FromPort: 9090
         ToPort: 9090
-        CidrIp: 0.0.0.0/0
+        SourceSecurityGroupId: !Ref MozDefLoadBalancerSecurityGroup
       Tags:
         - Key: application
           Value: mozdef
@@ -61,6 +61,11 @@ Resources:
           FromPort: 9090
           ToPort: 9090
           CidrIp: 0.0.0.0/0
+        - Description: Allow 9443 inbound from everywhere
+          IpProtocol: tcp
+          FromPort: 9443
+          ToPort: 9443
+          CidrIp: 0.0.0.0/0
       Tags:
         - Key: application
           Value: mozdef

cloudy_mozdef/cloudformation/mozdef-sqs.yml

@@ -23,6 +23,28 @@ Resources:
           Resource: !GetAtt MozDefSQSQueue.Arn
       Queues:
         - !Ref MozDefSQSQueue
+  MozDefSQSAlertTaskQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      Tags:
+      - Key: application
+        Value: mozdef
+      - Key: stack
+        Value: !Ref AWS::StackName
+  MozDefAlertTaskSQSQueuePolicy:
+    Type: AWS::SQS::QueuePolicy
+    Properties:
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+        - Sid: AllowThisAccountSendToSQS
+          Effect: Allow
+          Principal:
+            AWS: !Join [ '', [ 'arn:', 'aws:', 'iam::', !Ref 'AWS::AccountId', ':root' ] ]
+          Action: sqs:SendMessage
+          Resource: !GetAtt MozDefSQSAlertTaskQueue.Arn
+      Queues:
+        - !Ref MozDefSQSAlertTaskQueue
 Outputs:
   SQSQueueArn:
     Description: ARN of the SQS Queue that MozDef will consume events from
@@ -30,3 +52,15 @@ Outputs:
   SQSQueueName:
     Description: Name of the SQS Queue that MozDef will consume events from
     Value: !GetAtt MozDefSQSQueue.QueueName
+  SQSQueueUrl:
+    Description: URL of the SQS Queue that MozDef will consume events from
+    Value: !Ref MozDefSQSQueue
+  AlertTaskSQSQueueArn:
+    Description: ARN of the SQS Queue that MozDef will consume events from
+    Value: !GetAtt MozDefSQSAlertTaskQueue.Arn
+  AlertTaskSQSQueueName:
+    Description: Name of the SQS Queue that MozDef will consume events from
+    Value: !GetAtt MozDefSQSAlertTaskQueue.QueueName
+  AlertTaskSQSQueueUrl:
+    Description: The SQS queue url for the alerttask exchange as used in kombu.
+    Value: !Ref MozDefSQSAlertTaskQueue

cloudy_mozdef/cloudformation/mozdef-vpc-flow-logs.yml

@@ -0,0 +1,176 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Pipeline to send VPC Flow Logs to MozDef
+Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+    Default: vpc-dc8eacb4
+    Description: 'The VPC ID of the VPC to deploy in (Example : vpc-abcdef12)'
+  MozDefSQSQueueUrl:
+    Type: String
+    Description: 'The SQS URL to send MozDef structured events to for consumption'
+  MozDefSQSQueueArn:
+    Type: String
+    Description: 'The SQS ARN to send MozDef structured events to for consumption'
+Resources:
+  LogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      RetentionInDays: 1
+  FlowLogRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: vpc-flow-logs.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: AllowWriteCloudWatchLogs
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - logs:CreateLogGroup
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                  - logs:DescribeLogGroups
+                  - logs:DescribeLogStreams
+                Resource: "*"
+  FlowLog:
+    Type: AWS::EC2::FlowLog
+    Properties:
+      DeliverLogsPermissionArn: !GetAtt FlowLogRole.Arn
+      # We can't use !GetAtt LogGroup.Arn because it actually returns and Arn suffixed with ":*"
+      LogDestination: !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group', !Ref 'LogGroup' ] ]
+      ResourceId: !Ref VpcId
+      ResourceType: VPC
+      TrafficType: ALL
+  FlowLogProcessorRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+      Policies:
+        - PolicyName: AllowSendToSQS
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - sqs:DeleteMessage
+                  - sqs:DeleteMessageBatch
+                  - sqs:GetQueueAttributes
+                  - sqs:GetQueueUrl
+                  - sqs:SendMessage
+                  - sqs:SendMessageBatch
+                Resource: !Ref MozDefSQSQueueArn
+  FlowLogProcessor:
+    Type: AWS::Lambda::Function
+    Properties:
+      Code:
+        ZipFile: |
+          import os, boto3, gzip, base64, json, socket, sys
+          from datetime import datetime
+
+          PROTO_NUM_MAP = {num: name[8:] for name, num in vars(socket).items() if name.startswith("IPPROTO")}
+          FIELD_NAMES = [
+              'version', 'account-id', 'interface-id', 'srcaddr', 'dstaddr', 'srcport',
+              'dstport', 'protocol', 'packets', 'bytes', 'start', 'end', 'action',
+              'log-status']
+
+          def lambda_handler(event, context):
+              client = boto3.client('sqs')
+              raw_data = event.get('awslogs', {}).get('data')
+              data = json.loads(
+                  gzip.decompress(base64.b64decode(raw_data)).decode('utf-8'))
+              entries = []
+              for log_event_record in data.get('logEvents', ''):
+                  log_event_record_values = log_event_record['message'].split(' ')
+                  log_event = {FIELD_NAMES[i]: log_event_record_values[i]
+                               for i in range(len(FIELD_NAMES))}
+                  if log_event.get('log-status') != 'OK':
+                      print('Skipping {} entry : {}'.format(log_event.get('log-status'), log_event_record['message']))
+                      continue
+
+                  # TODO : Do we want to do something with log_status NODATA and SKIPDATA events?
+                  message = dict(
+                      category='vpc-flow',
+                      hostname=socket.getfqdn(),
+                      processid=os.getpid(),
+                      processname=sys.argv[0],
+                      severity='INFO',
+                      source='vpc_flow')
+                  message['utctimestamp'] = datetime.utcfromtimestamp(
+                      int(log_event_record['timestamp'] / 1000)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
+                  message['summary'] = '{srcaddr}:{srcport} -> {dstaddr}:{dstport} {bytes} bytes {action}'.format(**log_event)
+                  message['details'] = dict(
+                      destinationipaddress=log_event['dstaddr'],
+                      destinationport=int(log_event['dstport']),
+                      sourceipaddress=log_event['srcaddr'],
+                      sourceport=int(log_event['srcport']),
+                      success=log_event['action'] == 'ACCEPT',
+                      capture_window_start=datetime.utcfromtimestamp(
+                          int(log_event['start'])).strftime('%Y-%m-%dT%H:%M:%S+00:00'),
+                      capture_window_end=datetime.utcfromtimestamp(
+                          int(log_event['end'])).strftime('%Y-%m-%dT%H:%M:%S+00:00'),
+                      version=int(log_event['version']),
+                      pkts=int(log_event['packets']),
+                      proto=PROTO_NUM_MAP.get(int(log_event['protocol']), 'unknown').lower(),
+                      recipientaccountid=log_event['account-id'],
+                      interface_id=log_event['interface-id'],
+                      bytes=int(log_event['bytes']))
+                  entry = dict(
+                      Id=log_event_record['id'],
+                      MessageBody=json.dumps(message))
+                  entries.append(entry)
+                  print('Going to send entry : {}'.format(entry))
+                  if len(entries) == 10:
+                      print('sending batch')
+                      response = client.send_message_batch(
+                          QueueUrl=os.getenv('SQS_URL'),
+                          Entries=entries)
+                      # TODO : Process the response and do something about failures
+                      del entries[:]
+              if len(entries) > 0:
+                  print('sending final batch')
+                  response = client.send_message_batch(
+                      QueueUrl=os.getenv('SQS_URL'),
+                      Entries=entries)
+      Description: Transform VPC Flow logs into MozDef events
+      Environment:
+        Variables:
+          SQS_URL: !Ref MozDefSQSQueueUrl
+      Handler: index.lambda_handler
+      MemorySize: 128
+      Role: !GetAtt FlowLogProcessorRole.Arn
+      Runtime: python3.7
+      Tags:
+      - Key: application
+        Value: mozdef
+      - Key: stack
+        Value: !Ref AWS::StackName
+      Timeout: 30
+  FlowLogProcessorPermission:
+    Type: AWS::Lambda::Permission
+    Properties:
+      Action: lambda:InvokeFunction
+      FunctionName: !GetAtt FlowLogProcessor.Arn
+      Principal: !Join [ '.', [ 'logs', !Ref 'AWS::Region', 'amazonaws.com' ] ]
+      SourceAccount: !Ref 'AWS::AccountId'
+      SourceArn: !GetAtt LogGroup.Arn
+  FlowLogSubscriptionFilter:
+    Type: AWS::Logs::SubscriptionFilter
+    Properties:
+      DestinationArn: !GetAtt FlowLogProcessor.Arn
+      FilterPattern: '[version, account, eni, source, destination, srcport, destport="22", protocol="6", packets, bytes, windowstart, windowend, action="ACCEPT", flowlogstatus]'
+      LogGroupName: !Ref LogGroup

cloudy_mozdef/cloudformation/mozdef-vpc.yml

@@ -0,0 +1,133 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "Create a vpc for Mozilla Deployment of Cloudy Mozdef."
+Resources:
+  InternetGateway:
+    Type: "AWS::EC2::InternetGateway"
+    Properties:
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+  VPC:
+    Type: "AWS::EC2::VPC"
+    Properties:
+      CidrBlock: "10.0.0.0/16"
+      EnableDnsSupport: True
+      EnableDnsHostnames: True
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+  AttachGateway:
+    Type: AWS::EC2::VPCGatewayAttachment
+    Properties:
+      VpcId:
+        Ref: VPC
+      InternetGatewayId:
+        Ref: InternetGateway
+  RouteTable:
+    Type: "AWS::EC2::RouteTable"
+    Properties:
+      VpcId:
+        Ref: VPC
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+  DefaultRoute:
+    Type: AWS::EC2::Route
+    Properties:
+      RouteTableId:
+        Ref: RouteTable
+      DestinationCidrBlock: 0.0.0.0/0
+      GatewayId:
+        Ref: InternetGateway
+  Subnet1:
+    Type: "AWS::EC2::Subnet"
+    Properties:
+      AvailabilityZone:
+        Fn::Select:
+          - 0
+          - Fn::GetAZs: ""
+      CidrBlock: "10.0.0.0/24"
+      MapPublicIpOnLaunch: True
+      VpcId:
+        Ref: VPC
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+  Subnet2:
+    Type: "AWS::EC2::Subnet"
+    Properties:
+      AvailabilityZone:
+        Fn::Select:
+          - 1
+          - Fn::GetAZs: ""
+      CidrBlock: "10.0.1.0/24"
+      MapPublicIpOnLaunch: True
+      VpcId:
+        Ref: VPC
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+  Subnet3:
+    Type: "AWS::EC2::Subnet"
+    Properties:
+      AvailabilityZone:
+        Fn::Select:
+          - 2
+          - Fn::GetAZs: ""
+      CidrBlock: "10.0.2.0/24"
+      MapPublicIpOnLaunch: True
+      VpcId:
+        Ref: VPC
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+  RouteAc1:
+    Type: "AWS::EC2::SubnetRouteTableAssociation"
+    Properties:
+      RouteTableId:
+        Ref: RouteTable
+      SubnetId:
+        Ref: Subnet1
+  RouteAc2:
+    Type: "AWS::EC2::SubnetRouteTableAssociation"
+    Properties:
+      RouteTableId:
+        Ref: RouteTable
+      SubnetId:
+        Ref: Subnet2
+  RouteAc3:
+    Type: "AWS::EC2::SubnetRouteTableAssociation"
+    Properties:
+      RouteTableId:
+        Ref: RouteTable
+      SubnetId:
+        Ref: Subnet3
+Outputs:
+  VpcId:
+    Description: The ID of the VPC created.
+    Value:
+      Ref: VPC
+  Subnet1:
+    Description: The id of subnet1 in the first az.
+    Value:
+      Ref: Subnet1
+  Subnet2:
+    Description: The id of subnet2 in the second az.
+    Value:
+      Ref: Subnet2
+  Subnet3:
+    Description: The id of subnet3 in the third az.
+    Value:
+      Ref: Subnet3

cloudy_mozdef/experiments/alert-writer.yml

@@ -0,0 +1,63 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Setup an alert writers environment for use with MozDef for AWS.  Note this is PoC only.
+Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+    Description: 'The VPC ID of the VPC to deploy in (Example : vpc-abcdef12)'
+  PublicSubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+    Description: 'A comma delimited list of public subnet IDs (Example: subnet-abcdef12,subnet-bcdef123)'
+  MozDefSecurityGroup:
+    Type: AWS::EC2::SecurityGroup::Id
+    Description: The security group the MozDef instance runs in.  This is needed to access ES.
+  ESUrl:
+    Type: String
+    Description: 'The location of elasticsearch deployed in managed-es.'
+Resources:
+    MozDefLayer:
+        Type: AWS::Lambda::LayerVersion
+        Properties:
+            LayerName: MozDef
+            Description: Mozilla Enterprise Defense Platform Dependencies
+            Content:
+                S3Bucket: public.us-west-2.security.allizom.org
+                S3Key: mozdef-lambda-layer/layer-latest.zip
+            CompatibleRuntimes:
+              - python2.7
+            LicenseInfo: 'MPL 2.0'
+    LambdalertIAMRole:
+      Type: AWS::IAM::Role
+      Properties:
+        AssumeRolePolicyDocument:
+          Version: 2012-10-17
+          Statement:
+            - Effect: Allow
+              Principal:
+                Service: lambda.amazonaws.com
+              Action: sts:AssumeRole
+        ManagedPolicyArns:
+          -  arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole 
+    AlertWritersEnv: 
+      Type: "AWS::Lambda::Function"
+      Properties: 
+        Handler: "lambdalert.handle"
+        Role: 
+          Fn::GetAtt: 
+            - "LambdalertIAMRole"
+            - "Arn"
+        Code: 
+          S3Bucket: public.us-west-2.security.allizom.org
+          S3Key: mozdef-lambda-layer/function-latest.zip
+        Layers:
+          - !Ref MozDefLayer
+        Environment:
+          Variables:
+            OPTIONS_ESSERVERS: !Ref ESUrl
+            OPTIONS_MQPROTOCOL: sqs
+        VpcConfig:
+          SecurityGroupIds: 
+            - !Ref MozDefSecurityGroup
+          SubnetIds: !Ref PublicSubnetIds
+        ReservedConcurrentExecutions: 1
+        Runtime: "python2.7"
+        Timeout: 120

cloudy_mozdef/lambda_layer/.gitignore

@@ -0,0 +1,7 @@
+function-latest.zip
+layer-latest.zip
+build/lib/*
+build/python/*
+lib/*
+python/*
+build/lambdalert.py

cloudy_mozdef/lambda_layer/Makefile

@@ -0,0 +1,46 @@
+# Makefile to build and pack a lambda layer containing MozDef Alert Capabilities
+
+ROOT_DIR	:= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+STAGE			:= ${STAGE}
+STAGE			:= $(if $(STAGE),$(STAGE),testing)
+PYDIR			:= python/
+all:
+	@echo 'Available make targets:'
+	@grep '^[^#[:space:]^\.PHONY.*].*:' Makefile
+
+.PHONY: clean
+clean:
+	rm -rf $(ROOT_DIR)/build/*
+
+.PHONY: deploy-shell
+deploy-shell: 
+	docker run -ti -v ~/.aws:/root/.aws -v ${PWD}:/var/task mozdef/mozdef_base:latest
+
+.PHONY: package-layer
+package-layer: clean
+	rm -rf $(ROOT_DIR)/layer-latest.zip
+	docker run -ti -v ~/.aws:/root/.aws -v ${PWD}:/var/task mozdef/mozdef_base:latest pip install -r requirements.txt -t /var/task/cloudy_mozdef/lambda_layer/build/${PYDIR}
+	docker run -ti -v ~/.aws:/root/.aws -v ${PWD}:/var/task mozdef/mozdef_base:latest pip install /var/task/mozdef_util/ -t /var/task/cloudy_mozdef/lambda_layer/build/${PYDIR}
+	docker run -ti -v ~/.aws:/root/.aws -v ${PWD}:/var/task mozdef/mozdef_base:latest bash -c "yum install zip -y && cd /var/task/cloudy_mozdef/lambda_layer/build/ && \
+	zip -r /var/task/cloudy_mozdef/lambda_layer/layer-latest.zip . "
+
+.PHONY: package-function
+package-function: clean
+	rm -rf $(ROOT_DIR)/function-latest.zip
+	cp ${PWD}/cloudy_mozdef/lambda_layer/lambdalert.py ${PWD}/cloudy_mozdef/lambda_layer/build/
+	cp -r ${PWD}/alerts/lib ${PWD}/cloudy_mozdef/lambda_layer/build/
+	docker run -ti -v ~/.aws:/root/.aws -v ${PWD}:/var/task mozdef/mozdef_base:latest \
+	bash -c "yum install zip -y && cd /var/task/cloudy_mozdef/lambda_layer/build && \
+		zip -r /var/task/cloudy_mozdef/lambda_layer/function-latest.zip ."
+
+.PHONY: upload-s3
+upload-s3: 
+	aws s3 cp ${PWD}/cloudy_mozdef/lambda_layer/layer-latest.zip s3://public.us-west-2.security.allizom.org/mozdef-lambda-layer/layer-latest.zip
+	aws s3 cp ${PWD}/cloudy_mozdef/lambda_layer/function-latest.zip s3://public.us-west-2.security.allizom.org/mozdef-lambda-layer/function-latest.zip
+
+.PHONY: publish-layer
+publish-layer: upload-s3
+	aws lambda publish-layer-version \
+		--layer-name mozdef --compatible-runtimes python2.7 \
+		--content S3Bucket=public.us-west-2.security.allizom.org,S3Key=mozdef-lambda-layer/layer-latest.zip
+

cloudy_mozdef/lambda_layer/build/.gitignore

@@ -0,0 +1,2 @@
+lib/
+python/

cloudy_mozdef/lambda_layer/build/lambdalert.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+import logging
+import sys
+from lib.alerttask import AlertTask
+from mozdef_util.query_models import SearchQuery, TermMatch
+
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging():
+    logger = logging.getLogger()
+    h = logging.StreamHandler(sys.stdout)
+    logger.setLevel(logging.DEBUG)
+    return logger
+
+
+class AlertCloudtrailLoggingDisabled(AlertTask):
+    def _configureKombu(self):
+        """Override the normal behavior of this in order to run in lambda."""
+        pass
+
+    def alertToMessageQueue(self, alertDict):
+        """Override the normal behavior of this in order to run in lambda."""
+        pass
+
+    def main(self):
+        # How many minutes back in time would you like to search?
+        search_query = SearchQuery(minutes=15)
+
+        # What would you like to search for?
+        # search_query.add_must([
+        #    TermMatch('source', 'cloudtrail'),
+        #    TermMatch('details.eventname', 'DescribeTable')
+        # ])
+
+        self.filtersManual(search_query)
+        self.searchEventsSimple()
+        self.walkEvents()
+
+    def onEvent(self, event):
+        category = 'AWSCloudtrail'
+
+        # Useful tag and severity rankings for your alert.
+        tags = ['cloudtrail', 'aws', 'cloudtrailpagerduty']
+        severity = 'CRITICAL'
+
+        # What message should surface in the user interface when this fires?
+        summary = 'The alert fired!'
+
+        return self.createAlertDict(summary, category, tags, [event], severity)
+
+        # Learn more about MozDef alerts by exploring the "Alert class!"
+
+
+def handle(event, context):
+    logger = setup_logging()
+    logger.debug('Function initialized.')
+    a = AlertCloudtrailLoggingDisabled()
+    return a.main()

cloudy_mozdef/lambda_layer/build/lib/alert_plugin_set.py

@@ -0,0 +1,12 @@
+from mozdef_util.plugin_set import PluginSet
+from mozdef_util.utilities.logger import logger
+
+
+class AlertPluginSet(PluginSet):
+
+    def send_message_to_plugin(self, plugin_class, message, metadata=None):
+        if 'utctimestamp' in message and 'summary' in message:
+            message_log_str = u'{0} received message: ({1}) {2}'.format(plugin_class.__module__, message['utctimestamp'], message['summary'])
+            logger.info(message_log_str)
+
+        return plugin_class.onMessage(message), metadata

cloudy_mozdef/lambda_layer/build/lib/alerttask.py

@@ -0,0 +1,553 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+import collections
+import json
+import kombu
+import os
+import sys
+import socket
+import netaddr
+
+from configlib import getConfig, OptionParser
+from datetime import datetime
+from collections import Counter
+from celery import Task
+from celery.utils.log import get_task_logger
+from config import RABBITMQ, ES, ALERT_PLUGINS
+
+from mozdef_util.utilities.toUTC import toUTC
+from mozdef_util.elasticsearch_client import ElasticsearchClient
+from mozdef_util.query_models import TermMatch, ExistsMatch
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
+from lib.alert_plugin_set import AlertPluginSet
+
+
+# utility functions used by AlertTask.mostCommon
+# determine most common values
+# in a list of dicts
+def keypaths(nested):
+    """ return a list of nested dict key paths
+        like: [u'_source', u'details', u'program']
+    """
+    for key, value in nested.iteritems():
+        if isinstance(value, collections.Mapping):
+            for subkey, subvalue in keypaths(value):
+                yield [key] + subkey, subvalue
+        else:
+            yield [key], value
+
+
+def dictpath(path):
+    """ split a string representing a
+        nested dictionary path key.subkey.subkey
+    """
+    for i in path.split("."):
+        yield "{0}".format(i)
+
+
+def getValueByPath(input_dict, path_string):
+    """
+        Gets data/value from a dictionary using a dotted accessor-string
+        http://stackoverflow.com/a/7534478
+        path_string can be key.subkey.subkey.subkey
+    """
+    return_data = input_dict
+    for chunk in path_string.split("."):
+        return_data = return_data.get(chunk, {})
+    return return_data
+
+
+def hostname_from_ip(ip):
+    try:
+        reversed_dns = socket.gethostbyaddr(ip)
+        return reversed_dns[0]
+    except socket.herror:
+        return None
+
+
+def add_hostname_to_ip(ip, output_format, require_internal=True):
+    ip_obj = netaddr.IPNetwork(ip)[0]
+    if require_internal and not ip_obj.is_private():
+        return ip
+    hostname = hostname_from_ip(ip)
+    if hostname is None:
+        return ip
+    else:
+        return output_format.format(ip, hostname)
+
+
+class AlertTask(Task):
+
+    abstract = True
+
+    def __init__(self):
+        self.alert_name = self.__class__.__name__
+        self.main_query = None
+
+        # Used to store any alerts that were thrown
+        self.alert_ids = []
+
+        # List of events
+        self.events = None
+        # List of aggregations
+        # e.g. when aggregField is email: [{value:'evil@evil.com',count:1337,events:[...]}, ...]
+        self.aggregations = None
+
+        self.log.debug("starting {0}".format(self.alert_name))
+        self.log.debug(RABBITMQ)
+        self.log.debug(ES)
+
+        self._configureKombu()
+        self._configureES()
+
+        # We want to select all event indices
+        # and filter out the window based on timestamp
+        # from the search query
+        self.event_indices = ["events-*"]
+
+    def classname(self):
+        return self.__class__.__name__
+
+    @property
+    def log(self):
+        return get_task_logger("%s.%s" % (__name__, self.alert_name))
+
+    def parse_config(self, config_filename, config_keys):
+        myparser = OptionParser()
+        self.config = None
+        (self.config, args) = myparser.parse_args([])
+        for config_key in config_keys:
+            temp_value = getConfig(config_key, "", config_filename)
+            setattr(self.config, config_key, temp_value)
+
+    def _discover_task_exchange(self):
+        """Use configuration information to understand the message queue protocol.
+        return: amqp, sqs
+        """
+        return getConfig("mqprotocol", "amqp", None)
+
+    def __build_conn_string(self):
+        exchange_protocol = self._discover_task_exchange()
+        if exchange_protocol == "amqp":
+            connString = "amqp://{0}:{1}@{2}:{3}//".format(
+                RABBITMQ["mquser"],
+                RABBITMQ["mqpassword"],
+                RABBITMQ["mqserver"],
+                RABBITMQ["mqport"],
+            )
+            return connString
+        elif exchange_protocol == "sqs":
+            connString = "sqs://{}".format(getConfig("alertSqsQueueUrl", None, None))
+            if connString:
+                connString = connString.replace('https://','')
+            return connString
+
+    def _configureKombu(self):
+        """
+        Configure kombu for amqp or sqs
+        """
+        try:
+            connString = self.__build_conn_string()
+            self.mqConn = kombu.Connection(connString)
+            if connString.find('sqs') == 0:
+                self.mqConn.transport_options['region'] = os.getenv('DEFAULT_AWS_REGION', 'us-west-2')
+                self.alertExchange = kombu.Exchange(
+                    name=RABBITMQ["alertexchange"], type="topic", durable=True
+                )
+                self.alertExchange(self.mqConn).declare()
+                alertQueue = kombu.Queue(
+                    os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4], exchange=self.alertExchange
+                )
+            else:
+                self.alertExchange = kombu.Exchange(
+                    name=RABBITMQ["alertexchange"], type="topic", durable=True
+                )
+                self.alertExchange(self.mqConn).declare()
+                alertQueue = kombu.Queue(
+                    RABBITMQ["alertqueue"], exchange=self.alertExchange
+                )
+            alertQueue(self.mqConn).declare()
+            self.mqproducer = self.mqConn.Producer(serializer="json")
+            self.log.debug("Kombu configured")
+        except Exception as e:
+            self.log.error(
+                "Exception while configuring kombu for alerts: {0}".format(e)
+            )
+
+    def _configureES(self):
+        """
+        Configure elasticsearch client
+        """
+        try:
+            self.es = ElasticsearchClient(ES["servers"])
+            self.log.debug("ES configured")
+        except Exception as e:
+            self.log.error("Exception while configuring ES for alerts: {0}".format(e))
+
+    def mostCommon(self, listofdicts, dictkeypath):
+        """
+            Given a list containing dictionaries,
+            return the most common entries
+            along a key path separated by .
+            i.e. dictkey.subkey.subkey
+            returned as a list of tuples
+            [(value,count),(value,count)]
+        """
+        inspectlist = list()
+        path = list(dictpath(dictkeypath))
+        for i in listofdicts:
+            for k in list(keypaths(i)):
+                if not (set(k[0]).symmetric_difference(path)):
+                    inspectlist.append(k[1])
+
+        return Counter(inspectlist).most_common()
+
+    def alertToMessageQueue(self, alertDict):
+        """
+        Send alert to the kombu based message queue.  The default is rabbitmq.
+        """
+        try:
+            # cherry pick items from the alertDict to send to the alerts messageQueue
+            mqAlert = dict(severity="INFO", category="")
+            if "severity" in alertDict:
+                mqAlert["severity"] = alertDict["severity"]
+            if "category" in alertDict:
+                mqAlert["category"] = alertDict["category"]
+            if "utctimestamp" in alertDict:
+                mqAlert["utctimestamp"] = alertDict["utctimestamp"]
+            if "eventtimestamp" in alertDict:
+                mqAlert["eventtimestamp"] = alertDict["eventtimestamp"]
+            mqAlert["summary"] = alertDict["summary"]
+            self.log.debug(mqAlert)
+            ensurePublish = self.mqConn.ensure(
+                self.mqproducer, self.mqproducer.publish, max_retries=10
+            )
+            ensurePublish(
+                alertDict,
+                exchange=self.alertExchange,
+                routing_key=RABBITMQ["alertqueue"],
+            )
+            self.log.debug("alert sent to the alert queue")
+        except Exception as e:
+            self.log.error(
+                "Exception while sending alert to message queue: {0}".format(e)
+            )
+
+    def alertToES(self, alertDict):
+        """
+        Send alert to elasticsearch
+        """
+        try:
+            res = self.es.save_alert(body=alertDict)
+            self.log.debug("alert sent to ES")
+            self.log.debug(res)
+            return res
+        except Exception as e:
+            self.log.error("Exception while pushing alert to ES: {0}".format(e))
+
+    def tagBotNotify(self, alert):
+        """
+            Tag alert to be excluded based on severity
+            If 'ircchannel' is set in an alert, we automatically notify mozdefbot
+        """
+        alert["notify_mozdefbot"] = True
+        if alert["severity"] == "NOTICE" or alert["severity"] == "INFO":
+            alert["notify_mozdefbot"] = False
+
+        # If an alert sets specific ircchannel, then we should probably always notify in mozdefbot
+        if (
+            "ircchannel" in alert and alert["ircchannel"] != "" and alert["ircchannel"] is not None
+        ):
+            alert["notify_mozdefbot"] = True
+        return alert
+
+    def saveAlertID(self, saved_alert):
+        """
+        Save alert to self so we can analyze it later
+        """
+        self.alert_ids.append(saved_alert["_id"])
+
+    def filtersManual(self, query):
+        """
+        Configure filters manually
+
+        query is a search query object with date_timedelta populated
+
+        """
+        # Don't fire on already alerted events
+        duplicate_matcher = TermMatch("alert_names", self.determine_alert_classname())
+        if duplicate_matcher not in query.must_not:
+            query.add_must_not(duplicate_matcher)
+
+        self.main_query = query
+
+    def determine_alert_classname(self):
+        alert_name = self.classname()
+        # Allow alerts like the generic alerts (one python alert but represents many 'alerts')
+        # can customize the alert name
+        if hasattr(self, "custom_alert_name"):
+            alert_name = self.custom_alert_name
+        return alert_name
+
+    def executeSearchEventsSimple(self):
+        """
+        Execute the search for simple events
+        """
+        return self.main_query.execute(self.es, indices=self.event_indices)
+
+    def searchEventsSimple(self):
+        """
+        Search events matching filters, store events in self.events
+        """
+        try:
+            results = self.executeSearchEventsSimple()
+            self.events = results["hits"]
+            self.log.debug(self.events)
+        except Exception as e:
+            self.log.error("Error while searching events in ES: {0}".format(e))
+
+    def searchEventsAggregated(self, aggregationPath, samplesLimit=5):
+        """
+        Search events, aggregate matching ES filters by aggregationPath,
+        store them in self.aggregations as a list of dictionaries
+        keys:
+          value: the text value that was found in the aggregationPath
+          count: the hitcount of the text value
+          events: the sampled list of events that matched
+          allevents: the unsample, total list of matching events
+        aggregationPath can be key.subkey.subkey to specify a path to a dictionary value
+        relative to the _source that's returned from elastic search.
+        ex: details.sourceipaddress
+        """
+
+        # We automatically add the key that we're matching on
+        # for aggregation, as a query requirement
+        aggreg_key_exists = ExistsMatch(aggregationPath)
+        if aggreg_key_exists not in self.main_query.must:
+            self.main_query.add_must(aggreg_key_exists)
+
+        try:
+            esresults = self.main_query.execute(self.es, indices=self.event_indices)
+            results = esresults["hits"]
+
+            # List of aggregation values that can be counted/summarized by Counter
+            # Example: ['evil@evil.com','haxoor@noob.com', 'evil@evil.com'] for an email aggregField
+            aggregationValues = []
+            for r in results:
+                aggregationValues.append(getValueByPath(r["_source"], aggregationPath))
+
+            # [{value:'evil@evil.com',count:1337,events:[...]}, ...]
+            aggregationList = []
+            for i in Counter(aggregationValues).most_common():
+                idict = {"value": i[0], "count": i[1], "events": [], "allevents": []}
+                for r in results:
+                    if (
+                        getValueByPath(r["_source"], aggregationPath).encode(
+                            "ascii", "ignore"
+                        ) == i[0]
+                    ):
+                        # copy events detail into this aggregation up to our samples limit
+                        if len(idict["events"]) < samplesLimit:
+                            idict["events"].append(r)
+                        # also copy all events to a non-sampled list
+                        # so we mark all events as alerted and don't re-alert
+                        idict["allevents"].append(r)
+                aggregationList.append(idict)
+
+            self.aggregations = aggregationList
+            self.log.debug(self.aggregations)
+        except Exception as e:
+            self.log.error("Error while searching events in ES: {0}".format(e))
+
+    def walkEvents(self, **kwargs):
+        """
+        Walk through events, provide some methods to hook in alerts
+        """
+        if len(self.events) > 0:
+            for i in self.events:
+                alert = self.onEvent(i, **kwargs)
+                if alert:
+                    alert = self.tagBotNotify(alert)
+                    self.log.debug(alert)
+                    alert = self.alertPlugins(alert)
+                    alertResultES = self.alertToES(alert)
+                    self.tagEventsAlert([i], alertResultES)
+                    self.alertToMessageQueue(alert)
+                    self.hookAfterInsertion(alert)
+                    self.saveAlertID(alertResultES)
+        # did we not match anything?
+        # can also be used as an alert trigger
+        if len(self.events) == 0:
+            alert = self.onNoEvent(**kwargs)
+            if alert:
+                alert = self.tagBotNotify(alert)
+                self.log.debug(alert)
+                alertResultES = self.alertToES(alert)
+                self.alertToMessageQueue(alert)
+                self.hookAfterInsertion(alert)
+                self.saveAlertID(alertResultES)
+
+    def walkAggregations(self, threshold, config=None):
+        """
+        Walk through aggregations, provide some methods to hook in alerts
+        """
+        if len(self.aggregations) > 0:
+            for aggregation in self.aggregations:
+                if aggregation["count"] >= threshold:
+                    aggregation["config"] = config
+                    alert = self.onAggregation(aggregation)
+                    if alert:
+                        alert = self.tagBotNotify(alert)
+                        self.log.debug(alert)
+                        alert = self.alertPlugins(alert)
+                        alertResultES = self.alertToES(alert)
+                        # even though we only sample events in the alert
+                        # tag all events as alerted to avoid re-alerting
+                        # on events we've already processed.
+                        self.tagEventsAlert(aggregation["allevents"], alertResultES)
+                        self.alertToMessageQueue(alert)
+                        self.saveAlertID(alertResultES)
+
+    def alertPlugins(self, alert):
+        """
+        Send alerts through a plugin system
+        """
+
+        plugin_dir = os.path.join(os.path.dirname(__file__), "../plugins")
+        plugin_set = AlertPluginSet(plugin_dir, ALERT_PLUGINS)
+        alertDict = plugin_set.run_plugins(alert)[0]
+
+        return alertDict
+
+    def createAlertDict(
+        self,
+        summary,
+        category,
+        tags,
+        events,
+        severity="NOTICE",
+        url=None,
+        ircchannel=None,
+    ):
+        """
+        Create an alert dict
+        """
+        alert = {
+            "utctimestamp": toUTC(datetime.now()).isoformat(),
+            "severity": severity,
+            "summary": summary,
+            "category": category,
+            "tags": tags,
+            "events": [],
+            "ircchannel": ircchannel,
+        }
+        if url:
+            alert["url"] = url
+
+        for e in events:
+            alert["events"].append(
+                {
+                    "documentindex": e["_index"],
+                    "documentsource": e["_source"],
+                    "documentid": e["_id"],
+                }
+            )
+        self.log.debug(alert)
+        return alert
+
+    def onEvent(self, event, *args, **kwargs):
+        """
+        To be overriden by children to run their code
+        to be used when creating an alert using an event
+        must return an alert dict or None
+        """
+        pass
+
+    def onNoEvent(self, *args, **kwargs):
+        """
+        To be overriden by children to run their code
+        when NOTHING matches a filter
+        which can be used to trigger on the absence of
+        events much like a dead man switch.
+        This is to be used when creating an alert using an event
+        must return an alert dict or None
+        """
+        pass
+
+    def onAggregation(self, aggregation):
+        """
+        To be overriden by children to run their code
+        to be used when creating an alert using an aggregation
+        must return an alert dict or None
+        """
+        pass
+
+    def hookAfterInsertion(self, alert):
+        """
+        To be overriden by children to run their code
+        to be used when creating an alert using an aggregation
+        """
+        pass
+
+    def tagEventsAlert(self, events, alertResultES):
+        """
+        Update the event with the alertid/index
+        and update the alert_names on the event itself so it's
+        not re-alerted
+        """
+        try:
+            for event in events:
+                if "alerts" not in event["_source"]:
+                    event["_source"]["alerts"] = []
+                event["_source"]["alerts"].append(
+                    {"index": alertResultES["_index"], "id": alertResultES["_id"]}
+                )
+
+                if "alert_names" not in event["_source"]:
+                    event["_source"]["alert_names"] = []
+                event["_source"]["alert_names"].append(self.determine_alert_classname())
+
+                self.es.save_event(
+                    index=event["_index"], body=event["_source"], doc_id=event["_id"]
+                )
+            # We refresh here to ensure our changes to the events will show up for the next search query results
+            self.es.refresh(event["_index"])
+        except Exception as e:
+            self.log.error("Error while updating events in ES: {0}".format(e))
+
+    def main(self):
+        """
+        To be overriden by children to run their code
+        """
+        pass
+
+    def run(self, *args, **kwargs):
+        """
+        Main method launched by celery periodically
+        """
+        try:
+            self.main(*args, **kwargs)
+            self.log.debug("finished")
+        except Exception as e:
+            self.log.exception("Exception in main() method: {0}".format(e))
+
+    def parse_json_alert_config(self, config_file):
+        """
+        Helper function to parse an alert config file
+        """
+        alert_dir = os.path.join(os.path.dirname(__file__), "..")
+        config_file_path = os.path.abspath(os.path.join(alert_dir, config_file))
+        json_obj = {}
+        with open(config_file_path, "r") as fd:
+            try:
+                json_obj = json.load(fd)
+            except ValueError:
+                sys.stderr.write("FAILED to open the configuration file\n")
+
+        return json_obj

cloudy_mozdef/lambda_layer/build/lib/config.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+from celery.schedules import crontab, timedelta
+import time
+import logging
+import os
+
+ALERTS = {
+    # 'pythonfile.pythonclass':{'schedule': crontab(minute='*/10')},
+    # 'pythonfile.pythonclass':{'schedule': timedelta(minutes=10),'kwargs':dict(hostlist=['nsm3', 'nsm5'])},
+}
+
+ALERT_PLUGINS = [
+    # 'relative pythonfile name (exclude the .py) - EX: sso_dashboard',
+]
+
+ALERT_ACTIONS = [
+    # 'relative pythonfile name (exclude the .py) - EX: sso_dashboard',
+]
+
+RABBITMQ = {
+    'mqserver': 'localhost',
+    'mquser': 'guest',
+    'mqpassword': 'guest',
+    'mqport': 5672,
+    'alertexchange': 'alerts',
+    'alertqueue': 'mozdef.alert'
+}
+
+if os.getenv('OPTIONS_ESSERVERS'):
+    ES = {
+        'servers': [os.getenv('OPTIONS_ESSERVERS')]
+    }
+else:
+    ES = {
+        'servers': ['http://localhost:9200']
+    }
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'formatters': {
+        'simple': {
+            'format': '%(levelname)s %(message)s',
+            'datefmt': '%y %b %d, %H:%M:%S',
+        },
+        'standard': {
+            'format': '%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d: %(message)s'
+        }
+    },
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple'
+        },
+        'celery': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'celery.log',
+            'formatter': 'standard',
+            'maxBytes': 1024 * 1024 * 100,  # 100 mb
+        },
+    },
+    'loggers': {
+        'celery': {
+            'handlers': ['celery', 'console'],
+            'level': 'INFO',
+        },
+    }
+}
+
+logging.Formatter.converter = time.gmtime

cloudy_mozdef/lambda_layer/build/lib/deadman_alerttask.py

@@ -0,0 +1,9 @@
+from alerttask import AlertTask
+
+
+class DeadmanAlertTask(AlertTask):
+
+    def executeSearchEventsSimple(self):
+        # We override this method to specify the size as 1
+        # since we only care about if ANY events are found or not
+        return self.main_query.execute(self.es, indices=self.event_indices, size=1)

cloudy_mozdef/lambda_layer/lambdalert.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+import logging
+import sys
+from lib.alerttask import AlertTask
+from mozdef_util.query_models import SearchQuery, TermMatch
+
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging():
+    logger = logging.getLogger()
+    h = logging.StreamHandler(sys.stdout)
+    logger.setLevel(logging.DEBUG)
+    return logger
+
+
+class AlertCloudtrailLoggingDisabled(AlertTask):
+    def _configureKombu(self):
+        """Override the normal behavior of this in order to run in lambda."""
+        pass
+
+    def alertToMessageQueue(self, alertDict):
+        """Override the normal behavior of this in order to run in lambda."""
+        pass
+
+    def main(self):
+        # How many minutes back in time would you like to search?
+        search_query = SearchQuery(minutes=15)
+
+        # What would you like to search for?
+        # search_query.add_must([
+        #    TermMatch('source', 'cloudtrail'),
+        #    TermMatch('details.eventname', 'DescribeTable')
+        # ])
+
+        self.filtersManual(search_query)
+        self.searchEventsSimple()
+        self.walkEvents()
+
+    def onEvent(self, event):
+        category = 'AWSCloudtrail'
+
+        # Useful tag and severity rankings for your alert.
+        tags = ['cloudtrail', 'aws', 'cloudtrailpagerduty']
+        severity = 'CRITICAL'
+
+        # What message should surface in the user interface when this fires?
+        summary = 'The alert fired!'
+
+        return self.createAlertDict(summary, category, tags, [event], severity)
+
+        # Learn more about MozDef alerts by exploring the "Alert class!"
+
+
+def handle(event, context):
+    logger = setup_logging()
+    logger.debug('Function initialized.')
+    a = AlertCloudtrailLoggingDisabled()
+    return a.main()

cloudy_mozdef/packer/packer.json

@@ -59,7 +59,7 @@
         "cd /opt/mozdef",
         "sudo git checkout {{ user `github_branch`}}",
         "sudo git rev-parse HEAD",
-        "sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sns_sqs.env docker/compose/cloudy_mozdef_kibana.env",
+        "sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sqs.env docker/compose/cloudy_mozdef_kibana.env",
         "sudo sed --in-place s/latest/{{ user `github_branch`}}/g docker/compose/docker-compose-cloudy-mozdef.yml",
         "sudo docker-compose --file docker/compose/docker-compose-cloudy-mozdef.yml --project-name mozdef pull",
         "sudo rm --recursive --force --verbose /tmp/* /home/ec2-user/.bash_history /root/.ssh /home/ec2-user/.ssh/known_hosts /home/ec2-user/.ssh/authorized_keys"

cron/createIPBlockList.py

@@ -137,12 +137,13 @@ def main():
             {"$project": {"address": 1}},
             {"$limit": options.iplimit}
         ])
-        IPList = []
+        ips = []
         for ip in ipCursor:
-            IPList.append(ip['address'])
+            ips.append(ip['address'])
+        uniq_ranges = netaddr.cidr_merge(ips)
         # to text
         with open(options.outputfile, 'w') as outputfile:
-            for ip in IPList:
+            for ip in uniq_ranges:
                 outputfile.write("{0}\n".format(ip))
         outputfile.close()
         # to s3?

cron/syncAlertsToMongo.py

@@ -26,7 +26,7 @@ def getESAlerts(es):
     # We use an ExistsMatch here just to satisfy the
     # requirements of a search query must have some "Matchers"
     search_query.add_must(ExistsMatch('summary'))
-    results = search_query.execute(es, indices=['alerts-*'], size=10000)
+    results = search_query.execute(es, indices=['alerts'], size=10000)
     return results
 
 

docker/compose/docker-compose-cloudy-mozdef.yml

@@ -1,8 +1,14 @@
 ---
-version: '2.2'
+version: '2.3'
 services:
   nginx:
-    image: mozillaiam/mozilla.oidc.accessproxy
+    image: mozdef/mozdef_cognito_proxy:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
+    command: ["/usr/bin/openresty", "-g", "daemon off;"]
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -11,23 +17,22 @@ services:
     links:
       - "meteor"
     ports:
-      - 80:80
-      - 443:80
-        #      - 8080:8080 is loginput
-        #      - 9090:9090 is kibana
-    networks:
-      - default
-  nginxkibana:
-    image: mozillaiam/mozilla.oidc.accessproxy
-    env_file:
-      - cloudy_mozdef_kibana.env
-    restart: always
-    ports:
-      - 9090:80
+      - 80:8000
+      - 9090:8090
     networks:
       - default
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost/health"]
+      interval: 1m30s
+      timeout: 10s
+      retries: 10
   mongodb:
     image: mozdef/mozdef_mongodb:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -38,6 +43,11 @@ services:
       - default
   bootstrap:
     image: mozdef/mozdef_bootstrap:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     command: bash -c 'python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/mozdefStateDefaultMappingTemplate.json cron/backup.conf http://kibana:5601'
@@ -48,6 +58,11 @@ services:
   # MozDef Specific Containers
   base:
     image: mozdef/mozdef_base:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     command: bash -c 'su - mozdef -c /opt/mozdef/envs/mozdef/cron/update_geolite_db.sh'
@@ -55,6 +70,11 @@ services:
       - geolite_db:/opt/mozdef/envs/mozdef/data
   alertactions:
     image: mozdef/mozdef_alertactions:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -67,6 +87,11 @@ services:
       - default
   alerts:
     image: mozdef/mozdef_alerts:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
       - rabbitmq.env
@@ -81,6 +106,11 @@ services:
       - default
   cron:
     image: mozdef/mozdef_cron:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -95,6 +125,11 @@ services:
       - default
   loginput:
     image: mozdef/mozdef_loginput:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -108,6 +143,11 @@ services:
       - default
   meteor:
     image: mozdef/mozdef_meteor:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -122,6 +162,11 @@ services:
       - default
   rest:
     image: mozdef/mozdef_rest:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -134,6 +179,11 @@ services:
       - default
   syslog:
     image: mozdef/mozdef_syslog:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -147,6 +197,11 @@ services:
       - default
   rabbitmq:
     image: mozdef/mozdef_rabbitmq:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - rabbitmq.env
     restart: always
@@ -160,6 +215,11 @@ services:
       - default
   mq_eventtask:
     image: mozdef/mozdef_mq_worker:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
     restart: always
@@ -176,6 +236,11 @@ services:
       - geolite_db:/opt/mozdef/envs/mozdef/data/
   mq_cloudtrail:
     image: mozdef/mozdef_mq_worker:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
       - cloudy_mozdef_mq_cloudtrail.env
@@ -193,11 +258,16 @@ services:
       - geolite_db:/opt/mozdef/envs/mozdef/data/
   mq_sqs:
     image: mozdef/mozdef_mq_worker:latest
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "1"
+        max-size: "10m"
     env_file:
       - cloudy_mozdef.env
-      - cloudy_mozdef_mq_sns_sqs.env
+      - cloudy_mozdef_mq_sqs.env
     restart: always
-    command: bash -c 'python esworker_sns_sqs.py -c esworker_sns_sqs.conf'
+    command: bash -c 'python esworker_sqs.py -c esworker_sqs.conf'
     scale: 1
     depends_on:
       - base

docker/compose/docker-compose.yml

@@ -1,54 +1,6 @@
 ---
 version: '3.7'
 services:
-  nginx:
-    image: mozdef/mozdef_nginx
-    build:
-      context: ../../
-      dockerfile: docker/compose/nginx/Dockerfile
-      cache_from:
-        - mozdef/mozdef_nginx
-        - mozdef_nginx:latest
-    restart: always
-    command: /usr/sbin/nginx
-    depends_on:
-      - kibana
-      - meteor
-    ports:
-      - 80:80
-      - 8080:8080
-      - 9090:9090
-      # - 8081:8081
-    networks:
-      - default
-  mongodb:
-    image: mozdef/mozdef_mongodb
-    build:
-      context: ../../
-      dockerfile: docker/compose/mongodb/Dockerfile
-      cache_from:
-        - mozdef/mozdef_mongodb
-        - mozdef_mongodb:latest
-    restart: always
-    command: /usr/bin/mongod --smallfiles --config /etc/mongod.conf
-    volumes:
-      - mongodb:/var/lib/mongo
-    networks:
-      - default
-  kibana:
-    image: mozdef/mozdef_kibana
-    build:
-      context: ../../
-      dockerfile: docker/compose/kibana/Dockerfile
-      cache_from:
-        - mozdef/mozdef_kibana
-        - mozdef_kibana:latest
-    restart: always
-    command: bin/kibana --elasticsearch=http://elasticsearch:9200
-    depends_on:
-      - elasticsearch
-    networks:
-      - default
   elasticsearch:
     image: mozdef/mozdef_elasticsearch
     build:
@@ -82,6 +34,56 @@ services:
     #   - 15672:15672 # Admin interface
     networks:
       - default
+  mongodb:
+    image: mozdef/mozdef_mongodb
+    build:
+      context: ../../
+      dockerfile: docker/compose/mongodb/Dockerfile
+      cache_from:
+        - mozdef/mozdef_mongodb
+        - mozdef_mongodb:latest
+    restart: always
+    command: /usr/bin/mongod --smallfiles --config /etc/mongod.conf
+    volumes:
+      - mongodb:/var/lib/mongo
+    # ports:
+    #   - 3002:3002
+    networks:
+      - default
+  kibana:
+    image: mozdef/mozdef_kibana
+    build:
+      context: ../../
+      dockerfile: docker/compose/kibana/Dockerfile
+      cache_from:
+        - mozdef/mozdef_kibana
+        - mozdef_kibana:latest
+    restart: always
+    command: bin/kibana --elasticsearch=http://elasticsearch:9200
+    depends_on:
+      - elasticsearch
+    networks:
+      - default
+  nginx:
+    image: mozdef/mozdef_nginx
+    build:
+      context: ../../
+      dockerfile: docker/compose/nginx/Dockerfile
+      cache_from:
+        - mozdef/mozdef_nginx
+        - mozdef_nginx:latest
+    restart: always
+    command: /usr/sbin/nginx
+    depends_on:
+      - kibana
+      - meteor
+    ports:
+      - 80:80
+      - 8080:8080
+      - 9090:9090
+      # - 8081:8081
+    networks:
+      - default
 
   # MozDef Specific Containers
   base:
@@ -285,7 +287,14 @@ services:
         - mozdef_tester:latest
     networks:
       - default
-
+  cognito_proxy:
+    image: mozdef/mozdef_cognito_proxy
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_cognito_proxy/Dockerfile
+    command: bash -c 'exit 0'
+    networks:
+      - default
 volumes:
   elasticsearch:
   rabbitmq:

docker/compose/mozdef_alerts/Dockerfile

@@ -4,6 +4,7 @@ LABEL maintainer="mozdef@mozilla.com"
 
 COPY alerts /opt/mozdef/envs/mozdef/alerts
 COPY docker/compose/mozdef_alerts/files/config.py /opt/mozdef/envs/mozdef/alerts/lib/
+COPY docker/compose/mozdef_alerts/files/get_watchlist.conf /opt/mozdef/envs/mozdef/alerts/get_watchlist.conf
 
 RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/alerts
 

docker/compose/mozdef_alerts/files/config.py

@@ -12,6 +12,7 @@ import logging
 ALERTS = {
     'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')},
     'unauth_ssh.AlertUnauthSSH': {'schedule': crontab(minute='*/1')},
+    'get_watchlist.AlertWatchList': {'schedule': crontab(minute='*/1')},
 }
 
 ALERT_PLUGINS = [

docker/compose/mozdef_alerts/files/get_watchlist.conf

@@ -0,0 +1,5 @@
+[options]
+# set the following to your protected endpoint api_url
+api_url = http://rest:8081/getwatchlist
+jwt_secret = secret
+use_auth = false

docker/compose/mozdef_base/Dockerfile

@@ -13,6 +13,9 @@ RUN \
                   libstdc++ \
                   libffi-devel \
                   zlib-devel \
+                  libcurl-devel \
+                  openssl \ 
+                  openssl-devel \
                   git \
                   make && \
   useradd -ms /bin/bash -d /opt/mozdef -m mozdef && \
@@ -27,6 +30,10 @@ RUN \
   mkdir /opt/mozdef/envs/mozdef && \
   mkdir /opt/mozdef/envs/mozdef/cron
 
+# Force pycurl to understand we prefer nss backend
+# Pycurl with ssl support is required by kombu in order to use SQS
+ENV PYCURL_SSL_LIBRARY=nss
+
 # Create python virtual environment and install dependencies
 COPY requirements.txt /opt/mozdef/envs/mozdef/requirements.txt
 

docker/compose/mozdef_cognito_proxy/Dockerfile

@@ -0,0 +1,5 @@
+FROM openresty/openresty:centos
+ADD docker/compose/mozdef_cognito_proxy/files/htpasswd.example /etc/nginx/htpasswd
+ADD docker/compose/mozdef_cognito_proxy/files/default.conf /etc/nginx/conf.d/default.conf
+ADD docker/compose/mozdef_cognito_proxy/files/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
+RUN /usr/local/openresty/luajit/bin/luarocks install lua-resty-jwt

docker/compose/mozdef_cognito_proxy/README.md

@@ -0,0 +1,4 @@
+# mozdef-cognito-proxy
+
+This is a quick access proxy to use with the AWS authenticating ALB.
+This docker container is untested / alpha status.

docker/compose/mozdef_cognito_proxy/files/default.conf

@@ -0,0 +1,108 @@
+server {
+    listen       8000;
+    server_name  localhost;
+    resolver 127.0.0.11;
+
+
+    location = /health {
+      return 200;
+      access_log off;
+    }
+
+
+    location / {
+        lua_need_request_body on;
+        auth_basic_user_file /etc/nginx/htpasswd;
+        set_by_lua_block $backend {return os.getenv("METEOR_BACKEND")}
+        set_by_lua_block $auth_basic_set { if os.getenv("OIDC_CLIENT_ID") == "Unset" then return "yes" else return "no" end }
+        set $auth_basic off;
+        if ($auth_basic_set = yes) {
+           set $auth_basic Restricted;
+        }
+
+        set_by_lua_block $user {
+          if os.getenv("OIDC_CLIENT_ID") == "Unset" then
+            ngx.log(ngx.NOTICE, 'OIDC authentication is not in use.  Logging in as sample user.')
+            return "mozdefuser@sample.com"
+          else
+            ngx.log(ngx.NOTICE, 'OIDC authentication in use.  Attempting to parse headers.')
+            local resp = {headers=nil, body=nil}
+            local json_safe = require "cjson.safe"
+            local jwt = require "resty.jwt"
+            resp.headers = ngx.req.get_headers()
+            if resp.headers['x-amzn-oidc-data'] ~= nil then
+              local jwt_obj = jwt:load_jwt(resp.headers['x-amzn-oidc-data'])
+              resp['amzn-oidc-data'] = jwt_obj
+              --resp.oidc = resp.headers['x-amzn-oidc-data']
+            end
+            resp.httpMethod = ngx.req.get_method()
+            resp.queryStringParameters = ngx.req.get_uri_args()
+            resp.path = ngx.var.uri
+            if resp['amzn-oidc-data'] ~= nil then
+              local email = resp['amzn-oidc-data']['payload']['email']
+              ngx.log(ngx.NOTICE, 'Via header sent to meteor')
+              return email
+            else
+              ngx.status = 403
+            end
+          end
+        }
+        auth_basic $auth_basic;
+        proxy_set_header via $user;
+        proxy_pass http://$backend;
+    }
+}
+
+server {
+    listen       8090;
+    server_name  localhost;
+    resolver 127.0.0.11;
+
+    location = /health {
+      return 200;
+      access_log off;
+    }
+
+    location / {
+        lua_need_request_body on;
+        set_by_lua_block $backend {return os.getenv("ESBACKEND")}
+        auth_basic_user_file /etc/nginx/htpasswd;
+        set_by_lua_block $auth_basic_set { if os.getenv("OIDC_CLIENT_ID") == "Unset" then return "yes" else return "no" end }
+        set $auth_basic off;
+        if ($auth_basic_set = yes) {
+           set $auth_basic Restricted;
+        }
+
+        set_by_lua_block $user {
+          if os.getenv("OIDC_CLIENT_ID") == "Unset" then
+            return "mozdefuser@sample.com"
+          else
+            local resp = {headers=nil, body=nil}
+            local json_safe = require "cjson.safe"
+            local jwt = require "resty.jwt"
+            resp.headers = ngx.req.get_headers()
+            if resp.headers['x-amzn-oidc-data'] ~= nil then
+              local jwt_obj = jwt:load_jwt(resp.headers['x-amzn-oidc-data'])
+              resp['amzn-oidc-data'] = jwt_obj
+              --resp.oidc = resp.headers['x-amzn-oidc-data']
+            end
+            resp.httpMethod = ngx.req.get_method()
+            resp.queryStringParameters = ngx.req.get_uri_args()
+            resp.path = ngx.var.uri
+            if resp['amzn-oidc-data'] ~= nil then
+              return resp['amzn-oidc-data']['payload']['email']
+            else
+              ngx.status = 403
+            end
+          end
+        }
+        proxy_set_header       Authorization "";
+        proxy_set_header via $user;
+        auth_basic $auth_basic;
+        proxy_pass $backend;
+    }
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/local/openresty/nginx/html;
+    }
+}

docker/compose/mozdef_cognito_proxy/files/htpasswd.example

@@ -0,0 +1 @@
+mozdef:$apr1$gXiYI6mD$lsgYLnKtsHWfQJep/aCnh.

docker/compose/mozdef_cognito_proxy/files/nginx.conf

@@ -0,0 +1,16 @@
+worker_processes  1;
+env OIDC_CLIENT_ID;
+env METEOR_BACKEND;
+env ESBACKEND;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+    sendfile        on;
+    keepalive_timeout  65;
+    include /etc/nginx/conf.d/*.conf;
+}

docker/compose/mozdef_rest/files/index.conf

@@ -1,5 +1,4 @@
 [options]
-kibanaurl=http://localhost:9090/app/kibana
 esservers=http://elasticsearch:9200
 mongohost=mongodb
 mongoport=3002

meteor/.meteor/versions

@@ -98,7 +98,7 @@ templating-compiler@1.3.3
 templating-runtime@1.3.2
 templating-tools@1.1.2
 tracker@1.2.0
-twbs:bootstrap@3.3.6
+twbs:bootstrap@3.3.5
 ui@1.0.13
 underscore@1.0.10
 url@1.2.0

meteor/client/alertdetails.js

@@ -17,7 +17,7 @@ if (Meteor.isClient) {
 
         kibanaurl: function () {
             var esmetadata = alerts.findOne({'esmetadata.id': Session.get('alertID')}).esmetadata;
-            url=getSetting('kibanaURL') + '#/doc/alerts-*/' + esmetadata.index + '/doc?id=' + esmetadata.id;
+            url=resolveKibanaURL(getSetting('kibanaURL')) + '#/doc/alerts-*/' + esmetadata.index + '/doc?id=' + esmetadata.id;
             return url;
         }
     });

meteor/client/alertssummary.html

@@ -96,7 +96,7 @@ Copyright (c) 2014 Mozilla Corporation
     <tr class="alert-row">
         <td>{{utctimestamp}}</td>
         <td><a href="/alert/{{esmetadata.id}}">mozdef</a><br>
-            <a href="{{mozdef.kibanaURL}}#/doc/alerts-*/{{esmetadata.index}}/doc?id={{esmetadata.id}}" target="_blank">kibana</a>
+            <a href="{{ resolveKibanaURL mozdef.kibanaURL }}#/doc/alerts-*/{{esmetadata.index}}/doc?id={{esmetadata.id}}" target="_blank">kibana</a>
             {{#if url}}
                 <br><a href="{{url}}" target ="_blank">docs</a>
             {{/if}}

meteor/client/greeting.html

@@ -0,0 +1,8 @@
+<!--a nice greeting-->
+<template name="hello">
+    <div class="container">
+        <div class="row center">
+            <p class="welcome"><img class="mozdeflogo" src="/images/moz_defense-platform_01.png"><br/> {{greeting}}</p>
+        </div>
+    </div>
+</template>

meteor/client/greeting.js

@@ -0,0 +1,17 @@
+if ( Meteor.isClient ) {
+
+    Template.hello.helpers( {
+        greeting: function() {
+            if ( typeof console !== 'undefined' )
+                console.log( "mozdef starting" );
+            return "Hand made by Mozilla";
+        }
+    } );
+
+    Template.hello.events( {
+        'click': function() {
+            // template data, if any, is available in 'this'
+            Session.set( 'displayMessage', 'Welcome to mozdef.' )
+        }
+    } );
+};

meteor/client/js/jquery.highlight.js

@@ -1,108 +0,0 @@
-/*
- * jQuery Highlight plugin
- *
- * Based on highlight v3 by Johann Burkard
- * http://johannburkard.de/blog/programming/javascript/highlight-javascript-text-higlighting-jquery-plugin.html
- *
- * Code a little bit refactored and cleaned (in my humble opinion).
- * Most important changes:
- *  - has an option to highlight only entire words (wordsOnly - false by default),
- *  - has an option to be case sensitive (caseSensitive - false by default)
- *  - highlight element tag and class names can be specified in options
- *
- * Usage:
- *   // wrap every occurrance of text 'lorem' in content
- *   // with <span class='highlight'> (default options)
- *   $('#content').highlight('lorem');
- *
- *   // search for and highlight more terms at once
- *   // so you can save some time on traversing DOM
- *   $('#content').highlight(['lorem', 'ipsum']);
- *   $('#content').highlight('lorem ipsum');
- *
- *   // search only for entire word 'lorem'
- *   $('#content').highlight('lorem', { wordsOnly: true });
- *
- *   // don't ignore case during search of term 'lorem'
- *   $('#content').highlight('lorem', { caseSensitive: true });
- *
- *   // wrap every occurrance of term 'ipsum' in content
- *   // with <em class='important'>
- *   $('#content').highlight('ipsum', { element: 'em', className: 'important' });
- *
- *   // remove default highlight
- *   $('#content').unhighlight();
- *
- *   // remove custom highlight
- *   $('#content').unhighlight({ element: 'em', className: 'important' });
- *
- *
- * Copyright (c) 2009 Bartek Szopka
- *
- * Licensed under MIT license.
- *
- */
-
-jQuery.extend({
-    highlight: function (node, re, nodeName, className) {
-        if (node.nodeType === 3) {
-            var match = node.data.match(re);
-            if (match) {
-                var highlight = document.createElement(nodeName || 'span');
-                highlight.className = className || 'highlight';
-                var wordNode = node.splitText(match.index);
-                wordNode.splitText(match[0].length);
-                var wordClone = wordNode.cloneNode(true);
-                highlight.appendChild(wordClone);
-                wordNode.parentNode.replaceChild(highlight, wordNode);
-                return 1; //skip added node in parent
-            }
-        } else if ((node.nodeType === 1 && node.childNodes) && // only element nodes that have children
-                !/(script|style)/i.test(node.tagName) && // ignore script and style nodes
-                !(node.tagName === nodeName.toUpperCase() && node.className === className)) { // skip if already highlighted
-            for (var i = 0; i < node.childNodes.length; i++) {
-                i += jQuery.highlight(node.childNodes[i], re, nodeName, className);
-            }
-        }
-        return 0;
-    }
-});
-
-jQuery.fn.unhighlight = function (options) {
-    var settings = { className: 'highlight', element: 'span' };
-    jQuery.extend(settings, options);
-
-    return this.find(settings.element + "." + settings.className).each(function () {
-        var parent = this.parentNode;
-        parent.replaceChild(this.firstChild, this);
-        parent.normalize();
-    }).end();
-};
-
-jQuery.fn.highlight = function (words, options) {
-    var settings = { className: 'highlight', element: 'span', caseSensitive: false, wordsOnly: false };
-    jQuery.extend(settings, options);
-    
-    if (words.constructor === String) {
-        words = [words];
-    }
-    words = jQuery.grep(words, function(word, i){
-      return word != '';
-    });
-    words = jQuery.map(words, function(word, i) {
-      return word.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
-    });
-    if (words.length == 0) { return this; };
-
-    var flag = settings.caseSensitive ? "" : "i";
-    var pattern = "(" + words.join("|") + ")";
-    if (settings.wordsOnly) {
-        pattern = "\\b" + pattern + "\\b";
-    }
-    var re = new RegExp(pattern, flag);
-    
-    return this.each(function () {
-        jQuery.highlight(this, re, settings.element, settings.className);
-    });
-};
-

meteor/client/layout.js

@@ -52,7 +52,7 @@ if (Meteor.isClient) {
             Session.set('ipsearchipaddress',($(e.target).attr('data-ipaddress')));
             var ipText=$(e.target).attr('data-ipaddress');
             //console.log("IP: " + ipText)
-            var searchDomain=getSetting('kibanaURL');
+            var searchDomain=resolveKibanaURL(getSetting('kibanaURL'));
             var searchPath="#/discover?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-1h,mode:quick,to:now))&_a=(columns:!(_source),index:events-weekly,interval:auto,query:(query_string:(analyze_wildcard:!t,query:'details.sourceipaddress:"+ipText+"')),sort:!(utctimestamp,desc))";
             var url=searchDomain+searchPath;
             console.log("Opening url: " + url);

meteor/client/loading.html

@@ -0,0 +1,3 @@
+<template name='loading'>
+        loading...
+</template>

meteor/client/main.js

@@ -5,11 +5,16 @@ import { Mongo } from 'meteor/mongo';
 import { Session } from 'meteor/session';
 import { _ } from 'meteor/underscore';
 import { Blaze } from 'meteor/blaze';
+import '/client/loading.html';
+import '/client/greeting.html';
+import '/client/greeting.js';
 import '/imports/settings.js';
 import '/imports/collections.js';
 import '/imports/helpers.js';
 import '/imports/models.js';
-import '/client/about.html';
+import '/client/mozdef.html';
+import '/client/layout.js';
+import '/client/router.js';
 import '/client/alertdetails.html';
 import '/client/alertdetails.js';
 import '/client/alertssummary.html';
@@ -54,11 +59,12 @@ import '/client/logincounts.html';
 import '/client/logincounts.js';
 import '/client/mozdefhealth.html';
 import '/client/mozdefhealth.js';
+import '/client/about.html';
 import '/client/nameplate.html';
 import '/client/nameplate.js';
 import '/client/verisTags.html';
 import '/client/preferences.html';
 import '/client/preferences.js'
-import '/client/router.js';
+import '/public/css/dropdowns.css';
 import '/client/mozdef.js';
 

meteor/client/menu.html

@@ -11,8 +11,7 @@ Copyright (c) 2014 Mozilla Corporation
         <div id="header" class="row center">
             <span id="nav-main">
                 <ul>
-                    <li><img class="mozillalogo" src="/images/mozilla.svg"></li>
-                    <li><a class="mozdef" href="/" title="MOZDEF">MOZDEF</a></li>
+                    <li><a href="/">Home</a></li>
                     {{#if isFeature "kibana"}}
                         <li> <a target="_blank" href={{ resolveKibanaURL mozdef.kibanaURL }}>Kibana</a>
                             <ul>
@@ -94,7 +93,7 @@ Copyright (c) 2014 Mozilla Corporation
 
 <template name="kibanaDashboardItem">
     <li>
-        <a target=" _blank" href="{{url}}">{{name}}</a>
+        <a target=" _blank" href="{{ resolveKibanaURL mozdef.kibanaURL }}#/dashboard/{{id}}">{{name}}</a>
     </li>
 </template>
 

meteor/client/menu.js

@@ -2,31 +2,20 @@ import { Meteor } from 'meteor/meteor';
 import { Template } from 'meteor/templating';
 import { Tracker } from 'meteor/tracker'
 
-Template.menu.rendered = function () {
-    Tracker.autorun(function() {
-        Meteor.subscribe("features");
-    });
+Template.menu.rendered = function() {
+    Tracker.autorun( function() {
+        Meteor.subscribe( "features" );
+    } );
 };
 
-Template.menu.helpers({
-    haveFeatures: function(){
+Template.menu.helpers( {
+    haveFeatures: function() {
         //subscription has records?
-        return features.find().count() >0;
+        return features.find().count() > 0;
     },
-    resolveKibanaURL: function(url){
-        // special function just for the menu
-        // to adjust the kibana URL if we are told to make it 'relative'
-        // to whatever DNS name we are running on
-        // i.e. pass in http://relative:9090/app/kibana
-        // when the running dns is something.com
-        // and we will set the hostname to something.com instead of 'relative'
-        var kibanaURL = new URL(url);
-        if ( kibanaURL.hostname == 'relative' ){
-            // we were passed something like  OPTIONS_METEOR_KIBANAURL=http://relative:9090/app/kibana
-            // so lets figure out where we should be
-            dnsURL=new URL(document.URL);
-            kibanaURL.hostname = dnsURL.hostname;
-        }
-        return kibanaURL;
+    // loads kibana dashboards
+    kibanadashboards: function() {
+        Meteor.call( 'loadKibanaDashboards' );
+        return kibanadashboards.find();
     }
-  });
+} );

meteor/client/mozdef.html

@@ -8,7 +8,7 @@ Copyright (c) 2014 Mozilla Corporation
 <head>
     <meta name="viewport" content="width=1024">
     <link rel="shortcut icon" type="image/svg" href="/images/favicon.ico" />
-    <title>mozdef::mozilla defense platform</title>
+    <title>mozdef::mozilla enterprise defense platform</title>
 </head>
 
 <body>
@@ -17,52 +17,27 @@ Copyright (c) 2014 Mozilla Corporation
 <!--begin layout templates-->
 <!--the main, simple layout for the router to target-->
 <template name="layout">
-    {{> menu}}
-
     {{#if loggingIn}}
         <div class="row center">
-            <div class="col-xs-5 col-xs-offset-3 alert alert-info alert-dismissible" role="alert">
-                <button type="button" class="close" data-dismiss="alert">
-                    <span aria-hidden="true">&times;</span>
-                    <span class="sr-only">Close</span>
-                </button>
+            <div class="col-xs-5 col-xs-offset-3 alert alert-info" role="alert">
                 <strong>loading</strong>
             </div>
         </div>
     {{else}}
         {{#if currentUser}}
-            {{>yield}}
+            {{> Template.dynamic template=menuName }}
+            {{> yield }}
+            {{> whoismodal}}
+            {{> dshieldmodal}}
+            {{> blockIPModal}}
+            {{> intelmodal}}
+            {{> blockFQDNModal}}
+            {{> watchItemModal}}
         {{else}}
-            <div class="row center">
-                <div class="col-xs-5 col-xs-offset-3 alert alert-info alert-dismissible" role="alert">
-                    <button type="button" class="close" data-dismiss="alert">
-                        <span aria-hidden="true">&times;</span>
-                        <span class="sr-only">Close</span>
-                    </button>
-                    <strong>Please login</strong>
-                </div>
-            </div>
+            {{> loginmenu}}
         {{/if}}
     {{/if}}
-
-    {{>whoismodal}}
-    {{>dshieldmodal}}
-    {{>blockIPModal}}
-    {{>intelmodal}}
-    {{>blockFQDNModal}}
-    {{>watchItemModal}}
-
 </template>
 
-<!--a nice greeting-->
-<template name="hello">
-    <div class="container">
-        <div class="row center">
-            <p class="welcome">{{greeting}}<br>Hand made by Mozilla</p>
-        </div>
-    </div>
-</template>
 
-<template name='loading'>
-    loading...
-</template>
+

meteor/client/mozdef.js

@@ -5,20 +5,9 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
 Copyright (c) 2014 Mozilla Corporation
 */
 import { Meteor } from 'meteor/meteor';
-import { Template } from 'meteor/templating';
 import validator from 'validator';
-import '/imports/collections.js';
-import '/imports/settings.js';
-import '/imports/helpers.js';
-import '/client/js/jquery.highlight.js';
-import PNotify from 'pnotify';
 import 'pnotify/dist/pnotify.css';
-import './mozdef.html';
-import './menu.html';
-import './menu.js';
-import '/client/layout.js';
-import '/public/css/dropdowns.css';
-
+import PNotify from 'pnotify';
 
 
 if ( Meteor.isClient ) {
@@ -35,9 +24,9 @@ if ( Meteor.isClient ) {
         Session.set( 'blockIPipaddress', '' );
         Session.set( 'blockFQDN', '' );
         Session.set( 'watchItemwatchcontent', '' );
+        Session.set( 'menuname', 'menu' );
         getAllPlugins();
-        // use a default theme, overridden later by login per user
-        require( '/imports/themes/classic/mozdef.css' );
+
     } );
 
     prefs = function() {
@@ -170,33 +159,14 @@ if ( Meteor.isClient ) {
         return result
     };
 
-    Template.hello.helpers( {
-        greeting: function() {
-            if ( typeof console !== 'undefined' )
-                console.log( "mozdef starting" );
-            return "MozDef: The Mozilla Defense Platform";
-        }
-    } );
-
-    Template.hello.events( {
-        'click': function() {
-            // template data, if any, is available in 'this'
-            Session.set( 'displayMessage', 'Welcome & to mozdef.' )
-        }
-    } );
-
-    // loads kibana dashboards
-    Template.menu.helpers( {
-        kibanadashboards: function() {
-            Meteor.call( 'loadKibanaDashboards' );
-            return kibanadashboards.find();
-        }
-    } );
-
     UI.registerHelper( 'isFeature', function( featureName ) {
         return isFeature( featureName );
     } );
 
+    UI.registerHelper( 'resolveKibanaURL', function(url){
+        return resolveKibanaURL(url);
+    });
+
     UI.registerHelper( 'uiDateFormat', function( adate ) {
         return dateFormat( adate );
     } );
@@ -299,6 +269,19 @@ if ( Meteor.isClient ) {
         return pluginsForEndPoint( endpoint );
     } );
 
+    jQuery.fn.highlight = function (str, className) {
+        var regex = new RegExp(str, "gi");
+        return this.each(function () {
+            $(this).contents().filter(function() {
+                return this.nodeType == 3 && regex.test(this.nodeValue);
+            }).replaceWith(function() {
+                return (this.nodeValue || "").replace(regex, function(match) {
+                    return "<span class=\"" + className + "\">" + match + "</span>";
+                });
+            });
+        });
+    };
+
     UI.registerHelper( 'ipDecorate', function( elementText ) {
         //decorate text containing an ipv4 address
         var anelement = $( $.parseHTML( '<span>' + elementText + '</span>' ) )
@@ -307,23 +290,9 @@ if ( Meteor.isClient ) {
             //clean up potential interference chars
             w = w.replace( /,|:|;|\[|\]/g, '' )
             if ( isIPv4( w ) ) {
-                //console.log(w);
-                anelement.
-                    highlight( w,
-                        {
-                            wordsOnly: false,
-                            element: "em",
-                            className: "ipaddress"
-                        } );
+                anelement.highlight(w, 'ipaddress');
             } else if ( isHostname( w ) ) {
-                //console.log(w);
-                anelement.
-                    highlight( w,
-                        {
-                            wordsOnly: false,
-                            element: "em",
-                            className: "hostname"
-                        } );
+                anelement.highlight(w, 'hostname');
             }
         } );
         //add a drop down menu to any .ipaddress
@@ -360,7 +329,7 @@ if ( Meteor.isClient ) {
         anelement.children( '.hostname' ).each( function( index ) {
             hosttext = $( this ).text();
             $( this ).append( '<b></b>' );
-            var searchDomain = getSetting( 'kibanaURL' );
+            var searchDomain = resolveKibanaURL(getSetting( 'kibanaURL' ));
             searchPath = "#/discover?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-1h,mode:quick,to:now))&_a=(columns:!(_source),index:events-weekly,interval:auto,query:(query_string:(analyze_wildcard:!t,query:'hostname:" + hosttext + "')),sort:!(utctimestamp,desc))"
             searchURL = searchDomain + searchPath;
             $( this ).wrap( "<a href=" + searchURL + " target='_blank'></a>" );
@@ -380,6 +349,10 @@ if ( Meteor.isClient ) {
         }
     } )
 
+    UI.registerHelper( 'menuName', function() {
+        return Session.get( 'menuname' );
+    } )
+
     //Notify messages for the UI
     Deps.autorun( function() {
         //set Session.set('displayMessage','title&text')
@@ -498,12 +471,18 @@ if ( Meteor.isClient ) {
 
                 } else {
                     //console.log( 'client found preferences', preferenceRecord );
-
                     // import the preferred theme elements
+                    // html must be 'imported' from somewhere other than the 'import'
+                    // directory (hence the duplicate themes directory)
                     if ( preferenceRecord.theme == 'Dark' ) {
                         require( '/imports/themes/dark/mozdef.css' );
                     } else if ( preferenceRecord.theme == 'Light' ) {
                         require( '/imports/themes/light/mozdef.css' )
+                    } else if ( preferenceRecord.theme == 'Dark Side Nav' ) {
+                        require( '/client/themes/side_nav_dark/menu.html' )
+                        require( '/imports/themes/side_nav_dark/menu.js' )
+                        Session.set( 'menuname', 'side_nav_menu' );
+                        require( '/imports/themes/side_nav_dark/mozdef.css' );
                     } else {
                         require( '/imports/themes/classic/mozdef.css' );
                     }
@@ -511,4 +490,11 @@ if ( Meteor.isClient ) {
             } );
 
     } );
+
+    // finally,load the default starting point
+    // use a default theme and menu, overridden later by login per user preference
+    require( '/client/themes/none/menu-start.html' );
+    require( '/client/themes/none/menu-start.css' );
+    require( '/client/menu.html' );
+    require( '/client/menu.js' );
 }

meteor/client/mozdefhealth.js

@@ -5,119 +5,117 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
 Copyright (c) 2014 Mozilla Corporation
  */
 import { Template } from 'meteor/templating';
-import '/imports/collections.js';
-import '/imports/settings.js';
-import '/imports/helpers.js';
-import '/client/router.js';
-import '/client/mozdef.js';
 import crossfilter from 'crossfilter2';
 import dc from 'dc';
 import 'dc/dc.css';
 import { tooltip } from 'meteor/twbs:bootstrap';
 
-if (Meteor.isClient) {
+if ( Meteor.isClient ) {
 
     //elastic search cluster template functions
     //return es health items
-    Template.mozdefhealth.helpers({
+    Template.mozdefhealth.helpers( {
 
-        esclusterhealthitems: function () {
+        esclusterhealthitems: function() {
             return healthescluster.find();
         },
-        frontendhealthitems: function () {
-            return healthfrontend.find({},
-                                   {fields:{},
-                                    sort: {hostname: 1}
-                                   });
+        frontendhealthitems: function() {
+            return healthfrontend.find( {},
+                {
+                    fields: {},
+                    sort: { hostname: 1 }
+                } );
         },
-        sqsstatsitems: function () {
-            return sqsstats.find({},
-                             {fields:{},
-                              sort: {hostname: 1}
-                             });
+        sqsstatsitems: function() {
+            return sqsstats.find( {},
+                {
+                    fields: {},
+                    sort: { hostname: 1 }
+                } );
         },
-        esnodeshealthitems: function () {
-            return healthesnodes.find({},
-                                  {fields:{},
-                                  sort: {hostname: 1}
-                                  });
+        esnodeshealthitems: function() {
+            return healthesnodes.find( {},
+                {
+                    fields: {},
+                    sort: { hostname: 1 }
+                } );
         },
 
-        eshotthreadshealthitems: function () {
+        eshotthreadshealthitems: function() {
             return healtheshotthreads.find();
         }
-    });
+    } );
 
-   Template.mozdefhealth.rendered = function () {
-        var ringChartEPS   = dc.pieChart("#ringChart-EPS");
-        var totalEPS   = dc.numberDisplay("#total-EPS");
-        var ringChartLoadAverage = dc.pieChart("#ringChart-LoadAverage");
+    Template.mozdefhealth.rendered = function() {
+        var ringChartEPS = dc.pieChart( "#ringChart-EPS" );
+        var totalEPS = dc.numberDisplay( "#total-EPS" );
+        var ringChartLoadAverage = dc.pieChart( "#ringChart-LoadAverage" );
 
-        refreshChartData=function(){
-            var frontEndData=healthfrontend.find({}).fetch();
-            var ndx = crossfilter(frontEndData);
+        refreshChartData = function() {
+            var frontEndData = healthfrontend.find( {} ).fetch();
+            var ndx = crossfilter( frontEndData );
 
-            if ( frontEndData.length === 0 && ndx.size()>0){
-                debugLog('clearing ndx/dc.js');
+            if ( frontEndData.length === 0 && ndx.size() > 0 ) {
+                debugLog( 'clearing ndx/dc.js' );
                 dc.filterAll();
                 ndx.remove();
                 dc.redrawAll();
             } else {
-                ndx = crossfilter(frontEndData);
+                ndx = crossfilter( frontEndData );
             }
-            if ( ndx.size() >0){
-                var hostDim  = ndx.dimension(function(d) {return d.hostname;});
-                var hostEPS = hostDim.group().reduceSum(function(d) {return d.details.total_deliver_eps.toFixed(2);});
-                var hostLoadAverage = hostDim.group().reduceSum(function(d) {return d.details.loadaverage[0];});
-                var epsTotal = ndx.groupAll().reduceSum(function(d) {return d.details.total_deliver_eps;});
+            if ( ndx.size() > 0 ) {
+                var hostDim = ndx.dimension( function( d ) { return d.hostname; } );
+                var hostEPS = hostDim.group().reduceSum( function( d ) { return d.details.total_deliver_eps.toFixed( 2 ); } );
+                var hostLoadAverage = hostDim.group().reduceSum( function( d ) { return d.details.loadaverage[0]; } );
+                var epsTotal = ndx.groupAll().reduceSum( function( d ) { return d.details.total_deliver_eps; } );
 
                 totalEPS
-                    .valueAccessor(function(d){return d;})
-                    .group(epsTotal);
+                    .valueAccessor( function( d ) { return d; } )
+                    .group( epsTotal );
 
                 ringChartEPS
-                    .width(150).height(150)
-                    .dimension(hostDim)
-                    .group(hostEPS)
-                    .label(function(d) {return d.value ||''; })
-                    .innerRadius(30)
-                    .filter = function() {};
+                    .width( 150 ).height( 150 )
+                    .dimension( hostDim )
+                    .group( hostEPS )
+                    .label( function( d ) { return d.value || ''; } )
+                    .innerRadius( 30 )
+                    .filter = function() { };
 
                 ringChartLoadAverage
-                    .width(150).height(150)
-                    .dimension(hostDim)
-                    .group(hostLoadAverage)
-                    .label(function(d) {return d.value ||''; })
-                    .innerRadius(30)
-                    .filter = function() {};
+                    .width( 150 ).height( 150 )
+                    .dimension( hostDim )
+                    .group( hostLoadAverage )
+                    .label( function( d ) { return d.value || ''; } )
+                    .innerRadius( 30 )
+                    .filter = function() { };
 
                 dc.renderAll();
             }
         }
 
-        Deps.autorun(function() {
-            Meteor.subscribe("healthfrontend",onReady=function(){
+        Deps.autorun( function() {
+            Meteor.subscribe( "healthfrontend", onReady = function() {
                 refreshChartData();
-            });
-            Meteor.subscribe("sqsstats");
-            Meteor.subscribe("healthescluster");
-            Meteor.subscribe("healthesnodes");
-            Meteor.subscribe("healtheshotthreads");
+            } );
+            Meteor.subscribe( "sqsstats" );
+            Meteor.subscribe( "healthescluster" );
+            Meteor.subscribe( "healthesnodes" );
+            Meteor.subscribe( "healtheshotthreads" );
             //using dc.js doesn't trigger the reactive update
             //so update a UI object and refresh dc.js so both get data when it updates.
             var obj = healthfrontend.findOne();
-            if (obj) {
-               $('.lastupdate').text('Last Update: ' + obj.utctimestamp);
-               refreshChartData();
+            if ( obj ) {
+                $( '.lastupdate' ).text( 'Last Update: ' + obj.utctimestamp );
+                refreshChartData();
             }
-        }); //end deps.autorun
+        } ); //end deps.autorun
 
-        this.$('[data-toggle="tooltip"]').tooltip({
+        this.$( '[data-toggle="tooltip"]' ).tooltip( {
             'placement': 'top'
-        });
-     };
+        } );
+    };
 
-    Template.mozdefhealth.destroyed = function () {
+    Template.mozdefhealth.destroyed = function() {
         dc.deregisterAllCharts();
     };
 }

meteor/client/preferences.html

@@ -35,6 +35,7 @@
                                     <option {{ isselected 'Classic' theme }}>Classic</option>
                                     <option {{ isselected 'Dark' theme }}>Dark</option>
                                     <option {{ isselected 'Light' theme }}>Light</option>
+                                    <option {{ isselected 'Dark Side Nav' theme }}>Dark Side Nav</option>
                                 </select>
                             </div>
                         </div>

meteor/client/themes/classic/menu.html

@@ -0,0 +1,99 @@
+<!--
+This Source Code Form is subject to the terms of the Mozilla Public
+License, v. 2.0. If a copy of the MPL was not distributed with this
+file, You can obtain one at http://mozilla.org/MPL/2.0/.
+Copyright (c) 2014 Mozilla Corporation
+-->
+
+<template name="menu">
+    <div class="container headercontainer">
+        {{#if haveFeatures }}
+        <div id="header" class="row center">
+            <span id="nav-main">
+                <ul>
+                    <li><a href="/">Home</a></li>
+                    {{#if isFeature "kibana"}}
+                        <li> <a target="_blank" href={{ resolveKibanaURL mozdef.kibanaURL }}>Kibana</a>
+                            <ul>
+                                <li>
+                                    {{#each kibanadashboards as item}}
+                                        {{>kibanaDashboardItem}}
+                                    {{/each}}
+                                </li>
+                            </ul>
+                        </li>
+                    {{/if}}
+                    {{#if isFeature "alerts"}}
+                        <li><a href="/alerts/">Alerts</a>
+                            <ul>
+                                {{#if isFeature "watchlist"}}
+                                    <li><a href="/watchlist">watchlist</a></li>
+                                {{/if}}
+                                {{#if isFeature "ipblocklist"}}
+                                    <li><a href="/ipblocklist">ip blocklist</a></li>
+                                {{/if}}
+                                {{#if isFeature "fqdnblocklist"}}
+                                    <li><a href="/fqdnblocklist">fqdn blocklist</a></li>
+                                {{/if}}
+                            </ul>
+                        </li>
+                    {{/if}}
+                    {{#if isFeature "investigations"}}
+                        <li><a href="/investigations">Investigations</a>
+                            <ul>
+                                <li><a href="/investigation/new/">new investigation</a></li>
+                            </ul>
+                        </li>
+                    {{/if}}
+                    {{#if isFeature "incidents"}}
+                        <li><a href="/incidents">Incidents</a>
+                            <ul>
+                                <li><a href="/incident/new/">new incident</a></li>
+                                <li><a href="/incidents/veris/">veris stats</a></li>
+                            </ul>
+                        </li>
+                    {{/if}}
+                    {{#if isFeature "visualizations"}}
+                        <li><a href="#">Visualizations</a>
+                            <ul>
+                                {{#if isFeature "logincounts"}}
+                                    <li><a href="/logincounts/">logincounts</a></li>
+                                {{/if}}
+                                {{#if isFeature "attackers"}}
+                                    <li><a href="/attackers">attackers</a></li>
+                                {{/if}}
+                                {{#if isFeature "globe"}}
+                                    <li><a href="/globe">globe</a></li>
+                                {{/if}}
+                            </ul>
+                        </li>
+                    {{/if}}
+                    {{#if isFeature "about"}}
+                        <li><a href="/about">About</a>
+                            <ul>
+                                <li><a target="_blank" href="https://mozdef.readthedocs.io/en/latest/index.html">Documentation</a></li>
+                                <li><a href="/preferences">Preferences</a></li>
+                            </ul>
+                        </li>
+                    {{/if}}
+                </ul>
+                <label>
+                    {{#if loggingIn}}
+                    loading
+                    {{/if}}
+                    {{> loginButtons align="right"}}
+                </label>
+
+            </span>
+
+        </div>
+        {{/if}}
+    </div>
+</template>
+
+<template name="kibanaDashboardItem">
+    <li>
+        <a target=" _blank" href="{{url}}">{{name}}</a>
+    </li>
+</template>
+

meteor/client/themes/none/menu-start.css

@@ -0,0 +1,42 @@
+/* default start/login css */
+:root {
+    --bg-primary-color: #205799;
+    --bg-secondary-color: #444444;
+    --txt-primary-color: #fff;
+    --txt-secondary-color: #000;
+    --txt-shadow-color:  #000;
+    --txt-highlight-color: rgba(165, 170, 172, 0.904);
+    --a-link-color:  rgb(245, 222, 179);
+    --row-color-odd: rgba(30,87,153,.7);
+    --row-color-even: #636c85;
+  }
+
+html{
+	background: none;
+	min-height: 100%;
+}
+
+body{
+    background: var(--bg-primary-color);
+    padding: 0;
+    color: var(--txt-primary-color);
+    line-height: normal;
+    text-align: left;
+}
+
+.col-xs-offset-3 {
+  margin-left: 28%;
+}
+
+.welcome {
+  height: 180px;
+  width: 800px;
+  margin-left: 25%;
+  color: var(--txt-primary-color);
+  vertical-align: middle;
+}
+
+.mozdeflogo{
+  padding-top: 30%;
+  width: 100%;
+}

meteor/client/themes/none/menu-start.html

@@ -0,0 +1,17 @@
+<!--
+This Source Code Form is subject to the terms of the Mozilla Public
+License, v. 2.0. If a copy of the MPL was not distributed with this
+file, You can obtain one at http://mozilla.org/MPL/2.0/.
+Copyright (c) 2014 Mozilla Corporation
+-->
+
+
+<template name="loginmenu">
+    <div class="row center">
+        <div class="col-xs-5 col-xs-offset-3 alert alert-warning" role="alert">
+            <strong class="center">Please Login: </strong>
+            {{> loginButtons align="right"}}
+        </div>
+    </div>
+    <div><p class="welcome"><img class="mozdeflogo" src="/images/moz_defense-platform_01.png"></p></div>
+</template>

meteor/client/themes/side_nav_dark/menu.html

@@ -0,0 +1,180 @@
+<!--
+This Source Code Form is subject to the terms of the Mozilla Public
+License, v. 2.0. If a copy of the MPL was not distributed with this
+file, You can obtain one at http://mozilla.org/MPL/2.0/.
+Copyright (c) 2014 Mozilla Corporation
+-->
+
+
+<template name="side_nav_menu">
+    <div class="container itemcontainer">
+      <nav class="main-menu">
+      {{#if true }}
+        <ul>
+        <li>
+          <a href="https://mozilla.org">
+            <span class="moz">
+              <img src="/images/moz-logo2.png" width="30" height="30">
+            </span>
+            <span class="nav-text">
+              Mozilla
+            </span>
+          </a>
+        </li>
+        <li>
+          <a href="/">
+            <i class="fa fa-home fa-lg"></i>
+            <span class="nav-text">
+              Home
+            </span>
+          </a>
+        </li>
+        {{#if isFeature "kibana"}}
+        <li>
+          <a target="_blank" href={{ resolveKibanaURL mozdef.kibanaURL }}>
+            <span class="moz">
+            <img src="/images/logo-elastic-kibana-dk.svg" width="23" height="20">
+            </span>
+            <span class="nav-text">
+              Kibana
+            </span>
+          </a>
+          <ul>
+            <li class="has-subnav">
+              {{#each kibanadashboards as item}}
+                {{>kibanaDashboardItem}}
+              {{/each}}
+            </li>
+          </ul>
+        </li>
+        {{/if}}
+        {{#if isFeature "alerts"}}
+        <li class="has-subnav">
+          <a href="/alerts/">
+            <i class="fa fa-list fa-lg"></i>
+            <span class="nav-text">
+              Alerts
+            </span>
+          </a>
+          <ul>
+              <li class="has-subnav">
+              {{#if isFeature "watchlist"}}
+              <li><a href="/watchlist">watchlist</a></li>
+              {{/if}}
+              {{#if isFeature "ipblocklist"}}
+              <li>
+                <a href="/ipblocklist">ip blocklist</a>
+              </li>
+              {{/if}}
+              {{#if isFeature "fqdnblocklist"}}
+              <li>
+                <a href="/fqdnblocklist">fqdn blocklist</a>
+              </li>
+              {{/if}}
+            </li>
+          </ul>
+        </li>
+        {{/if}}
+        {{#if isFeature "investigations"}}
+        <li class="has-subnav">
+          <a href="/investigations">
+            <i class="fa fa-crosshairs fa-lg"></i>
+            <span class="nav-text">
+              Investigations
+            </span>
+          </a>
+          <ul>
+              <li class="has-subnav">
+                <li>
+                  <a href="/investigation/new/">new investigation</a>
+                </li>
+              </li>
+          </ul>
+        </li>
+        {{/if}}
+        {{#if isFeature "incidents"}}
+        <li class="has-subnav">
+          <a href="/incidents">
+            <i class="fa fa-exclamation-circle fa-lg"></i>
+            <span class="nav-text">
+              Incidents
+            </span>
+          </a>
+          <ul>
+            <li class="has-subnav">
+              <li>
+                <a href="/incident/new/">new incident</a>
+              </li>
+              <li>
+                <a href="/incidents/veris/">veris stats</a>
+              </li>
+            </li>
+          </ul>
+        </li>
+        {{/if}}
+        {{#if isFeature "visualizations"}}
+        <li class="has-subnav">
+          <a href="#">
+            <i class="fa fa-eye fa-lg"></i>
+            <span class="nav-text">
+              Visualizations
+            </span>
+          </a>
+          <ul>
+            <li class="has-subnav">
+              {{#if isFeature "logincounts"}}
+              <li>
+                <a href="/logincounts/">logincounts</a>
+              </li>
+              {{/if}}
+              {{#if isFeature "attackers"}}
+              <li>
+                <a href="/attackers">attackers</a>
+              </li>
+              {{/if}}
+              {{#if isFeature "globe"}}
+              <li>
+                <a href="/globe">globe</a>
+              </li>
+              {{/if}}
+            </li>
+          </ul>
+        </li>
+        {{/if}}
+        {{#if isFeature "about"}}
+        <li>
+          <a href="/about">
+            <i class="fa fa-font fa-lg"></i>
+            <span class="nav-text">
+              About
+            </span>
+          </a>
+            <ul>
+                <li><a target="_blank" href="https://mozdef.readthedocs.io/en/latest/index.html">Documentation</a></li>
+                <li><a href="/preferences">Preferences</a></li>
+            </ul>
+        </li>
+        {{/if}}
+        </ul>
+        <ul class="logout">
+          <li>
+            <a href="#">
+              <i class="fa fa-power-off fa-lg"></i>
+              <span class="nav-text">
+                <span class="login">
+                  <label>
+                    {{#if loggingIn}}
+                    loading
+                    {{/if}}
+                    {{> loginButtons align="left"}}
+                  </label>
+                </span>
+              </span>
+            </a>
+          </li>
+        </ul>
+        {{/if}}
+      </nav>
+    </div>
+</template>
+

meteor/imports/helpers.js

@@ -30,3 +30,21 @@ isFeature = function( featureName ) {
         return true;
     }
 };
+
+resolveKibanaURL = function(url){
+    // special function just for the menu
+    // to adjust the kibana URL if we are told to make it 'relative'
+    // to whatever DNS name we are running on
+    // i.e. pass in http://relative:9090/app/kibana
+    // when the running dns is something.com
+    // and we will set the hostname to something.com instead of 'relative'
+    var kibanaURL = new URL(url);
+    if ( kibanaURL.hostname == 'relative' ){
+        // we were passed something like  OPTIONS_METEOR_KIBANAURL=http://relative:9090/app/kibana
+        // so lets figure out where we should be
+        dnsURL = new URL(document.URL);
+        kibanaURL.hostname = dnsURL.hostname;
+        kibanaURL.protocol = dnsURL.protocol;
+    }
+    return kibanaURL;
+};

meteor/imports/themes/classic/mozdef.css

@@ -5,7 +5,7 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
 Copyright (c) 2014 Mozilla Corporation
 */
 
-/* variables */
+/* classic theme variables */
 :root {
     --bg-primary-color: rgb(32,87,153);
     --bg-secondary-color: #444444;
@@ -13,10 +13,13 @@ Copyright (c) 2014 Mozilla Corporation
     --txt-secondary-color: #000;
     --txt-shadow-color:  #000;
     --txt-highlight-color: rgba(165, 170, 172, 0.904);
+    --arm-color: #d1b61e;
+    --arm-focus-color: #e7c714a9;
+    --txt-disabled-color: #576d54;
     --a-link-color:  rgb(245, 222, 179);
     --row-color-odd: rgba(30,87,153,.7);
     --row-color-even: #636c85;
-  }
+}
 
 html{
 	background: none;
@@ -70,6 +73,8 @@ caption, legend {
   display: flex;
   align-items: center;
   justify-content: center;
+  width: 1200px;
+  padding-bottom: 30px;
 }
 
 #header a.mozilla {
@@ -81,6 +86,21 @@ caption, legend {
     overflow: visible;
 }
 
+/* Home Page Styling */
+.welcome {
+    height: 180px;
+    width: 600px;
+    margin-left: 25%;
+    text-align: center;
+    color: var(--txt-primary-color);
+    vertical-align: middle;
+}
+
+.mozdeflogo{
+  width: 500px;
+  vertical-align: middle;
+}
+
 #bottom-toolbar {
   background: var(--bg-secondary-color);
   width: 100%;
@@ -243,9 +263,29 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
 }
 
 .btn {
-  border: 1px outset;
-  border-radius: 4px;
-}
+    border: 1px outset;
+    border-radius: 4px;
+    color: var(--txt-primary-color);
+    background-color: var(--arm-color);
+  }
+  
+  .btn-warning.active,
+  .btn-warning:active,
+  .btn-warning:hover,
+  .open > .dropdown-toggle.btn-warning {
+    color: var(--txt-secondary-color);
+    background-color: var(--arm-focus-color);
+    border-color: var(--arm-color);
+  }
+
+  .btnAlertAcked,
+  .btnAlertAcked.active,
+  .btnAlertAcked:active,
+  .btnAlertAcked:hover > .btn {
+    color: var(--txt-disabled-color);
+    background-color: var(--arm-focus-color);
+    border-color: var(--arm-color);
+  }
 
 
 input[type="search"] {
@@ -273,7 +313,7 @@ input[type="search"] {
 
 .table-striped > tbody > tr:nth-of-type(2n+1) {
     background-color: var(--row-color-even)
-  }
+}
 
 .table-hover tbody tr:hover > td,
 .table-hover tbody tr:hover > th,
@@ -281,29 +321,44 @@ input[type="search"] {
   background-color: var(--txt-highlight-color);
 }
 
-td{
+td {
   color: var(--txt-primary-color);
 }
 
 .welcome {
+    height: 180px;
+    width: 600px;
+    margin-left: 25%;
     text-align: center;
+    color: var(--txt-primary-color);
+    vertical-align: middle;
 }
 
-.tabcontent{
+.mozdeflogo{
+  width: 500px;
+  vertical-align: middle;
+}
+
+.tabcontent {
     margin-top: 20px;
 }
 
-.tabnav a{
+.tabnav a {
     color: rgb(173, 216, 230);
 }
 
 /* don't float the 'create account' link*/
-#login-buttons #signup-link{
+#login-buttons #signup-link {
     float: none;
 }
 
+#login-buttons {
+    line-height: 1;
+    padding-top: 15px;
+}
+
 /* d3 circle styles for the logincounts visualization */
-.successcircle{
+.successcircle {
     /*
     fill: rgb(217,206,178);
 
@@ -313,7 +368,7 @@ td{
     fill: rgba(0,255,0,.5);
 }
 
-.failurecircle{
+.failurecircle {
     /*fill: rgb(213,222,217);*/
     /*fill: #BD2C00;*/
     /*fill: #f93;*/
@@ -321,7 +376,7 @@ td{
     fill: rgba(255,0,0,.5);
 }
 
-circle:hover{
+circle:hover {
   fill: white;
 }
 
@@ -330,7 +385,7 @@ circle:hover{
     stroke-width: 1.5px;
 }
 
-.textlabel{
+.textlabel {
     stroke-width: .2px;
     stroke: black;
 }
@@ -345,7 +400,7 @@ circle:hover{
     display: none;
 }
 
-#header a.mozdef{
+#header a.mozdef {
     color: var(--txt-primary-color);
     text-shadow: var(--txt-shadow-color) 5px 3px 3px;
     text-align: center;
@@ -353,11 +408,12 @@ circle:hover{
     margin-top: -.25em;
 }
 
-.mozillalogo{
-    width: 90px;
+.mozillalogo {
+    width: 300px;
+    padding-top: 15px;
 }
 
-#header label{
+#header label {
     display: inherit;
     cursor: auto;
     margin:.5em;
@@ -365,8 +421,7 @@ circle:hover{
 
 /* attacker side nav pull out styling */
 /* http://www.sitepoint.com/css3-sliding-menu/ */
-sidenav
-{
+sidenav {
     position: fixed;
     left: -25em;
     top: 0;
@@ -375,8 +430,6 @@ sidenav
     border-right: 15px solid #765;
     box-shadow: 4px 0 5px rgba(0,0,0,0.2);
     z-index: 1;
-
-
     text-align: left;
     font-weight: bolder;
     display: inline-block;
@@ -386,8 +439,7 @@ sidenav
 }
 
 /*pull out triangle */
-sidenav:after
-{
+sidenav:after {
     position: absolute;
     content: ' ';
     width: 0;
@@ -399,20 +451,18 @@ sidenav:after
     border-color: transparent transparent transparent #765;
 }
 
-sidenav ul
-{
+sidenav ul {
     width: 14em;
     list-style-type: none;
     margin: auto;
     padding: 1em;
 }
 
-sidenav div{
+sidenav div {
     margin:auto;
 }
 
-sidenav:hover
-{
+sidenav:hover {
     left: 0;
 }
 
@@ -426,8 +476,7 @@ sidenav .reset-filter {
     margin-top: 20px;
 }
 
-sidenav
-{
+sidenav {
     -webkit-transition: all 400ms ease;
     -moz-transition: all 400ms ease;
     -ms-transition: all 400ms ease;
@@ -490,7 +539,7 @@ sidenav
     width: 270px;
     position: absolute;
     left: 10px;
-    top: 63px;
+    top: 125px;
     background-color: rgba(0, 0, 0, 0.2);
     border-top: 1px solid rgba(255, 255, 255, 0.4);
     padding: 10px;
@@ -500,7 +549,7 @@ sidenav
     width: 400px;
     position: absolute;
     left: 10px;
-    top: 103px;
+    top: 180px;
     background-color: rgba(0, 0, 0, 0.2);
     border-top: 1px solid rgba(255, 255, 255, 0.4);
     padding: 10px;
@@ -586,7 +635,7 @@ sidenav
 #nav-main li span {
     border-color: #3a3c46;
     color: var(--txt-primary-color);
-    padding: 8px 10px 10px 10px;
+    padding: 20px 10px 10px 10px;
     height: 25px;
     display: block;
     border-style: solid;
@@ -636,7 +685,7 @@ sidenav
     transition-timing-function: linear;
     transition-delay: 0s;
     transition-property: #6363CE, box-shadow, border-right;
-    padding-bottom: 30px;
+    padding-bottom: 20px;
     outline: 0px none;
 }
 

meteor/imports/themes/dark/mozdef.css

@@ -5,6 +5,20 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
 Copyright (c) 2014 Mozilla Corporation
 */
 
+:root {
+    --bg-primary-color: #2a2f35;
+    --secondary-focus-color: #3b89ee;
+    --bg-secondary-color: #2d5fa0;
+    --row-color-odd: #2a2f35;
+    --row-color-even: #636c85;
+    --arm-color: #e69006;
+    --arm-focus-color: #d58512;
+    --txt-primary-color: #fff;
+    --txt-secondary-color: #000;
+    --txt-disabled-color: #576d54;
+    --a-link-color: #a2a9b2;
+}
+
 /*base css */
 html{
 	background: none;
@@ -50,6 +64,8 @@ caption, legend {
   display: flex;
   align-items: center;
   justify-content: center;
+  width: 1200px;
+  padding-bottom: 30px;
 }
 
 #header a.mozilla {
@@ -61,6 +77,21 @@ caption, legend {
     overflow: visible;
 }
 
+/* Home Page Styling */
+.welcome {
+    height: 180px;
+    width: 600px;
+    margin-left: 25%;
+    text-align: center;
+    color: var(--txt-primary-color);
+    vertical-align: middle;
+}
+
+.mozdeflogo{
+  width: 500px;
+  vertical-align: middle;
+}
+
 #bottom-toolbar {
   background: #444444;
   width: 100%;
@@ -127,12 +158,12 @@ caption, legend {
     color: rgba(127,255,255,0.75);
 }
 
-.attackercallout .blockip{
+.attackercallout .blockip {
     color: #FFF;
     text-transform: uppercase;
     margin-top: 20px;
 }
-.attackercallout ul{
+.attackercallout ul {
     list-style: none;
     float: left;
     left: auto;
@@ -140,14 +171,14 @@ caption, legend {
     padding: 0px;
 }
 
-.attackercallout .indicator{
+.attackercallout .indicator {
     color: yellow;
 }
-.attackercallout a{
+.attackercallout a {
     color: yellow;
 }
 
-.veris-wrapper td{
+.veris-wrapper td {
     color: black
 }
 
@@ -159,33 +190,33 @@ caption, legend {
 .alert.alert-NOTICE {
     --alert-bg-color: #4a6785;
     --alert-color: white;
-  }
+}
 .alert.alert-WARNING {
     --alert-bg-color: #ffd351;
     --alert-color: black;
-  }
+}
 .alert.alert-CRITICAL {
     --alert-bg-color: #d04437;
     --alert-color: white;
-  }
+}
 .alert.alert-INFO {
     --alert-bg-color: #cccccc;
     --alert-color: black;
-  }
+}
 .alert.alert-ERROR {
     --alert-bg-color: #d04437;
     --alert-color: white;
-  }
+}
 
-.alert  {
+.alert {
     color: var(--alert-color);
     background-color: var(--alert-bg-color);
     text-transform: uppercase;
     display: table-cell;
     font-weight: bold;
-  }
+}
 
-.alert-row a{
+.alert-row a {
     color: wheat;
 }
 
@@ -193,14 +224,42 @@ caption, legend {
     color: white;
 }
 
-.modal-body .row {
-    color: black;
+.modal-header {
+    color: var(--font-focus);
+}
+  
+  .modal-body {
+    color: var(--font-focus);
+}
+  
+  .modal-body .row {
+      color: black;
 }
 /*bootstrap overrides*/
 
 .btn {
-  border: 1px outset;
-  border-radius: 4px;
+    border: 1px outset;
+    border-radius: 4px;
+    color: var(--txt-primary-color);
+    background-color: var(--arm-color);
+}
+  
+  .btn-warning.active,
+  .btn-warning:active,
+  .btn-warning:hover,
+  .open > .dropdown-toggle.btn-warning {
+    color: var(--txt-secondary-color);
+    background-color: var(--arm-focus-color);
+    border-color: var(--arm-color);
+}
+
+  .btnAlertAcked,
+  .btnAlertAcked.active,
+  .btnAlertAcked:active,
+  .btnAlertAcked:hover > .btn {
+    color: var(--txt-disabled-color);
+    background-color: var(--arm-focus-color);
+    border-color: var(--arm-color);
 }
 
 
@@ -229,20 +288,30 @@ input[type="search"] {
   background-color: rgba(30,87,153,.5);
 }
 
-td{
+td {
   color:white;
 }
 
 
 .welcome {
+    height: 180px;
+    width: 600px;
+    margin-left: 25%;
     text-align: center;
+    color: var(--txt-primary-color);
+    vertical-align: middle;
 }
 
-.tabcontent{
+.mozdeflogo{
+  width: 500px;
+  vertical-align: middle;
+}
+
+.tabcontent {
     margin-top: 20px;
 }
 
-.tabnav a{
+.tabnav a {
     color: lightblue;
 }
 
@@ -266,12 +335,17 @@ td{
 }
 */
 /* don't float the 'create account' link*/
-#login-buttons #signup-link{
+#login-buttons #signup-link {
     float: none;
 }
 
+#login-buttons {
+    line-height: 1;
+    padding-top: 15px;
+}
+
 /* d3 circle styles */
-.successcircle{
+.successcircle {
     /*
     fill: rgb(217,206,178);
 
@@ -281,7 +355,7 @@ td{
     fill: rgba(0,255,0,.5);
 }
 
-.failurecircle{
+.failurecircle {
     /*fill: rgb(213,222,217);*/
     /*fill: #BD2C00;*/
     /*fill: #f93;*/
@@ -289,7 +363,7 @@ td{
     fill: rgba(255,0,0,.5);
 }
 
-circle:hover{
+circle:hover {
   fill: white;
 }
 
@@ -298,7 +372,7 @@ circle:hover{
     stroke-width: 1.5px;
 }
 
-.textlabel{
+.textlabel {
     stroke-width: .2px;
     stroke: black;
 }
@@ -313,7 +387,7 @@ circle:hover{
     display: none;
 }
 
-#header a.mozdef{
+#header a.mozdef {
     color:white;
     text-shadow: #000 5px 3px 3px;
     text-align: center;
@@ -321,16 +395,230 @@ circle:hover{
     margin-top: -.25em;
 }
 
-.mozillalogo{
-    width: 90px;
+.mozillalogo {
+    width: 300px;
+    padding-top: 15px;
 }
 
-#header label{
+#header label {
     display: inherit;
     cursor: auto;
     margin:.5em;
 }
 
+/* Attackers sidenav menu */
+sidenav {
+    background: var(--bg-primary-color);
+    border-right: 15px solid var(--bg-secondary-color);
+    text-align: left;
+    font-weight: bolder;
+    position: fixed;
+    top: 0;
+    bottom: 0;
+    height: 100%;
+    left: -25em;
+    margin: 0em;
+    padding-top: 1em;
+    display: inline-block;
+    line-height: normal;
+    -webkit-transform: translateZ(0) scale(1, 1);
+    z-index: 3;
+    -webkit-transition: all 400ms ease;
+    -moz-transition: all 400ms ease;
+    -ms-transition: all 400ms ease;
+    -o-transition: all 400ms ease;
+    transition: all 400ms ease;
+}
+  
+  /*pull out triangle*/
+sidenav:after {
+    position: absolute;
+    content: ' ';
+    width: 0;
+    height: 0;
+    right: -75px;
+    top: 50%;
+    border-width: 30px 30px;
+    border-style: solid;
+    border-color: transparent transparent transparent var(--bg-secondary-color);
+}
+sidenav ul {
+    width: 14em;
+    list-style-type: none;
+    margin: auto;
+    padding: 1em;
+}
+sidenav div{
+    margin:auto;
+}
+sidenav:hover {
+    left: 0;
+}
+sidenav .filters-col .row {
+    margin-top: 45px;
+    padding: 0 0.5em;
+}
+sidenav .reset-filter {
+    text-align: center;
+    margin-top: 20px;
+}
+.form-horizontal .form-group {
+  margin-left: 5px;
+  margin-right: 5px;
+  padding-top: 5px;
+}
+  
+@media screen and (max-width: 1000px) {
+  sidenav {
+    background: var(--bg-primary-color);
+    border-left: 15px solid var(--bg-secondary-color);
+    text-align: left;
+    font-weight: bolder;
+    position: fixed;
+    top: 0;
+    bottom: 0;
+    height: 100%;
+    right: -16em;
+    margin: 0em;
+    padding-top: 1em;
+    display: inline-block;
+    line-height: normal;
+    -webkit-transform: translateZ(0) scale(1, 1);
+    z-index: 3;
+    -webkit-transition: all 400ms ease;
+    -moz-transition: all 400ms ease;
+    -ms-transition: all 400ms ease;
+    -o-transition: all 400ms ease;
+    transition: all 400ms ease;
+  }
+  sidenav:after {
+    right: 230px;
+    border-top: 0;
+    border-bottom: 0;
+    border-right: 0;
+    content: none;
+  }
+  sidenav ul {
+    width: 14em;
+    list-style-type: none;
+    margin: auto;
+    padding: 1em;
+  }
+  sidenav div{
+    margin:auto;
+  }
+  sidenav:hover {
+    right: 0;
+    width: 230px;
+    overflow-y: scroll;
+    scrollbar-width: inherit;
+    scrollbar-color: var(--bg-secondary-color) black;
+  }
+  sidenav .filters-col .row {
+    margin-top: 25px;
+    padding: 0 1.5em;
+  }
+  sidenav .reset-filter {
+    text-align: center;
+    margin-top: 20px;
+  }
+  div.dc-chart {
+    float: none;
+  }
+}
+
+/* globe styling */
+.globe-container  {
+    background: #000000;
+    color: #ffffff;
+}
+
+.globe-info {
+    font-size: 11px;
+    position: absolute;
+    bottom: 5px;
+    background-color: rgba(0, 0, 0, 0.8);
+    border-radius: 3px;
+    right: 10px;
+    padding: 10px;
+    color: var(--ack-edit-color);
+}
+
+.globe-container a {
+    color: #aaa;
+    text-decoration: none;
+}
+
+.globe-container a:hover {
+    text-decoration: underline;
+}
+
+.globe-campaigns {
+    width: 270px;
+    position: absolute;
+    left: 10px;
+    top: 125px;
+    background-color: rgba(0, 0, 0, 0.2);
+    border-top: 1px solid rgba(255, 255, 255, 0.4);
+    padding: 10px;
+}
+
+.globe-time {
+    width: 400px;
+    position: absolute;
+    left: 10px;
+    top: 180px;
+    background-color: rgba(0, 0, 0, 0.2);
+    border-top: 1px solid rgba(255, 255, 255, 0.4);
+    padding: 5px;
+}
+
+.globe-facts {
+    width: 300px;
+    position: absolute;
+    left: 10px;
+    bottom: 0px;
+    background-color: rgba(0, 0, 0, 0.2);
+    border-top: 1px solid rgba(255, 255, 255, 0.4);
+    border-right: 1px solid rgba(255, 255, 255, 0.4);
+    padding: 10px;
+    font-size: 12px;
+    color: rgba(255, 255, 255, 0.9);
+    display: none;
+}
+
+.globe-campaigns .campaign{
+    font-size: 12px;
+    line-height: 26px;
+    height: 30px;
+    text-align: center;
+    float: left;
+    width: 60px;
+    color: rgba(255, 255, 255, 0.4);
+    cursor: pointer;
+    transition: all 0.1s ease-out;
+}
+
+.globe-time .time{
+    font-size: 12px;
+    line-height: 26px;
+    height: 30px;
+    text-align: center;
+    float: left;
+    width: 90px;
+    color: rgba(255, 255, 255, 0.4);
+    cursor: pointer;
+    transition: all 0.1s ease-out;
+}
+
+.globe-campaigns .campaign:hover,
+.globe-campaigns .campaign.active,
+.globe-time .time:hover,
+.globe-time .time.active {
+    font-size: 14px;
+    color: #fff;
+}
+
 /*menu styling*/
 #nav-main {
     text-align: left;
@@ -365,7 +653,7 @@ circle:hover{
 #nav-main li span {
     border-color: #3a3c46;
     color: #fff;
-    padding: 8px 10px 10px 10px;
+    padding: 20px 10px 10px 10px;
     height: 25px;
     display: block;
     border-style: solid;
@@ -415,7 +703,7 @@ circle:hover{
     transition-timing-function: linear;
     transition-delay: 0s;
     transition-property: #6363CE, box-shadow, border-right;
-    padding-bottom: 30px;
+    padding-bottom: 20px;
     outline: 0px none;
 }
 
@@ -500,7 +788,6 @@ circle:hover{
 #nav-main ul li.current ul li a:focus,
 #nav-main ul li.current ul li a:active,
 .js #nav-main ul li.current ul li a:focus {
-    background: rgb(227,235,244);
     background: rgba(41,48,54,0.2);
     box-shadow:
         inset rgba(41,48,54,0.3) 0 2px 0px,

meteor/imports/themes/light/mozdef.css

@@ -5,14 +5,16 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
 Copyright (c) 2014 Mozilla Corporation
 */
 
-/* variables */
+/* light theme variables */
 :root {
-    --bg-primary-color: rgba(255,255,255,.5);
-    --bg-secondary-color: #444444;
+    --bg-primary-color: rgba(230, 227, 227, 0.5);
+    --bg-secondary-color: #707070;
     --txt-primary-color: #000;
     --txt-secondary-color: #fff;
     --txt-shadow-color:  #aaa;
     --txt-highlight-color: rgba(165, 170, 172, 0.904);
+    --arm-color: #d1b61e;
+    --arm-focus-color: #e7c714a9;
     --a-link-color:  rgb(49, 130, 189);
     --row-color-odd: rgba(30,87,153,.1);
     --row-color-even: #636c85;
@@ -61,6 +63,8 @@ caption, legend {
   display: flex;
   align-items: center;
   justify-content: center;
+  width: 1200px;
+  padding-bottom: 30px;
 }
 
 #header a.mozilla {
@@ -72,6 +76,27 @@ caption, legend {
     overflow: visible;
 }
 
+/* Home Page Styling */
+.welcome {
+    height: 180px;
+    width: 600px;
+    margin-left: 25%;
+    text-align: center;
+    color: var(--txt-primary-color);
+    vertical-align: middle;
+}
+
+.mozdeflogo{
+  width: 500px;
+  vertical-align: middle;
+}
+
+/* dc-chart empty color */
+.dc-chart .empty-chart .pie-slice path {
+    fill: #bcb8b8;
+    cursor: default;
+}
+
 #bottom-toolbar {
   background: var(--bg-secondary-color);
   width: 100%;
@@ -138,12 +163,12 @@ caption, legend {
     color: rgba(127,255,255,0.75);
 }
 
-.attackercallout .blockip{
+.attackercallout .blockip {
     color: var(--txt-primary-color);;
     text-transform: uppercase;
     margin-top: 20px;
 }
-.attackercallout ul{
+.attackercallout ul {
     list-style: none;
     float: left;
     left: auto;
@@ -151,14 +176,14 @@ caption, legend {
     padding: 0px;
 }
 
-.attackercallout .indicator{
+.attackercallout .indicator {
     color: yellow;
 }
-.attackercallout a{
+.attackercallout a {
     color: yellow;
 }
 
-.veris-wrapper td{
+.veris-wrapper td {
     color: black
 }
 
@@ -170,23 +195,23 @@ caption, legend {
 .alert.alert-NOTICE {
     --alert-bg-color: #4a6785;
     --alert-color: black;
-  }
+}
 .alert.alert-WARNING {
     --alert-bg-color: #ffd351;
     --alert-color: black;
-  }
+}
 .alert.alert-CRITICAL {
     --alert-bg-color: #d04437;
     --alert-color: black;
-  }
+}
 .alert.alert-INFO {
     --alert-bg-color: #cccccc;
     --alert-color: black;
-  }
+}
 .alert.alert-ERROR {
     --alert-bg-color: #d04437;
     --alert-color: black;
-  }
+}
 
 .alert  {
     color: var(--alert-color);
@@ -194,9 +219,9 @@ caption, legend {
     text-transform: uppercase;
     display: table-cell;
     font-weight: bold;
-  }
+}
 
-.alert-row a{
+.alert-row a {
     color: var(--a-link-color);
 }
 
@@ -245,8 +270,28 @@ h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6 {
 }
 
 .btn {
-  border: 1px outset;
-  border-radius: 4px;
+    border: 1px outset;
+    border-radius: 4px;
+    color: var(--txt-primary-color);
+    background-color: var(--arm-color);
+}
+  
+  .btn-warning.active,
+  .btn-warning:active,
+  .btn-warning:hover,
+  .open > .dropdown-toggle.btn-warning {
+    color: var(--txt-secondary-color);
+    background-color: var(--arm-focus-color);
+    border-color: var(--arm-color);
+}
+
+  .btnAlertAcked,
+  .btnAlertAcked.active,
+  .btnAlertAcked:active,
+  .btnAlertAcked:hover > .btn {
+    color: var(--txt-shadow-color);
+    background-color: var(--arm-focus-color);
+    border-color: var(--arm-color);
 }
 
 
@@ -268,6 +313,11 @@ input[type="search"] {
     float: none;
 }
 
+.dc-chart .empty-chart .pie-slice path {
+    fill: #e3e3e3;
+    cursor: default;
+}
+
 .table-striped tbody > tr:nth-child(odd) > td,
 .table-striped tbody > tr:nth-child(odd) > th {
   background-color: var(--row-color-odd);
@@ -278,29 +328,44 @@ input[type="search"] {
   background-color: var(--txt-highlight-color);
 }
 
-td{
+td {
   color: var(--txt-primary-color);
 }
 
 .welcome {
+    height: 180px;
+    width: 600px;
+    margin-left: 25%;
     text-align: center;
+    color: var(--txt-primary-color);
+    vertical-align: middle;
 }
 
-.tabcontent{
+.mozdeflogo{
+  width: 500px;
+  vertical-align: middle;
+}
+
+.tabcontent {
     margin-top: 20px;
 }
 
-.tabnav a{
+.tabnav a {
     color: lightblue;
 }
 
 /* don't float the 'create account' link*/
-#login-buttons #signup-link{
+#login-buttons #signup-link {
     float: none;
 }
 
+#login-buttons {
+    line-height: 1;
+    padding-top: 15px;
+}
+
 /* d3 circle styles */
-.successcircle{
+.successcircle {
     /*
     fill: rgb(217,206,178);
 
@@ -310,7 +375,7 @@ td{
     fill: rgba(0,255,0,.5);
 }
 
-.failurecircle{
+.failurecircle {
     /*fill: rgb(213,222,217);*/
     /*fill: #BD2C00;*/
     /*fill: #f93;*/
@@ -318,7 +383,7 @@ td{
     fill: rgba(255,0,0,.5);
 }
 
-circle:hover{
+circle:hover {
   fill: white;
 }
 
@@ -327,7 +392,7 @@ circle:hover{
     stroke-width: 1.5px;
 }
 
-.textlabel{
+.textlabel {
     stroke-width: .2px;
     stroke: black;
 }
@@ -342,19 +407,12 @@ circle:hover{
     display: none;
 }
 
-#header a.mozdef{
-    color: var(--txt-primary-color);
-    text-shadow: var(--txt-shadow-color) 3px 2px 2px;
-    text-align: center;
-    font-size: large;
-    margin-top: -.25em;
+.mozillalogo {
+    width: 300px;
+    padding-top: 15px;
 }
 
-.mozillalogo{
-    width: 90px;
-}
-
-#header label{
+#header label {
     display: inherit;
     cursor: auto;
     margin:.5em;
@@ -362,14 +420,13 @@ circle:hover{
 
 /* attacker side nav pull out styling */
 /* http://www.sitepoint.com/css3-sliding-menu/ */
-sidenav
-{
+sidenav {
     position: fixed;
     left: -25em;
     top: 0;
     bottom: 0;
     background-color: var(--bg-secondary-color);
-    border-right: 15px solid #765;
+    border-right: 15px solid  #6d5e5e;
     box-shadow: 4px 0 5px rgba(0,0,0,0.2);
     z-index: 1;
 
@@ -383,8 +440,7 @@ sidenav
 }
 
 /*pull out triangle */
-sidenav:after
-{
+sidenav:after {
     position: absolute;
     content: ' ';
     width: 0;
@@ -393,23 +449,21 @@ sidenav:after
     top: 50%;
     border-width: 30px 30px;
     border-style: solid;
-    border-color: transparent transparent transparent #765;
+    border-color: transparent transparent transparent #6d5e5e;
 }
 
-sidenav ul
-{
+sidenav ul {
     width: 14em;
     list-style-type: none;
     margin: auto;
     padding: 1em;
 }
 
-sidenav div{
+sidenav div {
     margin:auto;
 }
 
-sidenav:hover
-{
+sidenav:hover {
     left: 0;
 }
 
@@ -423,8 +477,7 @@ sidenav .reset-filter {
     margin-top: 20px;
 }
 
-sidenav
-{
+sidenav {
     -webkit-transition: all 400ms ease;
     -moz-transition: all 400ms ease;
     -ms-transition: all 400ms ease;
@@ -434,7 +487,7 @@ sidenav
 
 /*veris tag styling*/
 
-.veristagform{
+.veristagform {
     padding: 10px;
     font-size: 95%;
     background: #999;
@@ -444,7 +497,7 @@ sidenav
 .veristagform .dropdown {
     margin: 10px 0;
 }
-.tag{
+.tag {
     cursor: pointer;
     text-align: left;
     word-wrap: break-word;
@@ -453,7 +506,7 @@ sidenav
     cursor: pointer;
     color: var(--txt-primary-color);
 }
-.veristagform .label{
+.veristagform .label {
     cursor: pointer;
 }
 
@@ -487,7 +540,7 @@ sidenav
     width: 270px;
     position: absolute;
     left: 10px;
-    top: 63px;
+    top: 125px;
     background-color: rgba(0, 0, 0, 0.2);
     border-top: 1px solid rgba(255, 255, 255, 0.4);
     padding: 10px;
@@ -497,10 +550,10 @@ sidenav
     width: 400px;
     position: absolute;
     left: 10px;
-    top: 103px;
+    top: 180px;
     background-color: rgba(0, 0, 0, 0.2);
     border-top: 1px solid rgba(255, 255, 255, 0.4);
-    padding: 10px;
+    padding: 5px;
 }
 
 .globe-facts {
@@ -582,12 +635,11 @@ sidenav
 #nav-main li a:visited,
 #nav-main li span {
     color:#000;
-    padding: 8px 10px 10px 10px;
+    padding: 20px 10px 10px 10px;
     height: 25px;
     display: block;
-    border-style: solid;
+    border-style: none;
     border-width: 0px 1px;
-    z-index: 100;
     position:relative;
     float:left;
     border-image: none;
@@ -608,6 +660,17 @@ sidenav
 
 /* Second-level Menu Items */
 
+#nav-main li:hover ul,
+.js #nav-main li.hover ul {
+    margin-top: 2.5em;
+    left: 0;
+    opacity: 1;
+    -moz-transition: opacity 0.2s ease-in-out;
+    -webkit-transition: opacity 0.2s ease-in-out;
+    -o-transition: opacity 0.2s ease-in-out;
+    transition: opacity 0.2s ease-in-out;
+}
+
 #nav-main ul ul,
 #nav-main ul ul li {
     height: auto;
@@ -625,10 +688,8 @@ sidenav
     z-index: 99;
     width: 190px;
     border-radius: 0px 4px 4px;
-    border: 1px solid #3a3c46;
-    background: #000;
-    background: rgba(0,0,0,0.9);
-
+    border: none;
+    background: rgb(73, 70, 70);
 }
 
 #nav-main ul ul li {
@@ -641,7 +702,7 @@ sidenav
 #nav-main ul ul a:link,
 #nav-main ul ul a:visited {
     overflow: hidden;
-    color: #fff;
+    color: #000;
     padding: 3px 10px;
     margin: 0px;
     height: auto;
@@ -656,11 +717,65 @@ sidenav
 #nav-main ul ul a:hover,
 #nav-main ul ul a:focus,
 #nav-main ul ul a:active {
-    color: #fff;
+    color: var(--txt-secondary-color);
     background: rgb(227,235,244);
     background: rgba(241,248,254,0.2);
 
 }
+/* Currently active menu items */
+#nav-main ul li.current span,
+#nav-main ul li.current a,
+#nav-main ul li.current a:link,
+#nav-main ul li.current a:visited {
+    border-bottom-width: 1px;
+    -moz-border-radius: 0 0 4px 4px;
+    -webkit-border-radius: 0 0 4px 4px;
+    border-radius: 0 0 4px 4px;
+    background: #fbfdff;
+    background: -moz-linear-gradient(top, rgba(255,255,255,0.4) 0%,
+                                     rgba(255,255,255,0.8) 100%);
+    box-shadow:
+        rgba(152,178,201,0.2) 0 0px 0px,
+        inset rgba(152,178,201,0.3) 0 -2px 0,
+        inset rgba(255,255,255,0.8) 0 -6px 6px 4px;
+    -moz-box-shadow:
+        rgba(152,178,201,0.2) 0 0px 0px,
+        inset rgba(152,178,201,0.3) 0 -2px 0,
+        inset rgba(255,255,255,0.8) 0 -6px 6px 4px;
+    -webkit-box-shadow:
+        rgba(152,178,201,0.2) 0 0px 0px,
+        inset rgba(152,178,201,0.3) 0 -2px 0,
+        inset rgba(255,255,255,0.8) 0 -6px 6px 4px;
+    padding-right: 30px;
+    padding-bottom: 15px;
+}
+
+#nav-main ul li.current:hover a,
+#nav-main ul li.current.hover a,
+#nav-main ul li.current a:hover,
+#nav-main ul li.current a:focus,
+#nav-main ul li.current a:active,
+.js #nav-main ul li.current a:focus {
+    box-shadow: none;
+    -moz-box-shadow: none;
+    -webbkit-box-shadow: none;
+    -webkit-border-radius: 0;
+    -moz-border-radius: 0;
+    border-radius: 0;
+    border-bottom: 0;
+}
+
+#nav-main ul li.current ul span,
+#nav-main ul li.current ul a,
+#nav-main ul li.current ul a:link,
+#nav-main ul li.current ul a:visited {
+    background: none;
+    height: auto;
+    box-shadow: none;
+    -webkit-box-shadow: none;
+    -moz-box-shadow: none;
+    padding: 8px 10px;
+}
 
 
 .mousepointer {

meteor/imports/themes/side_nav_dark/menu.js

@@ -0,0 +1,21 @@
+import { Meteor } from 'meteor/meteor';
+import { Template } from 'meteor/templating';
+import { Tracker } from 'meteor/tracker'
+
+Template.side_nav_menu.rendered = function() {
+    Tracker.autorun( function() {
+        Meteor.subscribe( "features" );
+    } );
+};
+
+Template.side_nav_menu.helpers( {
+    haveFeatures: function() {
+        //subscription has records?
+        return features.find().count() > 0;
+    },
+    // loads kibana dashboards
+    kibanadashboards: function() {
+        Meteor.call( 'loadKibanaDashboards' );
+        return kibanadashboards.find();
+    }
+} );

meteor/imports/themes/side_nav_dark/mozdef.css

@@ -0,0 +1,933 @@
+@import url('https://fonts.googleapis.com/css?family=Zilla+Slab+Highlight');
+@import url('https://fonts.googleapis.com/css?family=Zilla+Slab');
+@import url('https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css');
+/*
+This Source Code Form is subject to the terms of the Mozilla Public
+License, v. 2.0. If a copy of the MPL was not distributed with this
+file, You can obtain one at http://mozilla.org/MPL/2.0/.
+Copyright (c) 2014 Mozilla Corporation
+*/
+
+/*side-nav-dark css */
+
+:root {
+  --primary-bg-color: #2a2f35;
+  --secondary-focus-color: #3b89ee;
+  --secondary-color: #2d5fa0;
+  --row-color-odd: #2a2f35;
+  --row-color-even: #636c85;
+  --ack-edit-color: #a2a9b2;
+  --ack-edit-border-color: #adadad;
+  --txt-shadow-color: #576d54;
+  --arm-color: #e69006;
+  --arm-focus-color: #d58512;
+  --font-main: #fff;
+  --font-focus: #000;
+  --a-link-color: #a2a9b2;
+}
+
+html{
+	background: none;
+	min-height: 100%;
+}
+
+body{
+	/*gradient*/
+
+    background: rgb(42, 47, 53); /* Old browsers */
+    background: -moz-radial-gradient(ellipse, rgb(99, 108, 118) 0%, rgb(42, 47, 53) 100%); /* FF3.6+ */
+    background: -webkit-radial-gradient(ellipse, rgb(99, 108, 118) 0%, rgb(42, 47, 53) 100%); /* Chrome10+,Safari5.1+ */
+    background: -o-radial-gradient(ellipse, rgba(99, 108, 118, 1) 0%, rgb(42, 47, 53) 100%); /* Opera 11.10+ */
+    background: radial-gradient(ellipse, rgb(99, 108, 118) 0%, rgb(42, 47, 53) 100%); /* W3C */
+    filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#636c76",endColorstr="#2a2f35",GradientType=1); /* IE6-9 fallback on horizontal gradient */
+
+    font-size: 14px;
+    /*margin: 0;*/
+    /*min-width: 990px;*/
+    padding: 0;
+    color: var(--font-main);
+    line-height: normal;
+    text-align: left;
+}
+
+.container {
+  margin-top: 1%;
+}
+
+/*mozdef custom */
+.upperwhite {
+    color: var(--font-main);
+    text-transform: uppercase;
+}
+
+caption, legend {
+    color: var(--font-main);
+}
+
+.shadow {
+    text-shadow: #000 .7px .7px .7px;
+}
+
+.ipaddress {
+  color: var(--font-main);
+  font-style: normal;
+  font-weight: normal;
+
+}
+
+.headercontainer {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+#header a.mozilla {
+    position: relative;
+    float: right;
+    display: block;
+    height: 40px;
+    width: 100px;
+    overflow: visible;
+}
+
+#bottom-toolbar {
+  background: #636c76;
+  width: 100%;
+  height: 40px;
+  padding-left: 5px;
+  padding-top: 5px;
+  position: fixed;
+  bottom: 0;
+  opacity: .3;
+  z-index: 2;
+  font-size: 13px;
+  color: var(--font-main);
+}
+
+#bottom-toolbar:hover {
+  opacity: 1;
+}
+
+#bottom-toolbar .button {
+  margin: 3px 3px 3px 0;
+  padding: 2px 3px 2px 5px;
+  height: 13px;
+  line-height: 13px;
+}
+
+
+.attackshoverboard {
+    /*width: 500px;*/
+    /*height: 500px;*/
+    /*background-color: green;*/
+    -moz-transform: scaleY(-1);
+    -webkit-transform: scaleY(-1);
+    -o-transform: scaleY(-1);
+    transform: scaleY(-1);
+    display: none;
+}
+
+.dropdown-submenu{position:relative;}
+.dropdown-submenu>.dropdown-menu{top:0;left:100%;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px;}
+.dropdown-submenu:active>.dropdown-menu, .dropdown-submenu:hover>.dropdown-menu {
+       display: block;
+       right:162px;
+}
+.dropdown-submenu>a:after{display:block;content:" ";float:right;width:0;height:0;border-color:transparent;border-style:solid;border-width:5px 0 5px 5px;border-left-color:#cccccc;margin-top:5px;margin-right:-10px;}
+.dropdown-submenu:active>a:after{border-left-color:#ffffff;}
+.dropdown-submenu.pull-left{float:none;}.dropdown-submenu.pull-left>.dropdown-menu{left:-100%;margin-left:10px;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px;}
+
+
+
+.attackercallout {
+    width: 120px;
+    height: 160px;
+    box-shadow: 0px 0px 12px rgba(0,255,255,0.5);
+    opacity: .5;
+    background: black;
+    border: 1px solid rgba(127,255,255,0.25);
+    text-align: center;
+    cursor: default;
+    z-index: 99;
+}
+
+
+
+.attackercallout .id {
+    font-size: 12px;
+    color: rgba(127,255,255,0.75);
+}
+
+.attackercallout .details {
+    text-align: left;
+    left: 2px;
+    font-size: 12px;
+    color: rgba(127,255,255,0.75);
+}
+
+.attackercallout .blockip{
+    color: #FFF;
+    text-transform: uppercase;
+    margin-top: 20px;
+}
+.attackercallout ul{
+    list-style: none;
+    float: left;
+    left: auto;
+    margin: 0px;
+    padding: 0px;
+}
+
+.attackercallout .indicator{
+    color: yellow;
+}
+.attackercallout a{
+    color: yellow;
+}
+
+.veris-wrapper td{
+    color: black
+}
+
+.veris-wrapper {
+    background-color: rgba(245,245,245,.7)
+}
+
+/*alerts screen alert color scheme*/
+.alert.alert-NOTICE {
+    --alert-bg-color: #4a6785;
+    --alert-color: white;
+}
+.alert.alert-WARNING {
+    --alert-bg-color: #ffd351;
+    --alert-color: black;
+}
+.alert.alert-CRITICAL {
+    --alert-bg-color: #d04437;
+    --alert-color: white;
+}
+.alert.alert-INFO {
+    --alert-bg-color: #cccccc;
+    --alert-color: black;
+}
+.alert.alert-ERROR {
+    --alert-bg-color: #d04437;
+    --alert-color: white;
+}
+
+.alert  {
+    color: var(--alert-color);
+    background-color: var(--alert-bg-color);
+    text-transform: uppercase;
+    display: table-cell;
+    font-weight: bold;
+}
+
+.alert-row a{
+    color: var(--a-link-color);
+}
+
+.row {
+    color: white;
+}
+
+.modal-header {
+  color: var(--font-focus);
+}
+
+.modal-body {
+  color: var(--font-focus);
+}
+
+.modal-body .row {
+    color: black;
+}
+/*bootstrap overrides*/
+
+.btn {
+  border: 1px outset;
+  border-radius: 4px;
+  color: var(--font-main);
+  background-color: var(--arm-color);
+}
+
+.btn-warning.active,
+.btn-warning:active,
+.btn-warning:hover,
+.open > .dropdown-toggle.btn-warning {
+  color: var(--font-focus);
+  background-color: var(--arm-focus-color);
+  border-color: var(--arm-color);
+}
+
+.btnAlertAcked,
+.btnAlertAcked.active,
+.btnAlertAcked:active,
+.btnAlertAcked:hover > .btn {
+  color: var(--txt-shadow-color);
+  background-color: var(--arm-focus-color);
+  border-color: var(--arm-color);
+}
+
+.btn-notice {
+  border: 1px outset;
+  border-radius: 4px;
+  color: var(--font-focus);
+  background-color: var(--ack-edit-color);
+}
+
+.btn-notice.active,
+.btn-notice:active,
+.btn-notice:hover,
+.open > .dropdown-toggle.btn-notice {
+  color: var(--font-main);
+  background-color: var(--ack-edit-focus-color);
+  border-color: var(--ack-edit-border-color);
+}
+
+.btn-notice:disabled, button[disabled] {
+  color: var(--font-main);
+  background-color: var(--ack-edit-disabled-color);
+  border-color: var(--ack-edit-border-color);
+}
+
+.btn-generic {
+  border: 1px outset;
+  border-radius: 4px;
+  color: var(--font-focus);
+  background-color: var(--ack-edit-color);
+}
+
+.btn-generic:focus {
+  color: var(--font-main);
+  background-color: #286090;
+  border-color: #204d74;
+}
+
+.btn-generic.active,
+.btn-generic:active,
+.btn-genric:hover,
+.open > .dropdown-toggle.btn-generic {
+  color: var(--font-focus);
+  background-color: var(--ack-edit-focus-color);
+  border-color: var(--ack-edit-border-color);
+}
+
+.btn-primary {
+  height: 30px;
+}
+
+input[type="search"] {
+  border-radius: 15px;
+}
+
+[class*="span"].centerspan {
+    margin: 0 auto;
+    float: none;
+}
+
+.center {
+    margin: 0 auto;
+    float: none;
+}
+
+.table-striped tbody > tr:nth-child(odd) > td,
+.table-striped tbody > tr:nth-child(odd) > th {
+  background-color: var(--row-color-odd);
+}
+
+.table-striped > tbody > tr:nth-of-type(2n+1) {
+  background-color: var(--row-color-even);
+}
+
+.table-hover tbody tr:hover > td,
+.table-hover tbody tr:hover > th,
+.table-hover > tbody > tr:hover {
+  background-color: #9a9ea5;
+  color: var(--font-focus);
+}
+
+td{
+  color: var(--font-main);
+}
+
+.welcome {
+    background: none;
+    height: 180px;
+    width: 600px;
+    margin-left: 25%;
+    text-align: center;
+    color: var(--font-focus);
+    border: none;
+    vertical-align: middle;
+}
+
+.mozdeflogo{
+  width: 500px;
+}
+
+.tabcontent{
+    margin-top: 20px;
+}
+
+.tabnav a{
+    color: #5f6e81;
+}
+
+/* uncomment this login ui css to hide the local account/password signup options
+.logins-wrapper .svgtextlabel {
+  font-size: 20px;
+  color: white;
+}
+
+.loginButtons {
+  text-align: right;
+  color:white;
+}
+
+.login-password-form{
+  display:none;
+}
+
+.or {
+  display: none;
+}
+*/
+/* don't float the 'create account' link*/
+#login-buttons #signup-link{
+    float: none;
+}
+
+/* d3 circle styles */
+.successcircle{
+    /*
+    fill: rgb(217,206,178);
+
+    fill: rgb(60,179,113);
+    */
+    /*fill: rgba(0,150,221,.8);*/
+    fill: rgba(0,255,0,.5);
+}
+
+.failurecircle{
+    /*fill: rgb(213,222,217);*/
+    /*fill: #BD2C00;*/
+    /*fill: #f93;*/
+    /*fill: rgba(193, 56, 50, .8);*/
+    fill: rgba(255,0,0,.5);
+}
+
+circle:hover{
+  fill: var(--font-main);
+}
+
+.node {
+    stroke: var(--font-focus);
+    stroke-width: 1.5px;
+}
+
+.textlabel{
+    stroke-width: .2px;
+    stroke: var(--font-focus);
+}
+
+.vtagholders {
+    padding-left: 1px;
+    padding-right: 0px;
+}
+
+#header:after {
+    content: "";
+    display: none;
+}
+
+.mozillalogo{
+    width: 90px;
+}
+
+#header label{
+    display: inherit;
+    cursor: auto;
+    margin:.5em;
+}
+
+.mousepointer {
+    cursor: pointer;
+}
+
+.fa-xs {
+  font-size: .75em;
+}
+
+/* Attackers sidenav menu */
+sidenav {
+  background: var(--primary-bg-color);
+  border-left: 15px solid var(--secondary-color);
+  text-align: left;
+  font-weight: bolder;
+  position: fixed;
+  top: 0;
+  bottom: 0;
+  height: 100%;
+  right: -29em;
+  margin: 0em;
+  padding-top: 1em;
+  display: inline-block;
+  line-height: normal;
+  -webkit-transform: translateZ(0) scale(1, 1);
+  z-index: 3;
+  -webkit-transition: all 400ms ease;
+  -moz-transition: all 400ms ease;
+  -ms-transition: all 400ms ease;
+  -o-transition: all 400ms ease;
+  transition: all 400ms ease;
+}
+
+/*pull out triangle*/
+sidenav:after {
+    position: absolute;
+    content: ' ';
+    width: 0;
+    height: 0;
+    right: 405px;
+    top: 45%;
+    border-top: 45px solid transparent;
+    border-bottom: 45px solid transparent;
+    border-right: 45px solid var(--secondary-color);
+}
+sidenav ul {
+    width: 14em;
+    list-style-type: none;
+    margin: auto;
+    padding: 1em;
+}
+sidenav div{
+    margin:auto;
+}
+sidenav:hover {
+    right: 0;
+    width: 405px;
+}
+sidenav .filters-col .row {
+    margin-top: 45px;
+    padding: 0 0.5em;
+}
+sidenav .reset-filter {
+    text-align: center;
+    margin-top: 20px;
+}
+.form-horizontal .form-group {
+  margin-left: 5px;
+  margin-right: 5px;
+  padding-top: 5px;
+}
+
+@media screen and (max-width: 1000px) {
+  sidenav {
+    background: var(--primary-bg-color);
+    border-left: 15px solid var(--secondary-color);
+    text-align: left;
+    font-weight: bolder;
+    position: fixed;
+    top: 0;
+    bottom: 0;
+    height: 100%;
+    right: -16em;
+    margin: 0em;
+    padding-top: 1em;
+    display: inline-block;
+    line-height: normal;
+    -webkit-transform: translateZ(0) scale(1, 1);
+    z-index: 3;
+    -webkit-transition: all 400ms ease;
+    -moz-transition: all 400ms ease;
+    -ms-transition: all 400ms ease;
+    -o-transition: all 400ms ease;
+    transition: all 400ms ease;
+  }
+  sidenav:after {
+    right: 230px;
+    border-top: 0;
+    border-bottom: 0;
+    border-right: 0;
+    content: none;
+  }
+  sidenav ul {
+    width: 14em;
+    list-style-type: none;
+    margin: auto;
+    padding: 1em;
+  }
+  sidenav div{
+    margin:auto;
+  }
+  sidenav:hover {
+    right: 0;
+    width: 230px;
+    overflow-y: scroll;
+    scrollbar-width: inherit;
+    scrollbar-color: var(--secondary-color) black;
+  }
+  sidenav .filters-col .row {
+    margin-top: 25px;
+    padding: 0 1.5em;
+  }
+  sidenav .reset-filter {
+    text-align: center;
+    margin-top: 20px;
+  }
+  div.dc-chart {
+    float: none;
+  }
+}
+
+.fa-lg {
+  position: relative;
+  display: table-cell;
+  width: 60px;
+  height: 36px;
+  text-align: center;
+  vertical-align: middle;
+  font-size: 1.33333em;
+}
+
+/* globe styling */
+.globe-container  {
+  background: #000000;
+  color: #ffffff;
+}
+
+.globe-info {
+  font-size: 11px;
+  position: absolute;
+  bottom: 5px;
+  background-color: rgba(0, 0, 0, 0.8);
+  border-radius: 3px;
+  right: 10px;
+  padding: 10px;
+  color: var(--txt-secondary-color);
+}
+
+.globe-container a {
+  color: #aaa;
+  text-decoration: none;
+}
+
+.globe-container a:hover {
+  text-decoration: underline;
+}
+
+.globe-campaigns {
+  width: 270px;
+  position: absolute;
+  right: 280px;
+  top: 63px;
+  background-color: rgba(0, 0, 0, 0.2);
+  border-top: 1px solid rgba(255, 255, 255, 0.4);
+  padding: 10px;
+}
+
+.globe-time {
+  width: 400px;
+  position: absolute;
+  right: 150px;
+  top: 103px;
+  background-color: rgba(0, 0, 0, 0.2);
+  border-top: 1px solid rgba(255, 255, 255, 0.4);
+  padding: 10px;
+}
+
+.globe-facts {
+  width: 300px;
+  position: absolute;
+  left: 10px;
+  bottom: 0px;
+  background-color: rgba(0, 0, 0, 0.2);
+  border-top: 1px solid rgba(255, 255, 255, 0.4);
+  border-right: 1px solid rgba(255, 255, 255, 0.4);
+  padding: 10px;
+  font-size: 12px;
+  color: rgba(255, 255, 255, 0.9);
+  display: none;
+}
+
+.globe-campaigns .campaign{
+  font-size: 12px;
+  line-height: 26px;
+  height: 30px;
+  text-align: center;
+  float: left;
+  width: 60px;
+  color: rgba(255, 255, 255, 0.4);
+  cursor: pointer;
+  transition: all 0.1s ease-out;
+}
+
+.globe-time .time{
+  font-size: 12px;
+  line-height: 26px;
+  height: 30px;
+  text-align: center;
+  float: left;
+  width: 90px;
+  color: rgba(255, 255, 255, 0.4);
+  cursor: pointer;
+  transition: all 0.1s ease-out;
+}
+
+.globe-campaigns .campaign:hover,
+.globe-campaigns .campaign.active,
+.globe-time .time:hover,
+.globe-time .time.active {
+  font-size: 14px;
+  color: #fff;
+}
+
+.moz {
+  position: relative;
+  display: table-cell;
+  width: 60px;
+  height: 36px;
+  text-align: center;
+  vertical-align: middle;
+  font-size: 20px;
+}
+
+.main-menu:hover,
+nav.main-menu.expanded {
+  width: 225px;
+  overflow: visible;
+}
+
+.main-menu {
+  background: var(--primary-bg-color);
+  border-right: 1px solid var(--secondary-color);
+  position: fixed;
+  top: 0;
+  bottom: 0;
+  height: 100%;
+  left: 0;
+  width: 60px;
+  overflow: hidden;
+  -webkit-transition: width .05s linear;
+  transition: width .05s linear;
+  -webkit-transform: translateZ(0) scale(1, 1);
+  z-index: 1000;
+}
+
+.main-menu>ul {
+  margin: 7px 0;
+}
+
+.main-menu li {
+  position: relative;
+  display: block;
+  width: 225px;
+}
+
+.main-menu li:hover>a,
+nav.main-menu li.active>a,
+.dropdown-menu>li>a:hover,
+.dropdown-menu>li>a:focus,
+.dropdown-menu>.active>a,
+.dropdown-menu>.active>a:hover,
+.dropdown-menu>.active>a:focus,
+.no-touch .dashboard-page nav.dashboard-menu ul li:hover a,
+.dashboard-page nav.dashboard-menu ul li.active a {
+  color: rgb(0, 0, 0);
+  text-decoration: none;
+  background-color: var(--secondary-focus-color);
+}
+
+.main-menu li ul,
+.main-menu li ul li ul {
+  position: absolute;
+  height: auto;
+  min-width: 120px;
+  margin: 0;
+  background: var(--primary-bg-color);
+  opacity: 0;
+  visibility: hidden;
+  transition: all 300ms linear;
+  -o-transition: all 300ms linear;
+  -ms-transition: all 300ms linear;
+  -moz-transition: all 300ms linear;
+  -webkit-transition: all 300ms linear;
+  /*top: 130px;*/
+  z-index: 1000;
+  /* == */
+  left:200px;
+  top: 0px;
+  border-left: 5px solid var(--secondary-focus-color);
+  /* == */
+}
+
+.main-menu li ul:before {
+  content: "";
+  position: absolute;
+  /*top: -8px;
+  left: 23%;
+  border-bottom: 5px solid #97b4b4;
+  border-left: 5px solid transparent;
+  border-right: 5px solid transparent;*/
+  /* == */
+  top: 10px;
+  left: -9px;
+  border-right: 5px solid var(--secondary-focus-color);
+  border-bottom: 5px solid transparent;
+  border-top: 5px solid transparent;
+  /* == */
+}
+
+.main-menu li:hover > ul,
+.main-menu li ul li:hover > .main-menu li ul li {
+  display: block;
+  opacity: 1;
+  visibility: visible;
+  padding: 5px;
+  /*top: 100px;*/
+  /* == */
+  left:228px;
+  text-decoration: none;
+  /* == */
+}
+
+/*.main-menu li ul li {
+  float: none;
+}*/
+
+.main-menu li ul li a {
+  padding: 10px;
+  text-align: left;
+  border: 0;
+  border-bottom: 1px solid #EEE;
+  /* == */
+  height: auto;
+  /* == */
+}
+
+.main-menu li ul li a i {
+  font-size: 16px;
+  display: inline-block;
+  margin: 0 10px 0 0;
+}
+
+.main-menu li ul li ul {
+  left: 200px;
+  top: 0;
+  border: 0;
+  border-left: 4px solid var(--secondary-color);
+}
+
+.main-menu li ul li ul:before {
+  content: "";
+  position: absolute;
+  top: 15px;
+  /*left: -14px;*/
+  /* == */
+  left: -9px;
+  /* == */
+  border-right: 5px solid var(--secondary-color);
+  border-bottom: 5px solid transparent;
+  border-top: 5px solid transparent;
+}
+
+.main-menu li ul li:hover > ul {
+  top: 0px;
+  left: 200px;
+}
+
+.main-menu li>a {
+  position: relative;
+  display: table;
+  border-collapse: collapse;
+  border-spacing: 0;
+  color: rgb(255, 255, 255);
+  font-family: "Zilla Slab",sans-serif;
+  font-size: 14px;
+  text-decoration: none;
+  -webkit-transform: translateZ(0) scale(1, 1);
+  -webkit-transition: all .1s linear;
+  transition: all .5s linear;
+}
+
+.main-menu .nav-icon {
+  position: relative;
+  display: table-cell;
+  width: 60px;
+  height: 36px;
+  text-align: center;
+  vertical-align: middle;
+  font-size: 20px;
+}
+
+.main-menu .nav-text {
+  position: relative;
+  display: table-cell;
+  vertical-align: middle;
+  width: 163px;
+  font-family: 'Zilla Slab', serif;
+}
+
+.main-menu>ul.logout {
+  position: absolute;
+  left: 0;
+  bottom: 0;
+}
+
+.no-touch .scrollable.hover {
+  overflow-y: hidden;
+}
+
+.no-touch .scrollable.hover:hover {
+  overflow-y: auto;
+  overflow: visible;
+}
+
+a:hover,
+a:focus {
+  text-decoration: none;
+}
+
+nav {
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  -ms-user-select: none;
+  -o-user-select: none;
+  user-select: none;
+}
+
+nav ul,
+nav li {
+  outline: 0;
+  margin: 0;
+  padding: 0;
+}
+
+@font-face {
+  font-family: 'Zilla Slab';
+  font-style: normal;
+  font-weight: 300;
+  src: local('Zilla Slab'), local('ZillaSlab-Regular'), url(https://fonts.gstatic.com/s/zillaslab/v3/dFa6ZfeM_74wlPZtksIFajQ6_UyI.woff2) format('woff2');
+}
+
+.Heading {
+  margin-left: 100px;
+  margin-top: 0px;
+  color: #fff;
+  align-self: center;
+  font-family: 'Zilla Slab', serif;
+}
+
+h1 {
+  max-width: 600px;
+  color: #fff;
+  min-height: 100%;
+  background: rgb(0, 0, 0);
+  font-family: 'Zilla Slab', serif;
+}
+
+div#login-dropdown-list.accounts-dialog {
+    bottom: -16px;
+    top: unset;
+}

meteor/package-lock.json

@@ -11,436 +11,78 @@
         "regenerator-runtime": "^0.12.0"
       }
     },
-    "bcrypt": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-3.0.1.tgz",
-      "integrity": "sha512-DSTLQZdvzJ7znQ1WOqkN3X0Hutt6BTVaZNWyX8/B4P+s9SIxkYgtGKfgHokli1syPcWJUE63/kGVyV1ECA4d1A==",
+    "abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q=="
+    },
+    "ansi-regex": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
+    },
+    "aproba": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
+      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
+    },
+    "are-we-there-yet": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
+      "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
       "requires": {
-        "nan": "2.11.0",
-        "node-pre-gyp": "0.11.0"
-      },
-      "dependencies": {
-        "abbrev": {
-          "version": "1.1.1",
-          "bundled": true
-        },
-        "ansi-regex": {
-          "version": "2.1.1",
-          "bundled": true
-        },
-        "aproba": {
-          "version": "1.2.0",
-          "bundled": true
-        },
-        "are-we-there-yet": {
-          "version": "1.1.5",
-          "bundled": true,
-          "requires": {
-            "delegates": "^1.0.0",
-            "readable-stream": "^2.0.6"
-          }
-        },
-        "balanced-match": {
-          "version": "1.0.0",
-          "bundled": true
-        },
-        "brace-expansion": {
-          "version": "1.1.11",
-          "bundled": true,
-          "requires": {
-            "balanced-match": "^1.0.0",
-            "concat-map": "0.0.1"
-          }
-        },
-        "chownr": {
-          "version": "1.1.1",
-          "bundled": true
-        },
-        "code-point-at": {
-          "version": "1.1.0",
-          "bundled": true
-        },
-        "concat-map": {
-          "version": "0.0.1",
-          "bundled": true
-        },
-        "console-control-strings": {
-          "version": "1.1.0",
-          "bundled": true
-        },
-        "core-util-is": {
-          "version": "1.0.2",
-          "bundled": true
-        },
-        "debug": {
-          "version": "2.6.9",
-          "bundled": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "deep-extend": {
-          "version": "0.6.0",
-          "bundled": true
-        },
-        "delegates": {
-          "version": "1.0.0",
-          "bundled": true
-        },
-        "detect-libc": {
-          "version": "1.0.3",
-          "bundled": true
-        },
-        "fs-minipass": {
-          "version": "1.2.5",
-          "bundled": true,
-          "requires": {
-            "minipass": "^2.2.1"
-          }
-        },
-        "fs.realpath": {
-          "version": "1.0.0",
-          "bundled": true
-        },
-        "gauge": {
-          "version": "2.7.4",
-          "bundled": true,
-          "requires": {
-            "aproba": "^1.0.3",
-            "console-control-strings": "^1.0.0",
-            "has-unicode": "^2.0.0",
-            "object-assign": "^4.1.0",
-            "signal-exit": "^3.0.0",
-            "string-width": "^1.0.1",
-            "strip-ansi": "^3.0.1",
-            "wide-align": "^1.1.0"
-          }
-        },
-        "glob": {
-          "version": "7.1.2",
-          "bundled": true,
-          "requires": {
-            "fs.realpath": "^1.0.0",
-            "inflight": "^1.0.4",
-            "inherits": "2",
-            "minimatch": "^3.0.4",
-            "once": "^1.3.0",
-            "path-is-absolute": "^1.0.0"
-          }
-        },
-        "has-unicode": {
-          "version": "2.0.1",
-          "bundled": true
-        },
-        "iconv-lite": {
-          "version": "0.4.24",
-          "bundled": true,
-          "requires": {
-            "safer-buffer": ">= 2.1.2 < 3"
-          }
-        },
-        "ignore-walk": {
-          "version": "3.0.1",
-          "bundled": true,
-          "requires": {
-            "minimatch": "^3.0.4"
-          }
-        },
-        "inflight": {
-          "version": "1.0.6",
-          "bundled": true,
-          "requires": {
-            "once": "^1.3.0",
-            "wrappy": "1"
-          }
-        },
-        "inherits": {
-          "version": "2.0.3",
-          "bundled": true
-        },
-        "ini": {
-          "version": "1.3.5",
-          "bundled": true
-        },
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "bundled": true,
-          "requires": {
-            "number-is-nan": "^1.0.0"
-          }
-        },
-        "isarray": {
-          "version": "1.0.0",
-          "bundled": true
-        },
-        "minimatch": {
-          "version": "3.0.4",
-          "bundled": true,
-          "requires": {
-            "brace-expansion": "^1.1.7"
-          }
-        },
-        "minimist": {
-          "version": "0.0.8",
-          "bundled": true
-        },
-        "minipass": {
-          "version": "2.3.4",
-          "bundled": true,
-          "requires": {
-            "safe-buffer": "^5.1.2",
-            "yallist": "^3.0.0"
-          },
-          "dependencies": {
-            "safe-buffer": {
-              "version": "5.1.2",
-              "bundled": true
-            },
-            "yallist": {
-              "version": "3.0.2",
-              "bundled": true
-            }
-          }
-        },
-        "minizlib": {
-          "version": "1.1.0",
-          "bundled": true,
-          "requires": {
-            "minipass": "^2.2.1"
-          }
-        },
-        "mkdirp": {
-          "version": "0.5.1",
-          "bundled": true,
-          "requires": {
-            "minimist": "0.0.8"
-          }
-        },
-        "ms": {
-          "version": "2.0.0",
-          "bundled": true
-        },
-        "needle": {
-          "version": "2.2.3",
-          "bundled": true,
-          "requires": {
-            "debug": "^2.1.2",
-            "iconv-lite": "^0.4.4",
-            "sax": "^1.2.4"
-          }
-        },
-        "node-pre-gyp": {
-          "version": "0.11.0",
-          "bundled": true,
-          "requires": {
-            "detect-libc": "^1.0.2",
-            "mkdirp": "^0.5.1",
-            "needle": "^2.2.1",
-            "nopt": "^4.0.1",
-            "npm-packlist": "^1.1.6",
-            "npmlog": "^4.0.2",
-            "rc": "^1.2.7",
-            "rimraf": "^2.6.1",
-            "semver": "^5.3.0",
-            "tar": "^4"
-          }
-        },
-        "nopt": {
-          "version": "4.0.1",
-          "bundled": true,
-          "requires": {
-            "abbrev": "1",
-            "osenv": "^0.1.4"
-          }
-        },
-        "npm-bundled": {
-          "version": "1.0.5",
-          "bundled": true
-        },
-        "npm-packlist": {
-          "version": "1.1.11",
-          "bundled": true,
-          "requires": {
-            "ignore-walk": "^3.0.1",
-            "npm-bundled": "^1.0.1"
-          }
-        },
-        "npmlog": {
-          "version": "4.1.2",
-          "bundled": true,
-          "requires": {
-            "are-we-there-yet": "~1.1.2",
-            "console-control-strings": "~1.1.0",
-            "gauge": "~2.7.3",
-            "set-blocking": "~2.0.0"
-          }
-        },
-        "number-is-nan": {
-          "version": "1.0.1",
-          "bundled": true
-        },
-        "object-assign": {
-          "version": "4.1.1",
-          "bundled": true
-        },
-        "once": {
-          "version": "1.4.0",
-          "bundled": true,
-          "requires": {
-            "wrappy": "1"
-          }
-        },
-        "os-homedir": {
-          "version": "1.0.2",
-          "bundled": true
-        },
-        "os-tmpdir": {
-          "version": "1.0.2",
-          "bundled": true
-        },
-        "osenv": {
-          "version": "0.1.5",
-          "bundled": true,
-          "requires": {
-            "os-homedir": "^1.0.0",
-            "os-tmpdir": "^1.0.0"
-          }
-        },
-        "path-is-absolute": {
-          "version": "1.0.1",
-          "bundled": true
-        },
-        "process-nextick-args": {
-          "version": "2.0.0",
-          "bundled": true
-        },
-        "rc": {
-          "version": "1.2.8",
-          "bundled": true,
-          "requires": {
-            "deep-extend": "^0.6.0",
-            "ini": "~1.3.0",
-            "minimist": "^1.2.0",
-            "strip-json-comments": "~2.0.1"
-          },
-          "dependencies": {
-            "minimist": {
-              "version": "1.2.0",
-              "bundled": true
-            }
-          }
-        },
-        "readable-stream": {
-          "version": "2.3.5",
-          "bundled": true,
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.0.3",
-            "util-deprecate": "~1.0.1"
-          }
-        },
-        "rimraf": {
-          "version": "2.6.2",
-          "bundled": true,
-          "requires": {
-            "glob": "^7.0.5"
-          }
-        },
-        "safe-buffer": {
-          "version": "5.1.1",
-          "bundled": true
-        },
-        "safer-buffer": {
-          "version": "2.1.2",
-          "bundled": true
-        },
-        "sax": {
-          "version": "1.2.4",
-          "bundled": true
-        },
-        "semver": {
-          "version": "5.5.1",
-          "bundled": true
-        },
-        "set-blocking": {
-          "version": "2.0.0",
-          "bundled": true
-        },
-        "signal-exit": {
-          "version": "3.0.2",
-          "bundled": true
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "bundled": true,
-          "requires": {
-            "code-point-at": "^1.0.0",
-            "is-fullwidth-code-point": "^1.0.0",
-            "strip-ansi": "^3.0.0"
-          }
-        },
-        "string_decoder": {
-          "version": "1.0.3",
-          "bundled": true,
-          "requires": {
-            "safe-buffer": "~5.1.0"
-          }
-        },
-        "strip-ansi": {
-          "version": "3.0.1",
-          "bundled": true,
-          "requires": {
-            "ansi-regex": "^2.0.0"
-          }
-        },
-        "strip-json-comments": {
-          "version": "2.0.1",
-          "bundled": true
-        },
-        "tar": {
-          "version": "4.4.6",
-          "bundled": true,
-          "requires": {
-            "chownr": "^1.0.1",
-            "fs-minipass": "^1.2.5",
-            "minipass": "^2.3.3",
-            "minizlib": "^1.1.0",
-            "mkdirp": "^0.5.0",
-            "safe-buffer": "^5.1.2",
-            "yallist": "^3.0.2"
-          },
-          "dependencies": {
-            "safe-buffer": {
-              "version": "5.1.2",
-              "bundled": true
-            },
-            "yallist": {
-              "version": "3.0.2",
-              "bundled": true
-            }
-          }
-        },
-        "util-deprecate": {
-          "version": "1.0.2",
-          "bundled": true
-        },
-        "wide-align": {
-          "version": "1.1.3",
-          "bundled": true,
-          "requires": {
-            "string-width": "^1.0.2 || 2"
-          }
-        },
-        "wrappy": {
-          "version": "1.0.2",
-          "bundled": true
-        }
+        "delegates": "^1.0.0",
+        "readable-stream": "^2.0.6"
       }
     },
+    "balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
+    },
+    "bcrypt": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-3.0.6.tgz",
+      "integrity": "sha512-taA5bCTfXe7FUjKroKky9EXpdhkVvhE5owfxfLYodbrAR1Ul3juLmIQmIQBK4L9a5BuUcE6cqmwT+Da20lF9tg==",
+      "requires": {
+        "nan": "2.13.2",
+        "node-pre-gyp": "0.12.0"
+      }
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "requires": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "chownr": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
+      "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g=="
+    },
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+    },
+    "console-control-strings": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+    },
     "crossfilter2": {
       "version": "1.4.6",
       "resolved": "https://registry.npmjs.org/crossfilter2/-/crossfilter2-1.4.6.tgz",
@@ -463,10 +105,127 @@
         "d3": "^3"
       }
     },
+    "debug": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+      "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+      "requires": {
+        "ms": "^2.1.1"
+      }
+    },
+    "deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
+    },
+    "delegates": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
+    },
+    "detect-libc": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
+      "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
+    },
+    "fs-minipass": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz",
+      "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==",
+      "requires": {
+        "minipass": "^2.2.1"
+      }
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
+    },
+    "gauge": {
+      "version": "2.7.4",
+      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
+      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
+      "requires": {
+        "aproba": "^1.0.3",
+        "console-control-strings": "^1.0.0",
+        "has-unicode": "^2.0.0",
+        "object-assign": "^4.1.0",
+        "signal-exit": "^3.0.0",
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1",
+        "wide-align": "^1.1.0"
+      }
+    },
+    "glob": {
+      "version": "7.1.4",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
+      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+      "requires": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
+    "has-unicode": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
+    },
+    "iconv-lite": {
+      "version": "0.4.24",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "requires": {
+        "safer-buffer": ">= 2.1.2 < 3"
+      }
+    },
+    "ignore-walk": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz",
+      "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==",
+      "requires": {
+        "minimatch": "^3.0.4"
+      }
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "requires": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
+    },
+    "ini": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
+    },
+    "is-fullwidth-code-point": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+      "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+      "requires": {
+        "number-is-nan": "^1.0.0"
+      }
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+    },
     "jquery": {
-      "version": "1.12.3",
-      "resolved": "https://registry.npmjs.org/jquery/-/jquery-1.12.3.tgz",
-      "integrity": "sha512-FzM42/Ew+Hb8ha2OlhHRBLgWIZS32gZ0+NvWTf+ZvVvGaIlJkOiXQyb7VBjv4L6fJfmTrRf3EsAmbfsHDhfemw=="
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.4.0.tgz",
+      "integrity": "sha512-ggRCXln9zEqv6OqAGXFEcshF5dSBvCkzj6Gm2gzuR5fWawaX8t7cxKVkkygKODrDAzKdoYw3l/e3pm3vlT4IbQ=="
     },
     "jquery-ui": {
       "version": "1.12.1",
@@ -1005,10 +764,156 @@
         }
       }
     },
+    "minimatch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "requires": {
+        "brace-expansion": "^1.1.7"
+      }
+    },
+    "minimist": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
+      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+    },
+    "minipass": {
+      "version": "2.3.5",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz",
+      "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==",
+      "requires": {
+        "safe-buffer": "^5.1.2",
+        "yallist": "^3.0.0"
+      }
+    },
+    "minizlib": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz",
+      "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==",
+      "requires": {
+        "minipass": "^2.2.1"
+      }
+    },
+    "mkdirp": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "requires": {
+        "minimist": "0.0.8"
+      }
+    },
+    "ms": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+      "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
+    },
     "nan": {
-      "version": "2.11.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.0.tgz",
-      "integrity": "sha512-F4miItu2rGnV2ySkXOQoA8FKz/SR2Q2sWP0sbTxNxz/tuokeC8WxOhPMcwi0qIyGtVn/rrSeLbvVkznqCdwYnw=="
+      "version": "2.13.2",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.13.2.tgz",
+      "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw=="
+    },
+    "needle": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.1.tgz",
+      "integrity": "sha512-CaLXV3W8Vnbps8ZANqDGz7j4x7Yj1LW4TWF/TQuDfj7Cfx4nAPTvw98qgTevtto1oHDrh3pQkaODbqupXlsWTg==",
+      "requires": {
+        "debug": "^4.1.0",
+        "iconv-lite": "^0.4.4",
+        "sax": "^1.2.4"
+      }
+    },
+    "node-pre-gyp": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz",
+      "integrity": "sha512-4KghwV8vH5k+g2ylT+sLTjy5wmUOb9vPhnM8NHvRf9dHmnW/CndrFXy2aRPaPST6dugXSdHXfeaHQm77PIz/1A==",
+      "requires": {
+        "detect-libc": "^1.0.2",
+        "mkdirp": "^0.5.1",
+        "needle": "^2.2.1",
+        "nopt": "^4.0.1",
+        "npm-packlist": "^1.1.6",
+        "npmlog": "^4.0.2",
+        "rc": "^1.2.7",
+        "rimraf": "^2.6.1",
+        "semver": "^5.3.0",
+        "tar": "^4"
+      }
+    },
+    "nopt": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz",
+      "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=",
+      "requires": {
+        "abbrev": "1",
+        "osenv": "^0.1.4"
+      }
+    },
+    "npm-bundled": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz",
+      "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g=="
+    },
+    "npm-packlist": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz",
+      "integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==",
+      "requires": {
+        "ignore-walk": "^3.0.1",
+        "npm-bundled": "^1.0.1"
+      }
+    },
+    "npmlog": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
+      "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
+      "requires": {
+        "are-we-there-yet": "~1.1.2",
+        "console-control-strings": "~1.1.0",
+        "gauge": "~2.7.3",
+        "set-blocking": "~2.0.0"
+      }
+    },
+    "number-is-nan": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
+      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
+    },
+    "object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "os-homedir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
+      "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
+    },
+    "os-tmpdir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
+      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
+    },
+    "osenv": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
+      "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
+      "requires": {
+        "os-homedir": "^1.0.0",
+        "os-tmpdir": "^1.0.0"
+      }
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
     },
     "pivottable": {
       "version": "2.22.0",
@@ -1026,16 +931,141 @@
         "jquery": ">=1.6.0"
       }
     },
+    "process-nextick-args": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+    },
+    "rc": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "requires": {
+        "deep-extend": "^0.6.0",
+        "ini": "~1.3.0",
+        "minimist": "^1.2.0",
+        "strip-json-comments": "~2.0.1"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
+        }
+      }
+    },
+    "readable-stream": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+      "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+      "requires": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
     "regenerator-runtime": {
       "version": "0.12.1",
       "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz",
       "integrity": "sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg=="
     },
+    "rimraf": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
+      "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
+      "requires": {
+        "glob": "^7.1.3"
+      }
+    },
+    "safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
+    "safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+    },
+    "sax": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+    },
+    "semver": {
+      "version": "5.7.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
+      "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA=="
+    },
+    "set-blocking": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
+    },
+    "signal-exit": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
+      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
+    },
+    "string-width": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+      "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
+      "requires": {
+        "code-point-at": "^1.0.0",
+        "is-fullwidth-code-point": "^1.0.0",
+        "strip-ansi": "^3.0.0"
+      }
+    },
+    "string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "requires": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
+    "strip-ansi": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
+    "strip-json-comments": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
+    },
+    "tar": {
+      "version": "4.4.8",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz",
+      "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==",
+      "requires": {
+        "chownr": "^1.1.1",
+        "fs-minipass": "^1.2.5",
+        "minipass": "^2.3.4",
+        "minizlib": "^1.1.1",
+        "mkdirp": "^0.5.0",
+        "safe-buffer": "^5.1.2",
+        "yallist": "^3.0.2"
+      }
+    },
     "three-full": {
       "version": "11.3.2",
       "resolved": "https://registry.npmjs.org/three-full/-/three-full-11.3.2.tgz",
       "integrity": "sha512-0qv9kLe2bp1ksV3mgjyzWtIBO//U19+RzoyAuSGkSKKYF0M3zxAwWU5L6rRhVj6L/GGlbUb4EHMa8QhcW8/Rtw=="
     },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+    },
     "uuid": {
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
@@ -1045,6 +1075,24 @@
       "version": "10.8.0",
       "resolved": "https://registry.npmjs.org/validator/-/validator-10.8.0.tgz",
       "integrity": "sha512-mXqMxfCh5NLsVgYVKl9WvnHNDPCcbNppHSPPowu0VjtSsGWVY+z8hJF44edLR1nbLNzi3jYoYsIl8KZpioIk6g=="
+    },
+    "wide-align": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
+      "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
+      "requires": {
+        "string-width": "^1.0.2 || 2"
+      }
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+    },
+    "yallist": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
+      "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A=="
     }
   }
 }

meteor/package.json

@@ -9,9 +9,9 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.1.2",
-    "bcrypt": "^3.0.1",
+    "bcrypt": "^3.0.6",
     "dc": "^2.1.10",
-    "jquery": "^1.12.3",
+    "jquery": "^3.4.0",
     "jquery-ui": "^1.12.1",
     "meteor-node-stubs": "^0.4.1",
     "pivottable": "^2.22.0",

meteor/public/images/logo-elastic-kibana-dk.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 66 80"><defs><style>.cls-1{fill:#fff;fill-rule:evenodd;}</style></defs><title>Kibana White</title><polygon class="cls-1" points="58 8 8 8 8 65.58 58 8"/><path class="cls-1" d="M37,41.31,12.53,69.51,10.37,72H57A50,50,0,0,0,37,41.31Z"/></svg>

meteor/server/mozdef.js

@@ -193,6 +193,7 @@ function registerLoginViaHeader() {
 
         //grab the email from the header
         var userEmail = this.connection.httpHeaders[headerName];
+        console.log( 'target header:', userEmail );
 
         //our authentication logic
         //check for user email header
@@ -203,8 +204,7 @@ function registerLoginViaHeader() {
                 error: handleError( "SSO Login failure: email not found in the 'via' http header" )
             };
         }
-
-        console.log( 'target header:', userEmail );
+        
         console.log( 'handling login request', loginRequest );
 
         //we create a user if needed, and get the userId

mozdef_util/mozdef_util/query_models/search_query.py

@@ -46,7 +46,7 @@ class SearchQuery(object):
     def add_aggregation(self, input_obj):
         self.append_to_array(self.aggregation, input_obj)
 
-    def execute(self, elasticsearch_client, indices=['events-*'], size=1000, request_timeout=30):
+    def execute(self, elasticsearch_client, indices=['events', 'events-previous'], size=1000, request_timeout=30):
         if self.must == [] and self.must_not == [] and self.should == [] and self.aggregation == []:
             raise AttributeError('Must define a must, must_not, should query, or aggregation')
 

requirements.txt

@@ -6,6 +6,7 @@ boto3==1.7.67
 botocore==1.10.67
 bottle==0.12.4
 celery==4.1.0
+celery[sqs]==4.1.0
 cffi==1.9.1
 configlib==2.0.3
 configparser==3.5.0b2
@@ -40,6 +41,7 @@ packaging==16.8
 pyasn1==0.1.9
 pyasn1-modules==0.0.5
 pyOpenSSL==18.0.0
+pycurl==7.43.0.2
 pycparser==2.17
 pymongo==3.6.1
 pynsive==0.2.6

rest/index.conf

@@ -1,5 +1,4 @@
 [options]
-kibanaurl=http://localhost:9090/app/kibana
 esservers=http://localhost:9200
 mongoport=3002
 listen_host=0.0.0.0

rest/index.py

@@ -527,11 +527,7 @@ def kibanaDashboards():
         for dashboard in results['hits']:
             resultsList.append({
                 'name': dashboard['_source']['dashboard']['title'],
-                'url': "%s#/%s/%s" % (
-                    options.kibanaurl,
-                    "dashboard",
-                    dashboard['_id']
-                )
+                'id': dashboard['_id']
             })
 
     except ElasticsearchInvalidIndex as e:
@@ -627,9 +623,6 @@ def initConfig():
     options.esservers = list(getConfig('esservers',
                                        'http://localhost:9200',
                                        options.configfile).split(','))
-    options.kibanaurl = getConfig('kibanaurl',
-                                  'http://localhost:9090',
-                                  options.configfile)
 
     # mongo connectivity options
     options.mongohost = getConfig('mongohost', 'localhost', options.configfile)

rest/plugins/logincounts.py

@@ -80,10 +80,7 @@ class message(object):
         search_query.add_aggregation(Aggregation('details.success'))
         search_query.add_aggregation(Aggregation('details.username'))
 
-        # We want to select all event indices
-        # and filter out the window based on timestamp
-        # from the search query
-        results = search_query.execute(es_client, indices=['events-*'])
+        results = search_query.execute(es_client, indices=['events','events-previous'])
 
         # any usernames or words to ignore
         # especially useful if ES is analyzing the username field and breaking apart user@somewhere.com

tests/alerts/alert_test_suite.py

@@ -173,7 +173,12 @@ class AlertTestSuite(UnitTestSuite):
         if self.deadman:
             return
 
-        assert len(found_alert['_source']['events']) == len(test_case.full_events)
+        # If we override the number of expected events, let's use that value
+        num_events = len(test_case.full_events)
+        if hasattr(self, 'num_samples'):
+            num_events = self.num_samples
+        assert len(found_alert['_source']['events']) == num_events
+
         for event in found_alert['_source']['events']:
             event_id = event['documentid']
             found_event = self.es_client.get_event_by_id(event_id)

tests/alerts/test_cloudtrail_excessive_describe.py

@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+from positive_alert_test_case import PositiveAlertTestCase
+from negative_alert_test_case import NegativeAlertTestCase
+from alert_test_suite import AlertTestSuite
+
+
+class TestCloudtrailExcessiveDescribe(AlertTestSuite):
+    alert_filename = "cloudtrail_excessive_describe"
+    alert_classname = "AlertCloudtrailExcessiveDescribe"
+    num_samples = 2
+
+    default_event = {
+        "_source": {
+            "category": "AwsApiCall",
+            "source": "cloudtrail",
+            "details": {
+                "eventverb": "Describe",
+                "source": "dynamodb.application-autoscaling.amazonaws.com",
+            }
+        }
+    }
+
+    # This alert is the expected result from running this task
+    default_alert = {
+        "category": "access",
+        "tags": ['cloudtrail'],
+        "severity": "WARNING",
+        "summary": 'Excessive Describe calls on dynamodb.application-autoscaling.amazonaws.com (50)',
+    }
+
+    test_cases = []
+
+    test_cases.append(
+        PositiveAlertTestCase(
+            description="Positive test with default events and default alert expected",
+            events=AlertTestSuite.create_events(default_event, 50),
+            expected_alert=default_alert
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 50)
+    for event in events:
+        event['_source']['source'] = 'bad'
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with incorrect source",
+            events=events,
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 50)
+    for event in events:
+        event['_source']['details']['eventverb'] = 'bad'
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with incorrect details.eventverb",
+            events=events,
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 50)
+    for event in events:
+        event['_source']['details']['source'] = None
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with non-existent details.source",
+            events=events,
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 50)
+    for event in events:
+        event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 21})
+        event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 21})
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with old timestamp",
+            events=events,
+        )
+    )

tests/alerts/test_cloudtrail_logging_disabled.py

@@ -11,10 +11,12 @@ class TestAlertCloudtrailLoggingDisabled(AlertTestSuite):
     # alert to trigger
     default_event = {
         "_source": {
-            "eventname": "StopLogging",
             "source": "cloudtrail",
-            "requestparameters": {
-                "name": "cloudtrail_example_name",
+            "details": {
+                "eventname": "StopLogging",
+                "requestparameters": {
+                    "name": "cloudtrail_example_name",
+                }
             }
         }
     }
@@ -59,7 +61,7 @@ class TestAlertCloudtrailLoggingDisabled(AlertTestSuite):
     )
 
     event = AlertTestSuite.create_event(default_event)
-    event['_source']['eventname'] = 'Badeventname'
+    event['_source']['details']['eventname'] = 'Badeventname'
     test_cases.append(
         NegativeAlertTestCase(
             description="Negative test case with bad eventName",

tests/alerts/test_cloudtrail_public_bucket.py

@@ -0,0 +1,98 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+from positive_alert_test_case import PositiveAlertTestCase
+from negative_alert_test_case import NegativeAlertTestCase
+from alert_test_suite import AlertTestSuite
+
+
+class TestCloudtrailPublicBucket(AlertTestSuite):
+    alert_filename = "cloudtrail_public_bucket"
+    alert_classname = "AlertCloudtrailPublicBucket"
+
+    default_event = {
+        "_source": {
+            "source": "cloudtrail",
+            "details": {
+                "requestparameters": {
+                    "bucketpolicy": {
+                        "version": "2012-10-17",
+                        "statement": [{
+                            "action": "s3:GetObject",
+                            "principal": "*",
+                            "resource": "arn:aws:s3:::testbucket/*",
+                            "effect": "Allow",
+                            "sid": "AllowGetObject"
+                        }]
+                    },
+                    "bucketname": "testbucket"
+                },
+                "eventname": "PutBucketPolicy",
+            },
+        }
+    }
+
+    # This alert is the expected result from running this task
+    default_alert = {
+        "category": "access",
+        "tags": ['cloudtrail'],
+        "severity": "INFO",
+        "summary": 'The s3 bucket testbucket is listed as public',
+    }
+
+    test_cases = []
+
+    test_cases.append(
+        PositiveAlertTestCase(
+            description="Positive test with default events and default alert expected",
+            events=[AlertTestSuite.create_event(default_event)],
+            expected_alert=default_alert
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    event['_source']['source'] = 'bad'
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with incorrect source",
+            events=[event],
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    event['_source']['details']['eventname'] = 'bad'
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with incorrect details.eventname",
+            events=[event],
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    del(event['_source']['details']['requestparameters']['bucketpolicy']['statement'][0]['principal'])
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with missing field",
+            events=[event],
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    event['_source']['details']['requestparameters']['bucketpolicy']['statement'][0]['principal'] = 'bad'
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with incorrect principal",
+            events=[event],
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 21})
+    event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 21})
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with old timestamp",
+            events=[event],
+        )
+    )

tests/rest/index.conf

@@ -1,5 +1,4 @@
 [options]
-kibanaurl=http://localhost:9090/app/kibana
 esservers=http://localhost:9200
 mongoport=3002
 listen_host=0.0.0.0

tests/rest/test_rest_index.py

@@ -72,10 +72,10 @@ class TestKibanaDashboardsRoute(RestTestSuite):
 
             json_resp.sort()
 
-            assert json_resp[1]['url'].endswith("/app/kibana#/dashboard/Example-SSH-Dashboard") is True
+            assert json_resp[1]['id'] == "Example-SSH-Dashboard"
             assert json_resp[1]['name'] == 'Example SSH Dashboard'
 
-            assert json_resp[0]['url'].endswith("/app/kibana#/dashboard/Example-FTP-Dashboard") is True
+            assert json_resp[0]['id'] == "Example-FTP-Dashboard"
             assert json_resp[0]['name'] == 'Example FTP Dashboard'
 
 

tests/unit_test_suite.py

@@ -60,6 +60,7 @@ class UnitTestSuite(object):
         self.es_client.create_alias('events', self.event_index_name)
         self.es_client.create_index(self.previous_event_index_name, index_config=self.mapping_options)
         self.es_client.create_alias('events-previous', self.previous_event_index_name)
+        self.es_client.create_alias_multiple_indices('events-weekly', ['events', 'events-previous'])
         self.es_client.create_index(self.alert_index_name, index_config=self.mapping_options)
         self.es_client.create_alias('alerts', self.alert_index_name)