Merge remote-tracking branch 'origin/master' into update_cloudymozdef_es_version

cloudy_mozdef/cloudformation/mozdef-instance.yml

@@ -340,9 +340,11 @@ Resources:
             - content: |
                 # This configures the worker that pulls in CloudTrail logs
                 OPTIONS_TASKEXCHANGE=${CloudTrailSQSNotificationQueueName}
+                OPTIONS_REGION=${AWS::Region}
               path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env
             - content: |
                 OPTIONS_TASKEXCHANGE=${MozDefSQSQueueName}
+                OPTIONS_REGION=${AWS::Region}
               path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
             - content: |
                 [Unit]

cron/healthToMongo.py

@@ -77,11 +77,14 @@ def getEsNodesStats():
         load_str = "{0},{1},{2}".format(load_average['1m'], load_average['5m'], load_average['15m'])
         hostname = nodeid
         if 'host' in jsonobj['nodes'][nodeid]:
-            hostname=jsonobj['nodes'][nodeid]['host']
+            hostname = jsonobj['nodes'][nodeid]['host']
+
+        disk_free = "{0:.2f}".format(jsonobj['nodes'][nodeid]['fs']['total']['free_in_bytes'] / (1024 * 1024 * 1024))
+        disk_total = "{0:.2f}".format(jsonobj['nodes'][nodeid]['fs']['total']['total_in_bytes'] / (1024 * 1024 * 1024))
         results.append({
             'hostname': hostname,
-            'disk_free': jsonobj['nodes'][nodeid]['fs']['total']['free_in_bytes'] / (1024 * 1024 * 1024),
-            'disk_total': jsonobj['nodes'][nodeid]['fs']['total']['total_in_bytes'] / (1024 * 1024 * 1024),
+            'disk_free': disk_free,
+            'disk_total': disk_total,
             'mem_heap_per': jsonobj['nodes'][nodeid]['jvm']['mem']['heap_used_percent'],
             'gc_old': jsonobj['nodes'][nodeid]['jvm']['gc']['collectors']['old']['collection_time_in_millis'] / 1000,
             'cpu_usage': jsonobj['nodes'][nodeid]['os']['cpu']['percent'],

docs/source/installation.rst

@@ -113,6 +113,13 @@ Then::
   PYCURL_SSL_LIBRARY=nss pip install -r requirements.txt
 
 
+If you're using Mac OS X::
+
+  export PYCURL_SSL_LIBRARY=openssl
+  export LDFLAGS=-L/usr/local/opt/openssl/lib;export CPPFLAGS=-I/usr/local/opt/openssl/include
+  pip install -r requirements.txt
+
+
 Copy the following into a file called .bash_profile for the mozdef user within /opt/mozdef::
 
   [mozdef@server ~]$ vim /opt/mozdef/.bash_profile

mq/esworker_cloudtrail.py

@@ -228,10 +228,10 @@ class taskConsumer(object):
             self.flush_wait_time = (response['Credentials']['Expiration'] - current_time).seconds - 3
         else:
             role_creds = {}
+        role_creds['region_name'] = options.region
         self.s3_client = boto3.client(
             's3',
-            region_name=options.region,
-            **role_creds
+            **get_aws_credentials(**role_creds)
         )
 
     def reauth_timer(self):
@@ -284,11 +284,10 @@ class taskConsumer(object):
                 logger.info('Received network related error...reconnecting')
                 time.sleep(5)
                 self.sqs_queue = connect_sqs(
-                    task_exchange=options.taskexchange,
-                    **get_aws_credentials(
-                        options.region,
-                        options.accesskey,
-                        options.secretkey)
+                    region_name=options.region,
+                    aws_access_key_id=options.accesskey,
+                    aws_secret_access_key=options.secretkey,
+                    task_exchange=options.taskexchange
                 )
             time.sleep(options.sleep_time)
 
@@ -383,11 +382,10 @@ def main():
         sys.exit(1)
 
     sqs_queue = connect_sqs(
-        task_exchange=options.taskexchange,
-        **get_aws_credentials(
-            options.region,
-            options.accesskey,
-            options.secretkey)
+        region_name=options.region,
+        aws_access_key_id=options.accesskey,
+        aws_secret_access_key=options.secretkey,
+        task_exchange=options.taskexchange
     )
     # consume our queue
     taskConsumer(sqs_queue, es).run()
@@ -413,7 +411,6 @@ def initConfig():
     # rabbit message queue options
     options.mqserver = getConfig('mqserver', 'localhost', options.configfile)
     options.taskexchange = getConfig('taskexchange', 'eventtask', options.configfile)
-    options.eventexchange = getConfig('eventexchange', 'events', options.configfile)
     # rabbit: how many messages to ask for at once from the message queue
     options.prefetch = getConfig('prefetch', 10, options.configfile)
     # rabbit: user creds

mq/esworker_sns_sqs.py

@@ -24,7 +24,6 @@ from mozdef_util.utilities.logger import logger, initLogger
 from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../"))
-from mq.lib.aws import get_aws_credentials
 from mq.lib.plugins import sendEventToPlugins, registerPlugins
 from mq.lib.sqs import connect_sqs
 
@@ -192,11 +191,11 @@ def main():
         sys.exit(1)
 
     sqs_queue = connect_sqs(
-        task_exchange=options.taskexchange,
-        **get_aws_credentials(
-            options.region,
-            options.accesskey,
-            options.secretkey))
+        region_name=options.region,
+        aws_access_key_id=options.accesskey,
+        aws_secret_access_key=options.secretkey,
+        task_exchange=options.taskexchange
+    )
     # consume our queue
     taskConsumer(sqs_queue, es, options).run()
 

mq/esworker_sqs.py

@@ -29,7 +29,6 @@ from mozdef_util.utilities.logger import logger, initLogger
 from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../"))
-from mq.lib.aws import get_aws_credentials
 from mq.lib.plugins import sendEventToPlugins, registerPlugins
 from mq.lib.sqs import connect_sqs
 
@@ -331,11 +330,11 @@ def main():
         sys.exit(1)
 
     sqs_queue = connect_sqs(
-        task_exchange=options.taskexchange,
-        **get_aws_credentials(
-            options.region,
-            options.accesskey,
-            options.secretkey))
+        region_name=options.region,
+        aws_access_key_id=options.accesskey,
+        aws_secret_access_key=options.secretkey,
+        task_exchange=options.taskexchange
+    )
     # consume our queue
     taskConsumer(sqs_queue, es).run()
 
@@ -355,7 +354,6 @@ def initConfig():
     # rabbit message queue options
     options.mqserver = getConfig('mqserver', 'localhost', options.configfile)
     options.taskexchange = getConfig('taskexchange', 'eventtask', options.configfile)
-    options.eventexchange = getConfig('eventexchange', 'events', options.configfile)
     # rabbit: how many messages to ask for at once from the message queue
     options.prefetch = getConfig('prefetch', 10, options.configfile)
     # rabbit: user creds

mq/lib/aws.py

@@ -4,14 +4,14 @@
 # Copyright (c) 2017 Mozilla Corporation
 
 
-def get_aws_credentials(region=None, access_key=None, secret_key=None, security_token=None):
+def get_aws_credentials(region_name=None, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None):
     result = {}
-    if region and region != '<add_region>':
-        result['region_name'] = region
-    if access_key and access_key != '<add_accesskey>':
-        result['aws_access_key_id'] = access_key
-    if secret_key and secret_key != '<add_secretkey>':
-        result['aws_secret_access_key'] = secret_key
-    if security_token:
-        result['security_token'] = security_token
+    if region_name and region_name != '<add_region>':
+        result['region_name'] = region_name
+    if aws_access_key_id and aws_access_key_id != '<add_accesskey>':
+        result['aws_access_key_id'] = aws_access_key_id
+    if aws_secret_access_key and aws_secret_access_key != '<add_secretkey>':
+        result['aws_secret_access_key'] = aws_secret_access_key
+    if aws_session_token:
+        result['aws_session_token'] = aws_session_token
     return result

mq/lib/sqs.py

@@ -1,18 +1,12 @@
 import boto3
+from .aws import get_aws_credentials
 
 
 def connect_sqs(region_name=None, aws_access_key_id=None,
                 aws_secret_access_key=None, task_exchange=None):
-    credentials = {}
-    if aws_access_key_id is not None:
-        credentials['aws_access_key_id'] = aws_access_key_id
-    if aws_secret_access_key is not None:
-        credentials['aws_secret_access_key'] = aws_secret_access_key
-
     sqs = boto3.resource(
         'sqs',
-        region_name=region_name,
-        **credentials
+        **get_aws_credentials(region_name, aws_access_key_id, aws_secret_access_key)
     )
     queue = sqs.get_queue_by_name(QueueName=task_exchange)
     return queue

mq/plugins/cloudtrail.py

@@ -23,7 +23,8 @@ class message(object):
             'details.apiversion',
             'details.serviceeventdetails',
             'details.requestparameters.attribute',
-            'details.requestparameters.bucketpolicy.statement.principal',
+            'details.requestparameters.bucketpolicy.statement.principal.service',
+            'details.requestparameters.bucketpolicy.statement.principal.aws',
             'details.requestparameters.callerreference',
             'details.requestparameters.description',
             'details.requestparameters.describeflowlogsrequest.filter.value',