add options for multiple es servers

cron/cloudTrailAlerts.py

@@ -133,7 +133,7 @@ def createAlerts(es,esResults):
 def main():
     logger.debug('starting')
     logger.debug(options)
-    es=pyes.ES(("http",options.esserver,options.esport))
+    es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
     results=esCloudTrailSearch(es)
     createAlerts(es,results)
     logger.debug('finished')
@@ -150,8 +150,7 @@ def initConfig():
     options.sysloghostname=getConfig('sysloghostname','localhost',options.configfile)   #syslog hostname
     options.syslogport=getConfig('syslogport',514,options.configfile)                   #syslog port
     #elastic search server settings
-    options.esserver=getConfig('esserver','localhost',options.configfile)
-    options.esport=getConfig('esport',9200,options.configfile)
+    options.esservers=list(getConfig('esservers','http://localhost:9200',options.configfile).split(','))
     
 if __name__ == '__main__':
     parser=OptionParser()

cron/cloudtrail2mozdef.py

@@ -55,7 +55,7 @@ def main():
     logger.debug('started')
     #logger.debug(options)
     try:
-        es=pyes.ES(("http",options.esserver,options.esport))
+        es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
         boto.connect_cloudtrail(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
         #capture the time we start running so next time we catch any files created while we run.
         lastrun=toUTC(datetime.now()).isoformat()
@@ -109,8 +109,7 @@ def initConfig():
     options.defaultTimeZone=getConfig('defaulttimezone','US/Pacific',options.configfile)
     options.aws_access_key_id=getConfig('aws_access_key_id','',options.configfile)          #aws credentials to use to connect to cloudtrail
     options.aws_secret_access_key=getConfig('aws_secret_access_key','',options.configfile)
-    options.esserver=getConfig('esserver','localhost',options.configfile)
-    options.esport=getConfig('esport',9200,options.configfile)
+    options.esservers=list(getConfig('esservers','http://localhost:9200',options.configfile).split(','))
     options.lastrun=toUTC(getConfig('lastrun',toUTC(datetime.now()-timedelta(hours=1)),options.configfile))
     options.purge=getConfig('purge',False,options.configfile)
  

cron/esMaint.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 source  /home/mozdef/envs/mozdef/bin/activate
-/home/mozdef/envs/mozdef/cron/rotateIndexes.py
+/home/mozdef/envs/mozdef/cron/rotateIndexes.py -c /home/mozdef/envs/mozdef/cron/rotateIndexes.conf
 

cron/rotateIndexes.py

@@ -3,17 +3,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import sys
 import pyes
-from optparse import OptionParser
 from datetime import datetime
 from datetime import date
+from configlib import getConfig,OptionParser
 
 def esRotateIndexes():
-    es=pyes.ES(("http",options.server,options.port))
+    es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
     
-    #indexes=es.status()['indices'].keys()
     indexes=es.indices.stats()['indices'].keys()
-    print('[*]\tcurrent indexes: {0}'.format(indexes))
+    #print('[*]\tcurrent indexes: {0}'.format(indexes))
     
     #set index names events-MMYYDD, etc.
     dtNow=datetime.utcnow()
@@ -24,15 +24,15 @@ def esRotateIndexes():
     print('[*]\tlooking for current daily indexes: {0},{1},{2}'.format(eventsIndexName,alertsIndexName,correlationsIndexName))
     
     if eventsIndexName not in indexes:
-        print('[-]\tcreating: {0}'.format(eventsIndexName))
+        print('[*]\tcreating: {0}'.format(eventsIndexName))
         #es.create_index(eventsIndexName)
         es.indices.create_index(eventsIndexName)
     if alertsIndexName not in indexes:
-        print('[-]\tcreating: {0}'.format(alertsIndexName))
+        print('[*]\tcreating: {0}'.format(alertsIndexName))
         #es.create_index(alertsIndexName)
         es.indices.create_index(alertsIndexName)
     if correlationsIndexName not in indexes:
-        print('[-]\tcreating: {0}'.format(correlationsIndexName))
+        print('[*]\tcreating: {0}'.format(correlationsIndexName))
         #es.create_index(correlationsIndexName)
         es.indices.create_index(correlationsIndexName)
     
@@ -41,14 +41,12 @@ def esRotateIndexes():
     es.indices.set_alias('alerts', alertsIndexName)
     es.indices.set_alias('correlations', correlationsIndexName)
     
+def initConfig():
+    options.esservers=list(getConfig('esservers','http://localhost:9200',options.configfile).split(','))
     
-    
-        
 if __name__ == '__main__':
     parser=OptionParser()
-    parser=OptionParser()
-    parser.add_option("-s", "--server", dest='server' , default='localhost', help="elastic search servername or ip address")
-    parser.add_option("-p", "--port", dest='port', default=9200, type="int", help="elastic search port")
+    parser.add_option("-c", dest='configfile' , default='{0}.conf'.format(sys.argv[0]), help="configuration file to use")
     (options,args) = parser.parse_args()
-
+    initConfig()
     esRotateIndexes()