Merge branch 'master' of https://github.com/jeffbryner/MozDef

meteor/app/client/alertdetails.js

@@ -12,14 +12,16 @@ Anthony Verez averez@mozilla.com
 if (Meteor.isClient) {
 
     //alert details helpers
-    Template.alertdetails.thisalertevents = function () {
+    Template.alertdetails.helpers ({
+        thisalertevents: function () {
             return alerts.findOne({'esmetadata.id': Session.get('alertID')}).events;
-    };
+        },
     
-    Template.alertdetails.kibanaurl = function () {
+        kibanaurl: function () {
             url=getSetting('kibanaURL') + '#/dashboard/script/alert.js?id=' + Session.get('alertID');
             return url;
-    };
+        }
+    });
     
     Template.alertdetails.events({
         "click .makeinvestigation": function(event, template) {

meteor/app/client/alertsummary.js

@@ -14,7 +14,8 @@ if (Meteor.isClient) {
     Session.set('alertsSearch',null);
     Session.set('alertsDisplayed',0);
 
-    Template.alertssummary.selectedalerts = function () {
+    Template.alertssummary.helpers({
+        selectedalerts: function () {
             //console.log(moment().format(),Session.get('alertsSearch'));
             
             Session.set('alertsDisplayed',
@@ -40,8 +41,8 @@ if (Meteor.isClient) {
                                 sort: {utcepoch: -1},
                                 limit: Session.get('alertsrecordlimit'),
                                 reactive:true})
-    };
+        }
-    
+    });    
 
     Template.alertssummary.events({
         "click .reset": function(e,t){

meteor/app/client/globe.js

@@ -138,7 +138,7 @@ if (Meteor.isClient) {
       mesh.scale.set( 1.1, 1.1, 1.1 );
       scene.add(mesh);
 
-      geometry = new THREE.CubeGeometry(0.75, 0.75, 1);
+      geometry = new THREE.BoxGeometry(0.75, 0.75, 1);
       geometry.applyMatrix(new THREE.Matrix4().makeTranslation(0,0,-0.5));
 
       point = new THREE.Mesh(geometry);
@@ -227,7 +227,7 @@ if (Meteor.isClient) {
       var phi = (90 - lat) * Math.PI / 180;
       var theta = (180 - lng) * Math.PI / 180;
 
-      var geometry = new THREE.CubeGeometry(0.75, 0.75, 1);
+      var geometry = new THREE.BoxGeometry(0.75, 0.75, 1);
       geometry.applyMatrix(new THREE.Matrix4().makeTranslation(0,0,-0.5));
 
       var mypoint = new THREE.Mesh(geometry);

meteor/app/client/incidents.js

@@ -17,9 +17,11 @@ if (Meteor.isClient) {
     var timestamp = null;
     
 
-    Template.veristags.veris=function(){
+    Template.veristags.helpers({
+        veris: function() {
             return veris.find({tag:{$regex:'.*' +Session.get('verisfilter') + '.*',$options:'i'}},{limit:50});
-    };
+        }
+    });
 
     Template.veristags.events({
         'dragstart .tag': function(e){
@@ -36,11 +38,13 @@ if (Meteor.isClient) {
     
 
     //return all incidents
-    Template.incidents.incident = function () {
+    Template.incidents.helpers({
+        incident: function () {
             return incidents.find({},{
                                    sort: {dateOpened: -1}
                                 });
-    };
+        }
+    });
 
     //select an incident for editing
     Template.incidents.events({

meteor/app/client/investigations.js

@@ -19,11 +19,13 @@ if (Meteor.isClient) {
     var timestamp = null;
 
     //return all investigations
-    Template.investigations.investigation = function () {
+    Template.investigations.helpers({
+        investigation: function () {
             return investigations.find({},{
                                   sort: {dateOpened: -1}
                                 });
-    };
+        }
+    });
 
     //select an investigation for editing
     Template.investigations.events({

meteor/app/client/ipcif.js

@@ -47,15 +47,19 @@ if (Meteor.isClient) {
         }
     });
             
-    Template.ipcif.cif= function(){
+    Template.ipcif.helpers({
+        cif: function() {
             cifDep.depend();
             return cifresult;
-    };
+        }
+    });
     
-    Template.cifmodal.cif= function(){
+    Template.cifmodal.helpers({
+        cif: function() {
             cifDep.depend();
             return cifresult;
-    };
+        }
+    });
 
     Template.cifmodal.rendered = function () {
         //console.log(Session.get('ipcifipaddress'));

meteor/app/client/ipdshield.js

@@ -47,15 +47,19 @@ if (Meteor.isClient) {
         }
         });
             
-    Template.ipdshield.dshield= function(){
+    Template.ipdshield.helpers({
+        dshield: function() {
             dshieldDep.depend();
             return dshieldresult;
-    };
+        }
+    });
     
-    Template.dshieldmodal.dshield= function(){
+    Template.dshieldmodal.helpers({
+        dshield: function() {
             dshieldDep.depend();
             return dshieldresult;
-    };
+        }
+    });
 
     Template.dshieldmodal.rendered = function () {
         //console.log(Session.get('ipdshieldipaddress'));

meteor/app/client/ipintel.js

@@ -47,15 +47,19 @@ if (Meteor.isClient) {
         }
     });
             
-    Template.ipintel.intel= function(){
+    Template.ipintel.helpers({
+        intel: function(){
             intelDep.depend();
             return intelresult;
-    };
+        }
+    });
     
-    Template.intelmodal.intel= function(){
+    Template.intelmodal.helpers({
+        intel: function() {
             intelDep.depend();
             return intelresult;
-    };
+        }
+    });
 
     Template.intelmodal.rendered = function () {
         //console.log(Session.get('ipintelipaddress'));

meteor/app/client/ipwhois.js

@@ -47,15 +47,19 @@ if (Meteor.isClient) {
         }
         });
             
-    Template.ipwhois.whois= function(){
+    Template.ipwhois.helpers({
+        whois: function() {
             whoisDep.depend();
             return whoisresult;
-    };
+        }
+    });
     
-    Template.whoismodal.whois= function(){
+    Template.whoismodal.helpers({
+        whois: function() {
             whoisDep.depend();
             return whoisresult;
-    };
+        }
+    });
 
     Template.whoismodal.rendered = function () {
         //console.log(Session.get('ipwhoisipaddress'));

meteor/app/client/mozdef.js

@@ -163,11 +163,13 @@ if (Meteor.isClient) {
         return result
     }
 
-    Template.hello.greeting = function () {
+    Template.hello.helpers({
+        greeting: function() {
             if (typeof console !== 'undefined')
                 console.log("mozdef starting");
             return "MozDef: The Mozilla Defense Platform";
-    };
+        }
+    });
 
     Template.hello.events({
         'click' : function () {
@@ -177,10 +179,12 @@ if (Meteor.isClient) {
     });
 
     // loads kibana dashboards
-    Template.menu.kibanadashboards = function() {
+    Template.menu.helpers({
+        kibanadashboards: function() {
             Meteor.call('loadKibanaDashboards');
             return kibanadashboards.find();
-    };
+        }
+    });
 
     UI.registerHelper('uiDateFormat',function(adate){
         return dateFormat(adate);

meteor/app/client/mozdefhealth.js

@@ -13,27 +13,30 @@ if (Meteor.isClient) {
 
     //elastic search cluster template functions
     //return es health items
-    Template.mozdefhealth.esclusterhealthitems = function () {
+    Template.mozdefhealth.helpers({
-        return healthescluster.find();
-    };
         
-    Template.mozdefhealth.frontendhealthitems = function () {
+        esclusterhealthitems: function () {
+            return healthescluster.find();
+        },
+
+        frontendhealthitems: function () {
             return healthfrontend.find({},
                                    {fields:{},
                                     sort: {hostname: 1}
                                    });
-    };
+        },
 
-    Template.mozdefhealth.esnodeshealthitems = function () {
+        esnodeshealthitems: function () {
             return healthesnodes.find({},
                                   {fields:{},
                                   sort: {hostname: 1}
                                   });
-    };
+        },
 
-    Template.mozdefhealth.eshotthreadshealthitems = function () {
+        eshotthreadshealthitems: function () {
             return healtheshotthreads.find();
-    };
+        }
+    });
  
    Template.mozdefhealth.rendered = function () {
         var ringChartEPS   = dc.pieChart("#ringChart-EPS");

meteor/public/css/mozdef.css

@@ -140,6 +140,7 @@ caption, legend {
     background-color: rgba(245,245,245,.7)
 }
 
+
 /*bootstrap overrides*/
 
 .btn {

mq/esworker.py

@@ -19,6 +19,7 @@ import pytz
 import pynsive
 import re
 import sys
+import socket
 import time
 from configlib import getConfig, OptionParser
 from datetime import datetime, timedelta
@@ -154,6 +155,7 @@ def keyMapping(aDict):
 
     # set the timestamp when we received it, i.e. now
     returndict['receivedtimestamp'] = toUTC(datetime.now())
+    returndict['mozdefhostname'] = options.mozdefhostname
     try:
         for k, v in aDict.iteritems():
             k = removeAt(k).lower()
@@ -256,11 +258,10 @@ def esConnect(conn):
 
 class taskConsumer(ConsumerMixin):
 
-    def __init__(self, mqConnection, taskQueue, topicExchange, esConnection):
+    def __init__(self, mqConnection, taskQueue,  esConnection):
         self.connection = mqConnection
         self.esConnection = esConnection
         self.taskQueue = taskQueue
-        self.topicExchange = topicExchange
         self.mqproducer = self.connection.Producer(serializer='json')
         if hasUWSGI:
             self.muleid = uwsgi.mule_id()
@@ -378,10 +379,7 @@ class taskConsumer(ConsumerMixin):
                 except kombu.exceptions.MessageStateError:
                     # state may be already set.
                     return
-            # post the dict (kombu serializes it to json) to the events topic queue
+
-            # using the ensure function to shortcut connection/queue drops/stalls, etc.
-            # ensurePublish = self.connection.ensure(self.mqproducer, self.mqproducer.publish, max_retries=10)
-            # ensurePublish(normalizedDict, exchange=self.topicExchange, routing_key='mozdef.event')
             message.ack()
         except ValueError as e:
             sys.stderr.write("esworker exception in events queue %r\n" % e)
@@ -488,6 +486,9 @@ def dict2List(inObj):
             elif isinstance(v, list):
                 for l in dict2List(v):
                     yield l
+            elif isinstance(v,dict):
+                for d in dict2List(v):
+                    yield d
             else:
                 yield v
     else:
@@ -529,6 +530,10 @@ def sendEventToPlugins(anevent, metadata, pluginList):
 
 def main():
     # connect and declare the message queue/kombu objects.
+
+    # what sort of message queue are we talking to?
+    if options.mqprotocol in ('amqp', 'amqps'):
+
         # only py-amqp supports ssl and doesn't recognize amqps
         # so fix up the connection string accordingly
         connString = 'amqp://{0}:{1}@{2}:{3}/{4}'.format(options.mquser, options.mqpassword, options.mqserver, options.mqport, options.mqvhost)
@@ -553,43 +558,66 @@ def main():
         eventTaskQueue(mqConn).declare()
 
         # topic exchange for anyone who wants to queue and listen for mozdef.event
-    eventTopicExchange = Exchange(name=options.eventexchange, type='topic', durable=False, delivery_mode=1)
+        # commented out to begin deprecation for this feature
-    eventTopicExchange(mqConn).declare()
+        # eventTopicExchange = Exchange(name=options.eventexchange, type='topic', durable=False, delivery_mode=1)
+        # eventTopicExchange(mqConn).declare()
+
+    if options.mqprotocol in ('sqs'):
+        # amazon SQS
+        connString = 'sqs://%s:%s@' % (urllib.quote(options.accesskey, safe=''), urllib.quote(options.secretkey, safe=''))
+
+        mqConn = Connection(connString, transport_options=dict(region=options.region))
+        # for sqs, set taskexchange to the sqs queue name.
+        eventTaskQueue = mqConn.SimpleQueue(options.taskexchange)
+
 
     if hasUWSGI:
         sys.stdout.write("started as uwsgi mule {0}\n".format(uwsgi.mule_id()))
     else:
         sys.stdout.write('started without uwsgi\n')
-    # consume our queue and publish on the topic exchange
+    # consume our queue
-    taskConsumer(mqConn, eventTaskQueue, eventTopicExchange, es).run()
+    taskConsumer(mqConn, eventTaskQueue, es).run()
 
 
 def initConfig():
     # change this to your default zone for when it's not specified
     options.defaultTimeZone = getConfig('defaulttimezone', 'US/Pacific', options.configfile)
 
+    #capture the hostname
+    options.mozdefhostname = getConfig('mozdefhostname', socket.gethostname(), options.configfile)
+
     # elastic search options. set esbulksize to a non-zero value to enable bulk posting, set timeout to post no matter how many events after X seconds.
     options.esservers = list(getConfig('esservers', 'http://localhost:9200', options.configfile).split(','))
     options.esbulksize = getConfig('esbulksize', 0, options.configfile)
     options.esbulktimeout = getConfig('esbulktimeout', 30, options.configfile)
 
-    # message queue options
+    # set to either amqp or amqps for rabbitmq without/with ssl
+    # set to sqs for Amazon
+    options.mqprotocol = getConfig('mqprotocol', 'amqp', options.configfile)
+
+    # rabbit message queue options
     options.mqserver = getConfig('mqserver', 'localhost', options.configfile)
     options.taskexchange = getConfig('taskexchange', 'eventtask', options.configfile)
     options.eventexchange = getConfig('eventexchange', 'events', options.configfile)
-    # how many messages to ask for at once from the message queue
+    # rabbit: how many messages to ask for at once from the message queue
     options.prefetch = getConfig('prefetch', 50, options.configfile)
+    # rabbit: user creds
     options.mquser = getConfig('mquser', 'guest', options.configfile)
     options.mqpassword = getConfig('mqpassword', 'guest', options.configfile)
+    # rabbit: port/vhost
     options.mqport = getConfig('mqport', 5672, options.configfile)
     options.mqvhost = getConfig('mqvhost', '/', options.configfile)
-    # set to either amqp or amqps for ssl
+
-    options.mqprotocol = getConfig('mqprotocol', 'amqp', options.configfile)
+    # rabbit: run with message acking?
-    # run with message acking?
     # also toggles transient/persistant delivery (messages in memory only or stored on disk)
     # ack=True sets persistant delivery, False sets transient delivery
     options.mqack = getConfig('mqack', True, options.configfile)
 
+    # aws options
+    options.accesskey = getConfig('accesskey', '', options.configfile)
+    options.secretkey = getConfig('secretkey', '', options.configfile)
+    options.region = getConfig('region', 'us-west-1', options.configfile)
+
     # plugin options
     # secs to pass before checking for new/updated plugins
     # seems to cause memory leaks..