diff --git a/mq/plugins/broFixup.py b/mq/plugins/broFixup.py index 05ef8321..e5c7de20 100644 --- a/mq/plugins/broFixup.py +++ b/mq/plugins/broFixup.py @@ -6,8 +6,12 @@ # Contributors: # Jeff Bryner jbryner@mozilla.com # Brandon Myers bmyers@mozilla.com +# Michal Purzynski mpurzynski@mozilla.com import netaddr +from utilities.toUTC import toUTC +from datetime import datetime +from platform import node def isIPv4(ip): @@ -41,39 +45,493 @@ class message(object): self.registration = ['bro', 'nsm'] self.priority = 5 + try: + self.mozdefhostname = u'{0}'.format(node()) + except: + self.mozdefhostname = 'failed to fetch mozdefhostname' + pass + def onMessage(self, message, metadata): # set the doc type to bro # to avoid data type conflicts with other doc types # (int v string, etc) - metadata['doc_type']= 'bro' + # index holds documents of type 'type' + # index -> type -> doc + metadata['doc_type']= 'nsm' + whitelist = ['hostname', 'type', 'tags', 'customendpoint'] + + # move Bro specific fields under 'details' while preserving metadata + newmessage = dict() + + newmessage['details'] = message + newmessage['customendpoint'] = 'bro' + + + # add mandatory fields + if 'ts' in message.keys(): + newmessage[u'utctimestamp'] = toUTC(message['ts']).isoformat() + newmessage[u'timestamp'] = toUTC(message['ts']).isoformat() + else: + # a malformed message somehow managed to crawl to us, let's put it somewhat together + newmessage[u'utctimestamp'] = toUTC(datetime.now()).isoformat() + newmessage[u'timestamp'] = toUTC(datetime.now()).isoformat() + + newmessage[u'receivedtimestamp'] = toUTC(datetime.now()).isoformat() + + + # add mandatory fields + # move bro data under details + newmessage[u'eventsource'] = u'nsm' + newmessage[u'severity'] = u'INFO' + #newmessage[u'processname'] = u'' + #newmessage[u'processid'] = u'' + newmessage[u'mozdefhostname'] = self.mozdefhostname + newmessage[u'category'] = u'bro' + # re-arrange the position of some fields # {} vs {'details':{}} - if 'details' in message.keys(): - # details.tags -> tags - if 'tags' in message['details'].keys(): - message['tags'] = message['details']['tags'] - del message['details']['tags'] + if 'details' in newmessage.keys(): - # details.summary -> summary - if 'summary' in message['details'].keys(): - message['summary'] = message['details']['summary'] - del message['details']['summary'] + #newmessage[u'hostname'] = newmessage['details']['hostname'] + #del(newmessage['details']['hostname']) - # clean up the action notice IP addresses - if 'actions' in message['details'].keys(): - if message['details']['actions'] == "Notice::ACTION_LOG": - # retrieve indicator ip addresses from the sub field - # "sub": "Indicator: 1.2.3.4, Indicator: 5.6.7.8" - message['details']['indicators'] = [ip for ip - in findIPv4(message['details']['sub'])] + # some Bro logs need special treatment + if 'type' in newmessage['details']: + logtype = newmessage['details']['type'] + if logtype == 'conn': + newmessage[u'details'][u'originipbytes'] = newmessage['details']['orig_ip_bytes'] + newmessage[u'details'][u'responseipbytes'] = newmessage['details']['resp_ip_bytes'] + del(newmessage['details']['orig_ip_bytes']) + del(newmessage['details']['resp_ip_bytes']) + if 'history' not in newmessage['details']: + newmessage['details'][u'history'] = '' + newmessage[u'summary'] = ( + u'{sourceipaddress}:'+ + u'{sourceport} -> '+ + u'{destinationipaddress}:' + u'{destinationport} '+ + u'{history} '+ + u'{originipbytes} bytes / ' + u'{responseipbytes} bytes)' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'files': + if 'rx_hosts' in newmessage['details']: + newmessage['details'][u'sourceipaddress'] = u'{0}'.format(newmessage['details']['rx_hosts'][0]) + if 'tx_hosts' in newmessage['details']: + newmessage['details'][u'destinationipaddress'] = u'{0}'.format(newmessage['details']['tx_hosts'][0]) + if 'mime_type' not in newmessage['details']: + newmessage['details'][u'mime_type'] = u'unknown' + if 'filename' not in newmessage['details']: + newmessage['details'][u'filename'] = u'unknown' + if 'total_bytes' not in newmessage['details']: + newmessage['details'][u'total_bytes'] = u'0' + if 'md5' not in newmessage['details']: + newmessage['details'][u'md5'] = u'None' + if 'source' not in newmessage['details']: + newmessage['details'][u'source'] = u'None' + newmessage[u'summary'] = ( + u'{rx_hosts[0]} ' + u'downloaded (MD5) ' + u'{md5} ' + u'filename {filename} ' + u'MIME {mime_type} ' + u'({total_bytes} bytes) ' + u'from {tx_hosts[0]} ' + u'via {source}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'dns': + if 'qtype_name' not in newmessage['details']: + newmessage['details'][u'qtype_name'] = u'' + if 'query' not in newmessage['details']: + newmessage['details'][u'query'] = u'' + if 'rcode_name' not in newmessage['details']: + newmessage['details'][u'rcode_name'] = u'' + newmessage[u'summary'] = ( + u'{sourceipaddress} -> ' + u'{destinationipaddress}:{destinationport} ' + u'{qtype_name} ' + u'{query} ' + u'{rcode_name} ' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'http': + if 'method' not in newmessage['details']: + newmessage['details'][u'method'] = u'' + if 'host' not in newmessage['details']: + newmessage['details'][u'host'] = u'' + if 'uri' not in newmessage['details']: + newmessage['details'][u'uri'] = u'' + if 'status_code' not in newmessage['details']: + newmessage['details'][u'status_code'] = u'' + newmessage[u'summary'] = ( + u'{method} ' + u'{host} ' + u'{uri} ' + u'{status_code}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'ssl': + if 'server_name' not in newmessage['details']: + # fake it till you make it + newmessage['details'][u'server_name'] = newmessage['details']['destinationipaddress'] + newmessage[u'summary'] = ( + u'SSL: {sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'{server_name}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'dhcp': + newmessage[u'summary'] = ( + '{assigned_ip} assigned to ' + '{mac}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'ftp': + if 'command' not in newmessage['details']: + newmessage['details'][u'command'] = u'' + if 'user' not in newmessage['details']: + newmessage['details'][u'user'] = u'' + newmessage[u'summary'] = ( + u'FTP: {sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'{command} ' + u'{user}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'pe': + if 'os' not in newmessage['details']: + newmessage['details']['os'] = '' + if 'subsystem' not in newmessage['details']: + newmessage['details']['subsystem'] = '' + if 'sectionnames' not in newmessage['details']: + newmessage['details']['sectionnames'] = '' + newmessage[u'summary'] = ( + u'PE file: {os} ' + u'{subsystem} ' + u'{sectionnames}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'smtp': + if 'from' not in newmessage['details']: + newmessage['details'][u'from'] = u'' + if 'to' not in newmessage['details']: + newmessage['details'][u'to'] = [u''] + if 'msg_id' not in newmessage['details']: + newmessage['details'][u'msg_id'] = u'' + newmessage[u'summary'] = ( + u'SMTP: {sourceipaddress} -> ' + u'{destinationipaddress}: ' + u'{destinationport} ' + u'from {from} ' + u'to ' + u'{to[0]} ' + u'ID {msg_id}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'ssh': + if 'auth_success' not in newmessage['details']: + newmessage['details'][u'auth_success'] = u'unknown' + newmessage[u'summary'] = ( + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'status {auth_success}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'tunnel': + if 'tunnel_type' not in newmessage['details']: + newmessage['details'][u'tunnel_type'] = u'' + if 'action' not in newmessage['details']: + newmessage['details'][u'action'] = u'' + newmessage[u'summary'] = ( + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'{tunnel_type} ' + u'{action}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'intel': + newmessage[u'severity'] = u'WARNING' + if 'seenindicator' not in newmessage['details']: + newmessage['details'][u'seenindicator'] = u'' + newmessage[u'summary'] = ( + u'Bro intel match: ' + u'{seenindicator}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'knowncerts': + if 'serial' not in newmessage['details']: + newmessage['details'][u'serial'] = u'0' + newmessage[u'summary'] = ( + u'Certificate seen from: ' + u'{host}:' + u'{port_num} ' + u'serial {serial}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'knowndevices': + if 'mac' not in newmessage['details']: + newmessage['details'][u'mac'] = u'' + if 'dhcp_host_name' not in newmessage['details']: + newmessage['details'][u'dhcp_host_name'] = u'' + newmessage[u'summary'] = ( + u'New host: ' + u'{mac} ' + u'{dhcp_host_name}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'knownhosts': + if 'host' not in newmessage['details']: + newmessage['details'][u'host'] = u'' + newmessage[u'summary'] = ( + u'New host: ' + u'{host}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'knownservices': + if not newmessage['details']['service']: + newmessage['details'][u'service'] = [u'Unknown'] + if 'host' not in newmessage['details']: + newmessage['details'][u'host'] = u'unknown' + if 'port_num' not in newmessage['details']: + newmessage['details'][u'port_num'] = u'0' + if 'port_proto' not in newmessage['details']: + newmessage['details'][u'port_proto'] = u'' + newmessage[u'summary'] = ( + u'New service: ' + u'{service[0]} ' + u'on host ' + u'{host}:' + u'{port_num} / ' + u'{port_proto}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'notice': + newmessage[u'severity'] = u'NOTICE' + newmessage['details'][u'indicators'] = [] + if 'sub' not in newmessage['details']: + newmessage['details'][u'sub'] = u'' + if 'msg' not in newmessage['details']: + newmessage['details'][u'msg'] = u'' + if 'note' not in newmessage['details']: + newmessage['details'][u'note'] = u'' + newmessage[u'summary'] = ( + u'{note} ' + u'{msg} ' + u'{sub}' + ).format(**newmessage['details']) + # clean up the action notice IP addresses + if 'actions' in newmessage['details']: + if newmessage['details']['actions'] == "Notice::ACTION_LOG": + # retrieve indicator ip addresses from the sub field + # "sub": "Indicator: 1.2.3.4, Indicator: 5.6.7.8" + newmessage['details']['indicators'] = [ip for ip + in findIPv4(newmessage['details']['sub'])] # remove the details.src field and add it to indicators # as it may not be the actual source. - if 'src' in message['details'].keys(): - if isIPv4(message['details']['src']): - message['details']['indicators'].append(message['details']['src']) - del message['details']['src'] + if 'src' in newmessage['details']: + if isIPv4(newmessage['details']['src']): + newmessage['details']['indicators'].append(newmessage['details']['src']) + del newmessage['details']['src'] + return (newmessage, metadata) + + if logtype == 'rdp': + if 'cookie' not in newmessage['details']: + newmessage['details'][u'cookie'] = u'unknown' + newmessage[u'summary'] = ( + u'RDP: {sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'user {cookie}' + ).format(**newmessage['details']) + return (newmessage, metadata) - return (message, metadata) \ No newline at end of file + if logtype == 'sip': + if 'status_msg' not in newmessage['details']: + newmessage['details'][u'status_msg'] = u'unknown' + if 'uri' not in newmessage['details']: + newmessage['details'][u'uri'] = u'' + if 'method' not in newmessage['details']: + newmessage['details'][u'method'] = u'' + newmessage[u'summary'] = ( + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'method {method} ' + u'uri {uri} ' + u'status {status_msg}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'software': + if 'name' not in newmessage['details']: + newmessage['details'][u'name'] = u'' + if 'software_type' not in newmessage['details']: + newmessage['details'][u'software_type'] = u'' + if 'host' not in newmessage['details']: + newmessage['details'] = u'' + newmessage[u'summary'] = ( + u'Found {software_type} ' + u'{name} ' + u'on {host}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'socks': + if 'version' not in newmessage['details']: + newmessage['details'][u'version'] = u'0' + if 'status' not in newmessage['details']: + newmessage['details'][u'status'] = u'unknown' + if 'bound_p' not in newmessage['details']['bound_p']: + newmessage['details'][u'bound_p'] = u'0' + newmessage[u'summary'] = ( + u'SOCKSv{version}: ' + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'status {status}' + u'{bound_p}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'dcerpc': + if 'endpoint' not in newmessage['details']: + newmessage['details'][u'endpoint'] = u'unknown' + if 'operation' not in newmessage['details']: + newmessage['details'][u'operation'] = u'unknown' + newmessage[u'summary'] = ( + u'DCERPC: {sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'endpoint {endpoint} ' + u'operation {operation}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'kerberos': + if 'request_type' not in newmessage['details']: + newmessage['details'][u'request_type'] = u'unknown' + if 'client' not in newmessage['details']: + newmessage['details'][u'client'] = u'unknown' + if 'service' not in newmessage['details']: + newmessage['details'][u'service'] = u'unknown' + if 'success' not in newmessage['details']: + newmessage['details'][u'success'] = u'unknown' + if 'error_msg' not in newmessage['details']: + newmessage['details'][u'error_msg'] = u'' + newmessage[u'summary'] = ( + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'client {client} ' + u'requested {request_type} ' + u'service {service} ' + u'status {success} ' + u'{error_msg}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'ntlm': + if 'domainname' not in newmessage['details']: + newmessage['details'][u'domainname'] = u'' + if 'username' not in newmessage['details']: + newmessage['details'][u'username'] = u'' + if 'success' not in newmessage['details']: + newmessage['details'][u'success'] = u'unknown' + if 'status' not in newmessage['details']: + newmessage['details'][u'status'] = u'unknown' + newmessage[u'summary'] = ( + u'NTLM: {sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'user {username} ' + u'domain {domainname} ' + u'success {success} ' + u'status {status}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'smbfiles': + if 'path' not in newmessage['details']: + newmessage['details'][u'path'] = u'' + if 'name' not in newmessage['details']: + newmessage['details'][u'name'] = u'' + if 'action' not in newmessage['details']: + newmessage['details'][u'action'] = u'' + newmessage[u'summary'] = ( + 'SMB file operation: ' + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'{action} ' + u'{path} ' + u'{name}' + ).format(**newmessage['details']) + return(newmessage, metadata) + + if logtype == 'smbmapping': + if 'share_type' not in newmessage['details']: + newmessage['details'][u'share_type'] = u'' + if 'path' not in newmessage['details']: + newmessage['details'][u'path'] = u'' + newmessage[u'summary'] = ( + 'SMB mapping: ' + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'{share_type} ' + u'{path}' + ).format(**newmessage['details']) + return(newmessage, metadata) + + if logtype == 'snmp': + if 'version' not in newmessage['details']: + newmessage['details'][u'version'] = u'0' + newmessage['details']['getreqestssum'] = u'{0}'.format(newmessage['details']['get_bulk_requests'] + newmessage['details']['get_requests']) + getresp = newmessage['details'][u'get_responses'] + setreq = newmessage['details'][u'set_requests'] + newmessage[u'summary'] = ( + u'{version}: ' + u'{sourceipaddress} -> ' + u'{destinationipaddress}:' + u'{destinationport} ' + u'({getreqestssum} get / ' + u'{get_responses} put requests ' + u'{set_requests} responses)' + ).format(**newmessage['details']) + return (newmessage, metadata) + + if logtype == 'x509': + if 'certificateserial' not in newmessage['details']: + newmessage['details'][u'certificateserial'] = u'0' + newmessage[u'summary'] = ( + 'Certificate seen serial {certificateserial}' + ).format(**newmessage['details']) + return (newmessage, metadata) + + + return (newmessage, metadata)