diff --git a/alerts/bruteforce_ssh.py b/alerts/bruteforce_ssh.py index c890c0e8..7da103ea 100644 --- a/alerts/bruteforce_ssh.py +++ b/alerts/bruteforce_ssh.py @@ -18,7 +18,7 @@ class AlertBruteforceSsh(AlertTask): search_query.add_must([ PhraseMatch('summary', 'failed'), TermMatch('details.program', 'sshd'), - TermsMatch('summary', ['login', 'invalid', 'ldap_count_entries', 'publickey']) + TermsMatch('summary', ['login', 'invalid', 'ldap_count_entries', 'publickey', 'keyboard']) ]) for ip_address in self.config.skiphosts.split(): diff --git a/alerts/cloudtrail_logging_disabled.py b/alerts/cloudtrail_logging_disabled.py index 50248cbf..fb589270 100644 --- a/alerts/cloudtrail_logging_disabled.py +++ b/alerts/cloudtrail_logging_disabled.py @@ -15,10 +15,10 @@ class AlertCloudtrailLoggingDisabled(AlertTask): search_query.add_must([ TermMatch('source', 'cloudtrail'), - TermMatch('eventName', 'StopLogging') + TermMatch('eventname', 'StopLogging') ]) - search_query.add_must_not(TermMatch('errorCode', 'AccessDenied')) + search_query.add_must_not(TermMatch('errorcode', 'AccessDenied')) self.filtersManual(search_query) self.searchEventsSimple() @@ -29,6 +29,6 @@ class AlertCloudtrailLoggingDisabled(AlertTask): tags = ['cloudtrail', 'aws', 'cloudtrailpagerduty'] severity = 'CRITICAL' - summary = 'Cloudtrail Logging Disabled: ' + event['_source']['requestParameters']['name'] + summary = 'Cloudtrail Logging Disabled: ' + event['_source']['requestparameters']['name'] return self.createAlertDict(summary, category, tags, [event], severity) diff --git a/alerts/duo_authfail.py b/alerts/duo_authfail.py index 8bd18686..62a6cca6 100644 --- a/alerts/duo_authfail.py +++ b/alerts/duo_authfail.py @@ -13,7 +13,7 @@ class AlertDuoAuthFail(AlertTask): search_query = SearchQuery(minutes=15) search_query.add_must([ - TermMatch('category', 'event'), + TermMatch('category', 'authentication'), ExistsMatch('details.sourceipaddress'), ExistsMatch('details.username'), PhraseMatch('details.result', 'FRAUD') diff --git a/alerts/trace_audit.py b/alerts/trace_audit.py index 4e56d5a3..e382fff9 100644 --- a/alerts/trace_audit.py +++ b/alerts/trace_audit.py @@ -32,12 +32,12 @@ class TraceAudit(AlertTask): severity = 'WARNING' tags = ['audit'] - summary = ('{0} instances of Strace or Ptrace executed on a system by {1}'.format(aggreg['count'], aggreg['value'], )) - hostnames = self.mostCommon(aggreg['allevents'], '_source.hostname') - # did they modify more than one host? - # or just modify an existing configuration more than once? - if len(hostnames) > 1: - for i in hostnames[:5]: - summary += ' on {0} ({1} hosts)'.format(i[0], i[1]) + hosts = set([event['_source']['hostname'] for event in aggreg['events']]) + + summary = '{0} instances of Strace or Ptrace executed by {1} on {2}'.format( + aggreg['count'], + aggreg['value'], + ','.join(hosts) + ) return self.createAlertDict(summary, category, tags, aggreg['events'], severity) diff --git a/alerts/write_audit.conf b/alerts/write_audit.conf index a7291f45..a466ae35 100644 --- a/alerts/write_audit.conf +++ b/alerts/write_audit.conf @@ -1,2 +1,3 @@ [options] -skipprocess = process1 process2 \ No newline at end of file +skipprocess = process1 process2 +expectedusers = user1 user2 \ No newline at end of file diff --git a/alerts/write_audit.py b/alerts/write_audit.py index adaa96a4..b98a66c3 100644 --- a/alerts/write_audit.py +++ b/alerts/write_audit.py @@ -13,7 +13,7 @@ from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch class WriteAudit(AlertTask): def main(self): - self.parse_config('write_audit.conf', ['skipprocess']) + self.parse_config('write_audit.conf', ['skipprocess', 'expectedusers']) search_query = SearchQuery(minutes=15) search_query.add_must([ @@ -33,7 +33,22 @@ class WriteAudit(AlertTask): severity = 'WARNING' tags = ['audit'] - summary = ('{0} Filesystem write(s) to an auditd path by {1}'.format(aggreg['count'], aggreg['value'], )) + users = set() + paths = set() + for event in aggreg['events']: + users.add(event['_source']['details']['user']) + paths.add(event['_source']['summary'].split(' ')[1]) + + summary = '{0} Filesystem write(s) to an auditd path ({1}) by {2} ({3})'.format( + aggreg['count'], + ', '.join(paths), + ', '.join(users), + aggreg['value'] + ) + + if aggreg['value'] in self.config.expectedusers.split(' '): + severity = 'NOTICE' + hostnames = self.mostCommon(aggreg['allevents'], '_source.hostname') # did they modify more than one host? # or just modify an existing configuration more than once? diff --git a/cron/auth02mozdef.py b/cron/auth02mozdef.py index 825a4aae..7191e2f1 100644 --- a/cron/auth02mozdef.py +++ b/cron/auth02mozdef.py @@ -263,6 +263,9 @@ def process_msg(mozmsg, msg): See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs """ details = DotDict({}) + # defaults + details.username = "UNKNOWN" + details.userid = "UNKNNOWN" # key words used to set category and success/failure markers authentication_words = ['Login', 'Logout', 'Auth'] @@ -298,7 +301,7 @@ def process_msg(mozmsg, msg): pass try: - mozmsg.useragent = msg.user_agent + details['useragent'] = msg.user_agent except KeyError: pass diff --git a/cron/defaultMappingTemplate.json b/cron/defaultMappingTemplate.json index d3db29e7..51cd8526 100644 --- a/cron/defaultMappingTemplate.json +++ b/cron/defaultMappingTemplate.json @@ -120,7 +120,7 @@ }, "requestparameters" : { "properties" : { - "logStreamName": { + "logstreamname": { "properties": { "raw_value": { "type" : "keyword" diff --git a/docker/compose/kibana/Dockerfile b/docker/compose/kibana/Dockerfile index 58101827..12310cae 100644 --- a/docker/compose/kibana/Dockerfile +++ b/docker/compose/kibana/Dockerfile @@ -2,6 +2,8 @@ FROM centos:7 LABEL maintainer="mozdef@mozilla.com" +# When changing kibana version remember to edit +# docker/compose/mozdef_bootstrap/files/initial_setup.py accordingly ENV KIBANA_VERSION 5.6.7 RUN \ diff --git a/docker/compose/mozdef_bootstrap/Dockerfile b/docker/compose/mozdef_bootstrap/Dockerfile index f25efadc..bd57c587 100644 --- a/docker/compose/mozdef_bootstrap/Dockerfile +++ b/docker/compose/mozdef_bootstrap/Dockerfile @@ -7,6 +7,7 @@ RUN mkdir -p /opt/mozdef/envs/mozdef/docker/conf COPY cron/defaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/defaultMappingTemplate.json COPY docker/compose/mozdef_cron/files/backup.conf /opt/mozdef/envs/mozdef/cron/backup.conf COPY docker/compose/mozdef_bootstrap/files/initial_setup.py /opt/mozdef/envs/mozdef/initial_setup.py +COPY docker/compose/mozdef_bootstrap/files/index_mappings /opt/mozdef/envs/mozdef/index_mappings RUN chown -R mozdef:mozdef /opt/mozdef/envs/mozdef/ diff --git a/docker/compose/mozdef_bootstrap/files/index_mappings/alerts.json b/docker/compose/mozdef_bootstrap/files/index_mappings/alerts.json new file mode 100644 index 00000000..d44a4c0c --- /dev/null +++ b/docker/compose/mozdef_bootstrap/files/index_mappings/alerts.json @@ -0,0 +1,6 @@ +{ + "title": "alerts", + "timeFieldName": "utctimestamp", + "notExpandable": true, + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"category\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"notify_mozdefbot\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"severity\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"summary\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"utctimestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]" +} \ No newline at end of file diff --git a/docker/compose/mozdef_bootstrap/files/index_mappings/events-weekly.json b/docker/compose/mozdef_bootstrap/files/index_mappings/events-weekly.json new file mode 100644 index 00000000..96e339ea --- /dev/null +++ b/docker/compose/mozdef_bootstrap/files/index_mappings/events-weekly.json @@ -0,0 +1,6 @@ +{ + "title": "events-weekly", + "timeFieldName": "utctimestamp", + "notExpandable": true, + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"category\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.apiversion.raw_value\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.destinationipaddress\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.destinationport\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.requestparameters.logstreamname.raw_value\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.sourceipaddress\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.sourceipv4address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.sourceport\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.srcip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.success\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mozdefhostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"processid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"processname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"receivedtimestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"severity\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"summary\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"utctimestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]" +} diff --git a/docker/compose/mozdef_bootstrap/files/index_mappings/events.json b/docker/compose/mozdef_bootstrap/files/index_mappings/events.json new file mode 100644 index 00000000..5fc0513c --- /dev/null +++ b/docker/compose/mozdef_bootstrap/files/index_mappings/events.json @@ -0,0 +1,6 @@ +{ + "title": "events", + "timeFieldName": "utctimestamp", + "notExpandable": true, + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"category\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.apiversion.raw_value\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.destinationipaddress\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.destinationport\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.requestparameters.logstreamname.raw_value\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.sourceipaddress\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.sourceipv4address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.sourceport\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.srcip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"details.success\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mozdefhostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"processid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"processname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"receivedtimestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"severity\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"summary\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"utctimestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]" +} diff --git a/docker/compose/mozdef_bootstrap/files/initial_setup.py b/docker/compose/mozdef_bootstrap/files/initial_setup.py index 87a75d46..4528069d 100644 --- a/docker/compose/mozdef_bootstrap/files/initial_setup.py +++ b/docker/compose/mozdef_bootstrap/files/initial_setup.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta from time import sleep from configlib import getConfig import json +import time from elasticsearch.exceptions import ConnectionError @@ -38,6 +39,8 @@ event_index_name = current_date.strftime("events-%Y%m%d") previous_event_index_name = (current_date - timedelta(days=1)).strftime("events-%Y%m%d") weekly_index_alias = 'events-weekly' alert_index_name = current_date.strftime("alerts-%Y%m") +kibana_index_name = '.kibana' +kibana_version = '5.6.7' index_settings_str = '' with open(args.default_mapping_file) as data_file: @@ -77,6 +80,7 @@ index_settings['settings'] = { } } +# Create initial indices if event_index_name not in all_indices: print "Creating " + event_index_name client.create_index(event_index_name, index_config=index_settings) @@ -95,3 +99,25 @@ client.create_alias('alerts', alert_index_name) if weekly_index_alias not in all_indices: print "Creating " + weekly_index_alias client.create_alias_multiple_indices(weekly_index_alias, [event_index_name, previous_event_index_name]) + +if kibana_index_name not in all_indices: + print "Creating " + kibana_index_name + client.create_index(kibana_index_name) + # Create index patterns and assign default index mapping + time.sleep(1) + index_mappings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'index_mappings') + listing = os.listdir(index_mappings_path) + for infile in listing: + json_file_path = os.path.join(index_mappings_path, infile) + with open(json_file_path) as json_data: + mapping_data = json.load(json_data) + print "Creating {0} index mapping".format(mapping_data['title']) + client.save_object(body=mapping_data, index='.kibana', doc_type='index-pattern', doc_id=mapping_data['title']) + + # Assign default index to 'events' + client.flush('.kibana') + default_mapping_data = { + "defaultIndex": 'events' + } + print "Assigning events as default index mapping" + client.save_object(default_mapping_data, '.kibana', 'config', kibana_version) diff --git a/docs/source/alert_development_guide.rst b/docs/source/alert_development_guide.rst index 432ed076..ba28deaf 100644 --- a/docs/source/alert_development_guide.rst +++ b/docs/source/alert_development_guide.rst @@ -118,6 +118,8 @@ Change the TermMatch line to and you will get alerts for syslog labeled messages. Ideally you should edit your test to match, but it's not strictly necessary. +Scheduling your alert +--------------------- Next we will need to enable the log and to schedule it. At time of writing this is a bit annoying. Open the file :: diff --git a/meteor/public/css/dropdowns.css b/meteor/public/css/dropdowns.css index 34f26027..3b272e79 100644 --- a/meteor/public/css/dropdowns.css +++ b/meteor/public/css/dropdowns.css @@ -6,7 +6,7 @@ ul.dropdown { position: relative; display: inline-block; - width: min-content; + width: max-content; border: 1px solid rgba(0, 0, 0, 0.2); border-radius: 3px; box-shadow: 0px 5px 10px rgba(0, 0, 0, 0.2); @@ -93,4 +93,4 @@ ul.dropdown ul ul { } ul.dropdown li:hover > ul { visibility: visible; -} \ No newline at end of file +} diff --git a/mq/esworker_papertrail.py b/mq/esworker_papertrail.py index 4f622ec3..1e07f69c 100755 --- a/mq/esworker_papertrail.py +++ b/mq/esworker_papertrail.py @@ -255,6 +255,8 @@ class taskConsumer(object): event['summary'] = event['details']['message'] if 'severity' in event['details']: event['severity'] = event['details']['severity'] + if 'source_ip' in event['details']: + event['sourceipaddress'] = event['details']['source_ip'] else: event['severity'] = 'INFO' event['category'] = 'syslog' diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py index 88619fa6..564e0054 100644 --- a/mq/plugins/cloudtrail.py +++ b/mq/plugins/cloudtrail.py @@ -21,28 +21,28 @@ class message(object): self.modify_keys = [ 'details.additionaleventdata', 'details.apiversion', - 'details.requestparameters.attribute', - 'details.requestparameters.bucketPolicy.Statement.Principal', - 'details.requestparameters.callerReference', - 'details.requestparameters.description', - 'details.requestparameters.disableApiTermination', - 'details.requestparameters.domainName', - 'details.requestparameters.domainNames', - 'details.requestparameters.ebsOptimized', - 'details.requestparameters.filter', - 'details.requestparameters.iamInstanceProfile', - 'details.requestparameters.instanceType', - 'details.requestparameters.logStreamName', - 'details.requestparameters.rule', - 'details.requestparameters.source', - 'details.responseelements.distribution.distributionConfig.callerReference', - 'details.responseelements.endpoint', - 'details.responseelements.findings.service.additionalInfo.unusual', - 'details.responseelements.lastModified', - 'details.responseelements.role', - 'details.responseelements.securityGroups', - 'details.responseelements.subnets', 'details.serviceeventdetails', + 'details.requestparameters.attribute', + 'details.requestparameters.bucketpolicy.statement.principal', + 'details.requestparameters.callerreference', + 'details.requestparameters.description', + 'details.requestparameters.disableapitermination', + 'details.requestparameters.domainname', + 'details.requestparameters.domainnames', + 'details.requestparameters.ebsoptimized', + 'details.requestparameters.filter', + 'details.requestparameters.iaminstanceprofile', + 'details.requestparameters.instancetype', + 'details.requestparameters.logstreamname', + 'details.requestparameters.source', + 'details.responseelements.role', + 'details.requestparameters.rule', + 'details.responseelements.subnets', + 'details.responseelements.endpoint', + 'details.responseelements.securitygroups', + 'details.responseelements.lastmodified', + 'details.responseelements.findings.service.additionalinfo.unusual', + 'details.responseelements.distribution.distributionconfig.callerreference' ] def convert_key_raw_str(self, needle, haystack): diff --git a/mq/plugins/filterlog.py b/mq/plugins/filterlog.py index d9ed256d..e1c8b3a8 100644 --- a/mq/plugins/filterlog.py +++ b/mq/plugins/filterlog.py @@ -71,9 +71,9 @@ class message(object): if 'tcp' not in message['details']: message['details']['tcp'] = {} - message['details']['tcp']['source_port'] = summary_items[last_index + 4] - message['details']['tcp']['destination_port'] = summary_items[last_index + 5] - message['details']['tcp']['data_length'] = summary_items[last_index + 6] + message['details']['source_port'] = summary_items[last_index + 4] + message['details']['destination_port'] = summary_items[last_index + 5] + message['details']['data_length'] = summary_items[last_index + 6] message['details']['tcp']['flags'] = summary_items[last_index + 7] message['details']['tcp']['seq_number'] = summary_items[last_index + 8] message['details']['tcp']['ack_number'] = summary_items[last_index + 9] @@ -81,11 +81,8 @@ class message(object): message['details']['tcp']['urg'] = summary_items[last_index + 11] message['details']['tcp']['options'] = summary_items[last_index + 12] elif proto_id == 17: - if 'udp' not in message['details']: - message['details']['udp'] = {} - - message['details']['udp']['source_port'] = summary_items[last_index + 4] - message['details']['udp']['destination_port'] = summary_items[last_index + 5] - message['details']['udp']['data_length'] = summary_items[last_index + 6] + message['details']['source_port'] = summary_items[last_index + 4] + message['details']['destination_port'] = summary_items[last_index + 5] + message['details']['data_length'] = summary_items[last_index + 6] return (message, metadata) diff --git a/mq/plugins/fxaFixup.py b/mq/plugins/fxaFixup.py index 7bf689e3..55f392a5 100644 --- a/mq/plugins/fxaFixup.py +++ b/mq/plugins/fxaFixup.py @@ -81,23 +81,23 @@ class message(object): if message['category'] == 'logfile': message['category'] = 'weblog' - if 'remoteAddressChain' in message['details'].keys(): - if isinstance(message['details']['remoteAddressChain'], list): - sourceIP = message['details']['remoteAddressChain'][0] + if 'remoteaddresschain' in message['details'].keys(): + if isinstance(message['details']['remoteaddresschain'], list): + sourceIP = message['details']['remoteaddresschain'][0] if isIP(sourceIP): message['details']['sourceipaddress'] = sourceIP # handle the case of an escaped list: - # "remoteAddressChain": "[\"1.2.3.4\",\"5.6.7.8\",\"127.0.0.1\"]" - if (isinstance(message['details']['remoteAddressChain'], unicode) and - message['details']['remoteAddressChain'][0] == '[' and - message['details']['remoteAddressChain'][-1] == ']'): + # "remoteaddresschain": "[\"1.2.3.4\",\"5.6.7.8\",\"127.0.0.1\"]" + if (isinstance(message['details']['remoteaddresschain'], unicode) and + message['details']['remoteaddresschain'][0] == '[' and + message['details']['remoteaddresschain'][-1] == ']'): # remove the brackets and double quotes for i in ['[', ']', '"']: - message['details']['remoteAddressChain'] = message['details']['remoteAddressChain'].replace(i, '') + message['details']['remoteaddresschain'] = message['details']['remoteaddresschain'].replace(i, '') # make sure it's still a list - if ',' in message['details']['remoteAddressChain']: - sourceIP = message['details']['remoteAddressChain'].split(',')[0] + if ',' in message['details']['remoteaddresschain']: + sourceIP = message['details']['remoteaddresschain'].split(',')[0] if isIP(sourceIP): message['details']['sourceipaddress'] = sourceIP diff --git a/mq/plugins/guardDuty.py b/mq/plugins/guardDuty.py index dc6a263e..3ea39dfd 100644 --- a/mq/plugins/guardDuty.py +++ b/mq/plugins/guardDuty.py @@ -19,19 +19,19 @@ class message(object): # AWS guard duty sends dates as iso_8601 which ES doesn't appreciate # here's a list of date fields we'll convert to isoformat self.date_keys = [ - 'details.finding.eventLastSeen', - 'details.finding.eventFirstSeen', - 'details.resource.instanceDetails.launchTime', - 'details.createdAt', - 'details.updatedAt' + 'details.finding.eventlastseen', + 'details.finding.eventfirstseen', + 'details.resource.instancedetails.launchtime', + 'details.createdat', + 'details.updatedat' ] # AWS guard duty can send IPs in a bunch of places # Lets pick out some likely targets and format them # so other mozdef plugins can rely on their location self.ipaddress_keys = [ - 'details.finding.action.networkConnectionAction.remoteIpDetails.ipAddressV4', - 'details.finding.action.awsApiCallAction.remoteIpDetails.ipAdrressV4' + 'details.finding.action.networkconnectionaction.remoteipdetails.ipaddressv4', + 'details.finding.action.awsapicallaction.remoteipdetails.ipadrressv4' ] def convert_key_date_format(self, needle, haystack): @@ -73,14 +73,14 @@ class message(object): # if we still haven't found what we are looking for #U2 # sometimes it's in a list if 'sourceipaddress' not in message['details'].keys(): - if key_exists('details.finding.action.portProbeAction.portProbeDetails', message) \ - and isinstance(message.details.finding.action.portProbeAction.portProbeDetails, list): + if key_exists('details.finding.action.portprobeaction.portprobedetails', message) \ + and isinstance(message.details.finding.action.portprobeaction.portprobedetails, list): # inspect the first list entry and see if it contains an IP - portProbeDetails = DotDict( - message.details.finding.action.portProbeAction.portProbeDetails[0]) - if key_exists('remoteIpDetails.ipAddressV4', portProbeDetails): - message.details.sourceipaddress = portProbeDetails.remoteIpDetails.ipAddressV4 + portprobedetails = DotDict( + message.details.finding.action.portprobeaction.portprobedetails[0]) + if key_exists('remoteipdetails.ipaddressv4', portprobedetails): + message.details.sourceipaddress = portprobedetails.remoteipdetails.ipaddressv4 # recovert the message back to a plain dict return (dict(message), metadata) diff --git a/mq/plugins/lower_keys.py b/mq/plugins/lower_keys.py new file mode 100644 index 00000000..75f08664 --- /dev/null +++ b/mq/plugins/lower_keys.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2014 Mozilla Corporation + + +class message(object): + + def __init__(self): + ''' + takes an incoming message + and sets the keys to lowercase + ''' + + self.registration = ['cloudtrail', 'fxa-customsmozsvc', 'vidyo', 'suricata', 'guardduty'] + self.priority = 4 + + def onMessage(self, message, metadata): + def renameKeysToLower(message): + if isinstance(message, dict): + for key in message.keys(): + message[key.lower()] = message.pop(key) + if isinstance(message[key.lower()], dict) or isinstance(message[key.lower()], list): + message[key.lower()] = renameKeysToLower(message[key.lower()]) + elif isinstance(message, list): + for item in message: + item = renameKeysToLower(item) + return message + + message = renameKeysToLower(message) + return (message, metadata) diff --git a/mq/plugins/suricataFixup.py b/mq/plugins/suricataFixup.py index e792e90c..6b7bb2b2 100644 --- a/mq/plugins/suricataFixup.py +++ b/mq/plugins/suricataFixup.py @@ -46,21 +46,21 @@ class message(object): newmessage = dict() try: - newmessage['details'] = json.loads(message['MESSAGE']) + newmessage['details'] = json.loads(message['message']) except: newmessage['details'] = {} newmessage['rawdetails'] = message # move some fields that are expected at the event 'root' where they belong - if 'HOST_FROM' in message: - newmessage['hostname'] = message['HOST_FROM'] + if 'host_from' in message: + newmessage['hostname'] = message['host_from'] if 'tags' in message: newmessage['tags'] = message['tags'] if 'category' in message: newmessage['category'] = message['category'] newmessage[u'source'] = u'unknown' - if 'SOURCE' in message: - newmessage[u'source'] = message['SOURCE'] + if 'source' in message: + newmessage[u'source'] = message['source'] logtype = newmessage['source'] newmessage[u'event_type'] = u'unknown' if 'event_type' in message: @@ -103,12 +103,12 @@ class message(object): newmessage[u'details'][u'destinationport'] = newmessage['details']['dest_port'] del(newmessage['details']['dest_port']) - if 'FILE_NAME' in newmessage['details']: - del(newmessage['details']['FILE_NAME']) - if 'MESSAGE' in newmessage['details']: - del(newmessage['details']['MESSAGE']) - if 'SOURCE' in newmessage['details']: - del(newmessage['details']['SOURCE']) + if 'file_name' in newmessage['details']: + del(newmessage['details']['file_name']) + if 'message' in newmessage['details']: + del(newmessage['details']['message']) + if 'source' in newmessage['details']: + del(newmessage['details']['source']) if logtype == 'eve-log': if eventtype == 'alert': diff --git a/mq/plugins/vidyoCallID.py b/mq/plugins/vidyoCallID.py index 23d48937..e6d28c72 100644 --- a/mq/plugins/vidyoCallID.py +++ b/mq/plugins/vidyoCallID.py @@ -21,7 +21,7 @@ class message(object): self.priority = 5 def onMessage(self, message, metadata): - docid = hashlib.md5('vidyoUniqueCallID' + message['details']['UniqueCallID']).hexdigest() + docid = hashlib.md5('vidyouniquecallid' + message['details']['uniquecallid']).hexdigest() metadata['id'] = docid metadata['doc_type'] = 'vidyo' return (message, metadata) diff --git a/requirements.txt b/requirements.txt index 42814fbc..b3a56882 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ bottle==0.12.4 bugzilla==1.0.0 celery==4.1.0 cffi==1.9.1 -configlib==2.0.2 +configlib==2.0.3 configparser==3.5.0b2 cryptography==2.3.1 dnspython==1.15.0 diff --git a/tests/alerts/test_cloudtrail_logging_disabled.py b/tests/alerts/test_cloudtrail_logging_disabled.py index 5c1bc3c1..0f2f7094 100644 --- a/tests/alerts/test_cloudtrail_logging_disabled.py +++ b/tests/alerts/test_cloudtrail_logging_disabled.py @@ -12,10 +12,10 @@ class TestAlertCloudtrailLoggingDisabled(AlertTestSuite): default_event = { "_type": "cloudtrail", "_source": { - "eventName": "StopLogging", + "eventname": "StopLogging", "source": "cloudtrail", - "requestParameters": { - "name": "cloudtrail_example_name" + "requestparameters": { + "name": "cloudtrail_example_name", } } } @@ -60,7 +60,7 @@ class TestAlertCloudtrailLoggingDisabled(AlertTestSuite): ) event = AlertTestSuite.create_event(default_event) - event['_source']['eventName'] = 'Badeventname' + event['_source']['eventname'] = 'Badeventname' test_cases.append( NegativeAlertTestCase( description="Negative test case with bad eventName", @@ -79,7 +79,7 @@ class TestAlertCloudtrailLoggingDisabled(AlertTestSuite): ) event = AlertTestSuite.create_event(default_event) - event['_source']['errorCode'] = 'AccessDenied' + event['_source']['errorcode'] = 'AccessDenied' test_cases.append( NegativeAlertTestCase( description="Negative test case with excluding errorCode", diff --git a/tests/alerts/test_duo_authfail.py b/tests/alerts/test_duo_authfail.py index ebdee1a1..b19426f9 100644 --- a/tests/alerts/test_duo_authfail.py +++ b/tests/alerts/test_duo_authfail.py @@ -17,7 +17,7 @@ class TestAlertDuoAuthFail(AlertTestSuite): default_event = { "_type": "event", "_source": { - "category": "event", + "category": "authentication", "summary": 'authentication FRAUD for you@somewhere.com', "details": { "sourceipaddress": "1.2.3.4", diff --git a/tests/alerts/test_trace_audit.py b/tests/alerts/test_trace_audit.py index 0a5453fc..02367f4c 100644 --- a/tests/alerts/test_trace_audit.py +++ b/tests/alerts/test_trace_audit.py @@ -33,9 +33,8 @@ class TestTraceAudit(AlertTestSuite): default_alert = { "category": "trace", "severity": "WARNING", - "summary": "5 instances of Strace or Ptrace executed on a system by randomjoe", + "summary": "5 instances of Strace or Ptrace executed by randomjoe on exhostname", "tags": ['audit'], - "notify_mozdefbot": True, } test_cases = [] diff --git a/tests/alerts/test_write_audit.py b/tests/alerts/test_write_audit.py index ac84aa60..a286d531 100644 --- a/tests/alerts/test_write_audit.py +++ b/tests/alerts/test_write_audit.py @@ -19,12 +19,17 @@ class TestWriteAudit(AlertTestSuite): "_type": "auditd", "_source": { "category": "write", - "summary": "Write: /etc/audit/", + "summary": "Write: /etc/audit/plugins.d/temp-file.conf", "hostname": "exhostname", - "tags": ["audisp-json","2.1.0", "audit"], + "tags": [ + "audisp-json", + "2.1.0", + "audit" + ], "details": { "processname": "vi", "originaluser": "randomjoe", + "user": "root", "auditkey": "audit", } } @@ -34,7 +39,7 @@ class TestWriteAudit(AlertTestSuite): default_alert = { "category": "write", "severity": "WARNING", - "summary": "5 Filesystem write(s) to an auditd path by randomjoe", + "summary": "5 Filesystem write(s) to an auditd path (/etc/audit/plugins.d/temp-file.conf) by root (randomjoe)", "tags": ['audit'], } @@ -50,23 +55,15 @@ class TestWriteAudit(AlertTestSuite): events = AlertTestSuite.create_events(default_event, 5) for event in events: - event['_source']['summary'] = 'Write: /etc/audit/rules.d/.audit.rules.swp' + event['_source']['details']['originaluser'] = 'user1' + expected_alert = AlertTestSuite.create_alert(default_alert) + expected_alert['severity'] = 'NOTICE' + expected_alert['summary'] = "5 Filesystem write(s) to an auditd path (/etc/audit/plugins.d/temp-file.conf) by root (user1)" test_cases.append( PositiveAlertTestCase( - description="Positive test with events with a summary of 'Write: /etc/audit/rules.d/.audit.rules.swp'", + description="Positive test with expected downgraded severity", events=events, - expected_alert=default_alert - ) - ) - - events = AlertTestSuite.create_events(default_event, 5) - for event in events: - event['_source']['summary'] = 'Write: /etc/audit/rules.d/' - test_cases.append( - PositiveAlertTestCase( - description="Positive test with events with a summary of 'Write: /etc/audit/rules.d/'", - events=events, - expected_alert=default_alert + expected_alert=expected_alert ) ) diff --git a/tests/mq/plugins/test_broFixup.py b/tests/mq/plugins/test_broFixup.py index 6b976069..a803174c 100644 --- a/tests/mq/plugins/test_broFixup.py +++ b/tests/mq/plugins/test_broFixup.py @@ -134,7 +134,6 @@ class TestBroFixup(object): "eventsource": "systemslogs", "details": { "processid": "21233", - "Random": 2, "sourceipv4address": "10.22.74.208", "hostname": "hostname1.subdomain.domain.com", "program": "sshd", diff --git a/tests/mq/plugins/test_cloudtrail.py b/tests/mq/plugins/test_cloudtrail.py index dea33b0f..deb15560 100644 --- a/tests/mq/plugins/test_cloudtrail.py +++ b/tests/mq/plugins/test_cloudtrail.py @@ -43,7 +43,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'requestparameters': { - 'iamInstanceProfile': 'astringvalue', + 'iaminstanceprofile': 'astringvalue', } } } @@ -53,7 +53,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'requestparameters': { - 'iamInstanceProfile': { + 'iaminstanceprofile': { 'raw_value': 'astringvalue', } } @@ -295,7 +295,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'requestparameters': { - 'ebsOptimized': 'astringvalue', + 'ebsoptimized': 'astringvalue', } } } @@ -305,7 +305,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'requestparameters': { - 'ebsOptimized': { + 'ebsoptimized': { 'raw_value': 'astringvalue', } } @@ -319,7 +319,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'responseelements': { - 'securityGroups': 'astringvalue', + 'securitygroups': 'astringvalue', } } } @@ -329,7 +329,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'responseelements': { - 'securityGroups': { + 'securitygroups': { 'raw_value': 'astringvalue', } } @@ -343,7 +343,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'requestparameters': { - 'disableApiTermination': 'astringvalue' + 'disableapitermination': 'astringvalue' } } } @@ -353,7 +353,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'requestparameters': { - 'disableApiTermination': { + 'disableapitermination': { 'raw_value': 'astringvalue' } } @@ -367,7 +367,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'responseelements': { - 'lastModified': 'astringvalue' + 'lastmodified': 'astringvalue' } } } @@ -377,7 +377,7 @@ class TestCloudtrailPlugin(): 'source': 'cloudtrail', 'details': { 'responseelements': { - 'lastModified': { + 'lastmodified': { 'raw_value': 'astringvalue' } } @@ -393,7 +393,7 @@ class TestCloudtrailPlugin(): 'responseelements': { 'findings': { 'service': { - 'additionalInfo': { + 'additionalinfo': { 'unusual': 'astringvalue' } } @@ -409,7 +409,7 @@ class TestCloudtrailPlugin(): 'responseelements': { 'findings': { 'service': { - 'additionalInfo': { + 'additionalinfo': { 'unusual': { 'raw_value': 'astringvalue' } diff --git a/tests/mq/plugins/test_filterlog.py b/tests/mq/plugins/test_filterlog.py index 84b553a2..2c123d6f 100644 --- a/tests/mq/plugins/test_filterlog.py +++ b/tests/mq/plugins/test_filterlog.py @@ -45,11 +45,9 @@ class TestFilterlog(): 'sourceipaddress': '175.41.7.2', 'sub_rule_number': '', 'trackor': '1000000103', - 'udp': { - 'data_length': '72', - 'destination_port': '33443', - 'source_port': '57434' - } + 'data_length': '72', + 'destination_port': '33443', + 'source_port': '57434' }, 'summary': '9,,,1000000103,igb0,match,block,in,4,0x0,,6,60624,0,DF,17,udp,92,175.41.7.2,21.143.56.109,57434,33443,72' } diff --git a/tests/mq/plugins/test_lower_keys.py b/tests/mq/plugins/test_lower_keys.py new file mode 100644 index 00000000..1be6c611 --- /dev/null +++ b/tests/mq/plugins/test_lower_keys.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +import os +import sys +sys.path.append(os.path.join(os.path.dirname(__file__), "../../../mq/plugins")) +from lower_keys import message + + +class TestLowerKeysPlugin(): + def setup(self): + self.plugin = message() + + def test_uppercase_details(self): + msg = { + 'source': 'cloudtrail', + 'Details': { + 'requestparameters': { + 'description': 'astringvalue', + } + } + } + (retmessage, retmeta) = self.plugin.onMessage(msg, {}) + + expected_message = { + 'source': 'cloudtrail', + 'details': { + 'requestparameters': { + 'description': 'astringvalue', + } + } + } + assert retmessage == expected_message + assert retmeta == {} + + def test_uppercase_nested_keys(self): + msg = { + 'source': 'cloudtrail', + 'details': { + 'RequestParameters': { + 'Description': 'astringvalue', + } + } + } + (retmessage, retmeta) = self.plugin.onMessage(msg, {}) + + expected_message = { + 'source': 'cloudtrail', + 'details': { + 'requestparameters': { + 'description': 'astringvalue', + } + } + } + assert retmessage == expected_message + assert retmeta == {} + + def test_uppercase_nested_keys2(self): + msg = { + 'source': 'cloudtrail', + 'details': { + 'RequestParameters': { + 'Description': 'astringvalue', + 'ApplicationSource': { + 'someKey:': 'anothervalue', + } + } + } + } + (retmessage, retmeta) = self.plugin.onMessage(msg, {}) + + expected_message = { + 'source': 'cloudtrail', + 'details': { + 'requestparameters': { + 'description': 'astringvalue', + 'applicationsource': { + 'somekey:': 'anothervalue', + } + } + } + } + assert retmessage == expected_message + assert retmeta == {} diff --git a/tests/mq/plugins/test_parse_sshd.py b/tests/mq/plugins/test_parse_sshd.py index 0cbe7743..f2c9f83f 100644 --- a/tests/mq/plugins/test_parse_sshd.py +++ b/tests/mq/plugins/test_parse_sshd.py @@ -24,7 +24,6 @@ accept_message['mozdefhostname'] = 'mozdef4.private.scl3.mozilla.com' accept_message['eventsource'] = 'systemlogs' accept_message['details'] = {} accept_message['details']['processid'] = '5413' -accept_message['details']['Random'] = '9' accept_message['details']['sourceipv4address'] = '10.22.74.208' accept_message['details']['hostname'] = 'mysuperhost.somewhere.com' accept_message['details']['program'] = 'sshd' diff --git a/tests/mq/plugins/test_parse_su.py b/tests/mq/plugins/test_parse_su.py index 2d0793bd..96944eba 100644 --- a/tests/mq/plugins/test_parse_su.py +++ b/tests/mq/plugins/test_parse_su.py @@ -22,7 +22,6 @@ session_su['eventsource'] = 'systemlogs' session_su['hostname'] = 'syslog1.private.scl3.mozilla.com' session_su['mozdefhostname'] = 'mozdef4.private.scl3.mozilla.com' session_su['details'] = {} -session_su['details']['Random'] = '9' session_su['details']['program'] = 'su' session_su['details']['hostname'] = 'irc1.dmz.scl3.mozilla.com' diff --git a/tests/mq/plugins/test_suricataFixup.py b/tests/mq/plugins/test_suricataFixup.py index 43e08a9b..d7dd006d 100644 --- a/tests/mq/plugins/test_suricataFixup.py +++ b/tests/mq/plugins/test_suricataFixup.py @@ -56,7 +56,7 @@ class TestSuricataFixup(object): } event = { 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } @@ -72,7 +72,7 @@ class TestSuricataFixup(object): } event = { 'customendpoint': '', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } @@ -89,7 +89,7 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'alamakota', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } @@ -108,7 +108,7 @@ class TestSuricataFixup(object): 'category': 'suricata', 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log' + 'source': 'eve-log' } result, metadata = self.plugin.onMessage(event, metadata) @@ -125,7 +125,7 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alamakota' } @@ -143,7 +143,7 @@ class TestSuricataFixup(object): MESSAGE = { 'ts': 1505701210.163043 } - event['MESSAGE'] = json.dumps(MESSAGE) + event['message'] = json.dumps(MESSAGE) result, metadata = self.plugin.onMessage(event, self.metadata) assert result['category'] == 'suricata' @@ -154,13 +154,13 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'alamakota', + 'source': 'alamakota', 'event_type': 'alert' } MESSAGE = { 'ts': 1505701210.163043 } - event['MESSAGE'] = json.dumps(MESSAGE) + event['message'] = json.dumps(MESSAGE) result, metadata = self.plugin.onMessage(event, self.metadata) assert result['category'] == 'suricata' @@ -174,7 +174,7 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } result, metadata = self.plugin.onMessage(event, self.metadata) @@ -197,7 +197,6 @@ class TestSuricataFixup(object): "eventsource": "systemslogs", "details": { "processid": "21233", - "Random": 2, "sourceipv4address": "10.22.74.208", "hostname": "hostname1.subdomain.domain.com", "program": "sshd", @@ -274,7 +273,7 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } MESSAGE = { @@ -313,7 +312,7 @@ class TestSuricataFixup(object): "linktype":1 } } - event['MESSAGE'] = json.dumps(MESSAGE) + event['message'] = json.dumps(MESSAGE) result, metadata = self.plugin.onMessage(event, self.metadata) self.verify_defaults(result) @@ -336,7 +335,7 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } MESSAGE = { @@ -385,7 +384,7 @@ class TestSuricataFixup(object): "redirect":"afakedestination" }, } - event['MESSAGE'] = json.dumps(MESSAGE) + event['message'] = json.dumps(MESSAGE) result, metadata = self.plugin.onMessage(event, self.metadata) self.verify_defaults(result) @@ -404,7 +403,7 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } MESSAGE = { @@ -457,7 +456,7 @@ class TestSuricataFixup(object): MESSAGE['payload_printable'] = large_pseudorandom_string MESSAGE['http']['http_response_body'] = large_pseudorandom_string MESSAGE['http']['http_response_body_printable'] = large_pseudorandom_string - event['MESSAGE'] = json.dumps(MESSAGE) + event['message'] = json.dumps(MESSAGE) result, metadata = self.plugin.onMessage(event, self.metadata) self.verify_defaults(result) @@ -479,7 +478,7 @@ class TestSuricataFixup(object): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } MESSAGE = { @@ -519,27 +518,27 @@ class TestSuricataFixup(object): }, "vars":{ "flowbits":{ - "ET.http.javaclient.vulnerable":"true", - "ET.JavaNotJar":"true", - "ET.http.javaclient":"true" + "et.http.javaclient.vulnerable":"true", + "et.javanotjar":"true", + "et.http.javaclient":"true" } } } - event['MESSAGE'] = json.dumps(MESSAGE) + event['message'] = json.dumps(MESSAGE) result, metadata = self.plugin.onMessage(event, self.metadata) self.verify_defaults(result) self.verify_metadata(metadata) assert 'vars' in result['details'] assert 'flowbits' in result['details']['vars'] - assert result['details']['vars']['flowbits']['ET.http.javaclient.vulnerable'] == "True" - assert result['details']['vars']['flowbits']['ET.JavaNotJar'] == "true" + assert result['details']['vars']['flowbits']['et.http.javaclient.vulnerable'] == "true" + assert result['details']['vars']['flowbits']['et.javanotjar'] == "true" def test_eve_log_alert_rename(self): event = { 'customendpoint': '', 'category': 'suricata', - 'SOURCE': 'eve-log', + 'source': 'eve-log', 'event_type': 'alert' } MESSAGE = { @@ -578,7 +577,7 @@ class TestSuricataFixup(object): "linktype":1 } } - event['MESSAGE'] = json.dumps(MESSAGE) + event['message'] = json.dumps(MESSAGE) result, metadata = self.plugin.onMessage(event, self.metadata) self.verify_defaults(result)