From 03eac33139304ff2fcc55c3e94e7582e4cb6d79f Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Thu, 19 Dec 2019 11:04:20 -0600 Subject: [PATCH] Revert "Revert "Alert action invoke"" --- alerts/actions/dashboard_geomodel.py | 6 +- alerts/actions/pagerDutyTriggerEvent.py | 6 +- alerts/lib/alerttask.py | 29 +++--- bot/irc/mozdefbot.py | 22 ++--- bot/slack/mozdefbot.py | 7 +- .../alerts/actions/test_dashboard_geomodel.py | 90 ++++++++++--------- tests/alerts/alert_test_suite.py | 16 ++-- 7 files changed, 97 insertions(+), 79 deletions(-) diff --git a/alerts/actions/dashboard_geomodel.py b/alerts/actions/dashboard_geomodel.py index 01b67e96..4cf4ee28 100644 --- a/alerts/actions/dashboard_geomodel.py +++ b/alerts/actions/dashboard_geomodel.py @@ -50,7 +50,11 @@ class message(object): def write_db_entry(self, alert_record): self.dynamodb.put_item(Item=alert_record) - def onMessage(self, message): + def onMessage(self, alert): + # As of Dec. 3, 2019, alert actions are given entire alerts rather + # than just their source + message = alert['_source'] + if 'details' not in message: return message diff --git a/alerts/actions/pagerDutyTriggerEvent.py b/alerts/actions/pagerDutyTriggerEvent.py index d46e17fe..5ccf94bd 100644 --- a/alerts/actions/pagerDutyTriggerEvent.py +++ b/alerts/actions/pagerDutyTriggerEvent.py @@ -42,7 +42,11 @@ class message(object): except: self.options.docs = {} - def onMessage(self, message): + def onMessage(self, alert): + # As of Dec. 3, 2019, alert actions are given entire alerts rather + # than just their source + message = alert['_source'] + # here is where you do something with the incoming alert message doclink = 'unknown' if message['category'] in self.options.docs: diff --git a/alerts/lib/alerttask.py b/alerts/lib/alerttask.py index 15a2dd90..fd492f3f 100644 --- a/alerts/lib/alerttask.py +++ b/alerts/lib/alerttask.py @@ -216,18 +216,7 @@ class AlertTask(Task): Send alert to the kombu based message queue. The default is rabbitmq. """ try: - # cherry pick items from the alertDict to send to the alerts messageQueue - mqAlert = dict(severity="INFO", category="") - if "severity" in alertDict: - mqAlert["severity"] = alertDict["severity"] - if "category" in alertDict: - mqAlert["category"] = alertDict["category"] - if "utctimestamp" in alertDict: - mqAlert["utctimestamp"] = alertDict["utctimestamp"] - if "eventtimestamp" in alertDict: - mqAlert["eventtimestamp"] = alertDict["eventtimestamp"] - mqAlert["summary"] = alertDict["summary"] - self.log.debug(mqAlert) + self.log.debug(alertDict) ensurePublish = self.mqConn.ensure( self.mqproducer, self.mqproducer.publish, max_retries=10 ) @@ -377,7 +366,8 @@ class AlertTask(Task): alert = self.alertPlugins(alert) alertResultES = self.alertToES(alert) self.tagEventsAlert([i], alertResultES) - self.alertToMessageQueue(alert) + full_alert_doc = self.generate_full_doc(alert, alertResultES) + self.alertToMessageQueue(full_alert_doc) self.hookAfterInsertion(alert) self.saveAlertID(alertResultES) # did we not match anything? @@ -388,7 +378,8 @@ class AlertTask(Task): alert = self.tagBotNotify(alert) self.log.debug(alert) alertResultES = self.alertToES(alert) - self.alertToMessageQueue(alert) + full_alert_doc = self.generate_full_doc(alert, alertResultES) + self.alertToMessageQueue(full_alert_doc) self.hookAfterInsertion(alert) self.saveAlertID(alertResultES) @@ -406,11 +397,12 @@ class AlertTask(Task): self.log.debug(alert) alert = self.alertPlugins(alert) alertResultES = self.alertToES(alert) + full_alert_doc = self.generate_full_doc(alert, alertResultES) # even though we only sample events in the alert # tag all events as alerted to avoid re-alerting # on events we've already processed. self.tagEventsAlert(aggregation["allevents"], alertResultES) - self.alertToMessageQueue(alert) + self.alertToMessageQueue(full_alert_doc) self.saveAlertID(alertResultES) def alertPlugins(self, alert): @@ -546,3 +538,10 @@ class AlertTask(Task): logger.error("FAILED to open the configuration file\n") return json_obj + + def generate_full_doc(self, alert_body, alert_es): + return { + '_id': alert_es['_id'], + '_index': alert_es['_index'], + '_source': alert_body + } diff --git a/bot/irc/mozdefbot.py b/bot/irc/mozdefbot.py index 448629c2..18b44cb9 100755 --- a/bot/irc/mozdefbot.py +++ b/bot/irc/mozdefbot.py @@ -289,11 +289,11 @@ class alertConsumer(ConsumerMixin): try: # just to be safe..check what we were sent. if isinstance(body, dict): - bodyDict = body + full_body = body elif isinstance(body, str): try: - bodyDict = json.loads(body) # lets assume it's json - except ValueError as e: + full_body = json.loads(body) # lets assume it's json + except ValueError: # not json..ack but log the message logger.exception( "alertworker exception: unknown body type received %r" % body) @@ -303,7 +303,9 @@ class alertConsumer(ConsumerMixin): "alertworker exception: unknown body type received %r" % body) return - if 'notify_mozdefbot' in bodyDict and bodyDict['notify_mozdefbot'] is False: + body_dict = full_body['_source'] + + if 'notify_mozdefbot' in body_dict and body_dict['notify_mozdefbot'] is False: # If the alert tells us to not notify, then don't post to IRC message.ack() return @@ -311,9 +313,9 @@ class alertConsumer(ConsumerMixin): # process valid message # see where we send this alert ircchannel = options.alertircchannel - if 'ircchannel' in bodyDict: - if bodyDict['ircchannel'] in options.join.split(","): - ircchannel = bodyDict['ircchannel'] + if 'ircchannel' in body_dict: + if body_dict['ircchannel'] in options.join.split(","): + ircchannel = body_dict['ircchannel'] # see if we need to delay a bit before sending the alert, to avoid # flooding the channel @@ -324,11 +326,11 @@ class alertConsumer(ConsumerMixin): sys.stdout.write('throttling before writing next alert\n') time.sleep(1) self.lastalert = toUTC(datetime.now()) - if len(bodyDict['summary']) > 450: + if len(body_dict['summary']) > 450: sys.stdout.write('alert is more than 450 bytes, truncating\n') - bodyDict['summary'] = bodyDict['summary'][:450] + ' truncated...' + body_dict['summary'] = body_dict['summary'][:450] + ' truncated...' - self.ircBot.client.msg(ircchannel, formatAlert(bodyDict)) + self.ircBot.client.msg(ircchannel, formatAlert(body_dict)) message.ack() except ValueError as e: diff --git a/bot/slack/mozdefbot.py b/bot/slack/mozdefbot.py index 96d0b62a..3b3cd11e 100644 --- a/bot/slack/mozdefbot.py +++ b/bot/slack/mozdefbot.py @@ -48,17 +48,18 @@ class AlertConsumer(ConsumerMixin): try: # just to be safe..check what we were sent. if isinstance(body, dict): - body_dict = body + full_body = body elif isinstance(body, str): try: - body_dict = json.loads(body) # lets assume it's json - except ValueError as e: + full_body = json.loads(body) + except ValueError: # not json..ack but log the message logger.exception("mozdefbot_slack exception: unknown body type received %r" % body) return else: logger.exception("mozdefbot_slack exception: unknown body type received %r" % body) return + body_dict = full_body['_source'] if 'notify_mozdefbot' in body_dict and body_dict['notify_mozdefbot'] is False: # If the alert tells us to not notify, then don't post message diff --git a/tests/alerts/actions/test_dashboard_geomodel.py b/tests/alerts/actions/test_dashboard_geomodel.py index c959384f..4ccde7b0 100644 --- a/tests/alerts/actions/test_dashboard_geomodel.py +++ b/tests/alerts/actions/test_dashboard_geomodel.py @@ -20,37 +20,39 @@ class TestDashboardGeomodel(object): self.plugin = message() self.good_message_dict = { - "category": "geomodel", - "tags": ['geomodel'], - "summary": "ttesterson@mozilla.com NEWCOUNTRY Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before", - "events": [ - { - 'documentsource': { - 'details': { - 'event_time': '2018-08-08T02:11:41.85Z', + "_source": { + "category": "geomodel", + "tags": ['geomodel'], + "summary": "ttesterson@mozilla.com NEWCOUNTRY Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before", + "events": [ + { + 'documentsource': { + 'details': { + 'event_time': '2018-08-08T02:11:41.85Z', + } } } + ], + "details": { + "category": "NEWCOUNTRY", + 'previous_locality_details': { + 'city': 'Oakland', + 'country': 'United States' + }, + "locality_details": { + "city": "Diamond Bar", + "country": "United States" + }, + 'source_ip': '1.2.3.4', + "principal": "ttesterson@mozilla.com", } - ], - "details": { - "category": "NEWCOUNTRY", - 'previous_locality_details': { - 'city': 'Oakland', - 'country': 'United States' - }, - "locality_details": { - "city": "Diamond Bar", - "country": "United States" - }, - 'source_ip': '1.2.3.4', - "principal": "ttesterson@mozilla.com", } } def test_message_good(self): assert self.test_result_record is None result_message = self.plugin.onMessage(self.good_message_dict) - assert result_message == self.good_message_dict + assert result_message == self.good_message_dict['_source'] assert self.test_connect_called is True result_db_entry = self.test_result_record assert type(result_db_entry['alert_code']) is str @@ -66,14 +68,14 @@ class TestDashboardGeomodel(object): assert result_db_entry['url'] == 'https://www.mozilla.org' assert result_db_entry['url_title'] == 'Get Help' assert result_db_entry['user_id'] == 'ttesterson' - assert result_db_entry['alert_str_json'] == json.dumps(self.good_message_dict) + assert result_db_entry['alert_str_json'] == json.dumps(self.good_message_dict['_source']) def test_unknown_new_city_message(self): message_dict = self.good_message_dict - message_dict['details']['locality_details']['city'] = 'UNKNOWN' + message_dict['_source']['details']['locality_details']['city'] = 'UNKNOWN' assert self.test_result_record is None result_message = self.plugin.onMessage(message_dict) - assert result_message == self.good_message_dict + assert result_message == self.good_message_dict['_source'] assert self.test_connect_called is True result_db_entry = self.test_result_record assert type(result_db_entry['alert_code']) is str @@ -81,32 +83,34 @@ class TestDashboardGeomodel(object): def test_malformed_message_bad(self): message_dict = { - "category": "geomodel", - "tags": ['geomodel'], - "summary": "ttesterson@mozilla.com MOVEMENT Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before", - "details": { - "category": "MOVEMENT", - "locality_details": { - "city": "Diamond Bar", - "country": "United States" - }, - "principal": "ttesterson@mozilla.com", + "_source": { + "category": "geomodel", + "tags": ['geomodel'], + "summary": "ttesterson@mozilla.com MOVEMENT Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before", + "details": { + "category": "MOVEMENT", + "locality_details": { + "city": "Diamond Bar", + "country": "United States" + }, + "principal": "ttesterson@mozilla.com", + } } } assert self.test_result_record is None result_message = self.plugin.onMessage(message_dict) - assert result_message == message_dict + assert result_message == message_dict['_source'] assert self.test_connect_called is True assert self.test_result_record is None def test_str_location(self): - self.good_message_dict['summary'] = "ttesterson@mozilla.com NEWCOUNTRY \u0107abcd, \xe4Spain access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before" - self.good_message_dict['details']['locality_details']['city'] = '\u0107abcd' - self.good_message_dict['details']['locality_details']['country'] = '\xe4Spain' + self.good_message_dict['_source']['summary'] = "ttesterson@mozilla.com NEWCOUNTRY \u0107abcd, \xe4Spain access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before" + self.good_message_dict['_source']['details']['locality_details']['city'] = '\u0107abcd' + self.good_message_dict['_source']['details']['locality_details']['country'] = '\xe4Spain' assert self.test_result_record is None result_message = self.plugin.onMessage(self.good_message_dict) - assert result_message == self.good_message_dict + assert result_message == self.good_message_dict['_source'] assert self.test_connect_called is True assert self.test_result_record is not None assert type(result_message['summary']) is str @@ -114,10 +118,10 @@ class TestDashboardGeomodel(object): assert type(result_message['details']['locality_details']['country']) is str def test_str_username(self): - self.good_message_dict['details']['principal'] = '\xfcttesterson@mozilla.com' + self.good_message_dict['_source']['details']['principal'] = '\xfcttesterson@mozilla.com' assert self.test_result_record is None result_message = self.plugin.onMessage(self.good_message_dict) - assert result_message == self.good_message_dict + assert result_message == self.good_message_dict['_source'] assert self.test_connect_called is True assert self.test_result_record is not None assert type(result_message['summary']) is str @@ -126,7 +130,7 @@ class TestDashboardGeomodel(object): def test_written_details(self): assert self.test_result_record is None result_message = self.plugin.onMessage(self.good_message_dict) - assert result_message == self.good_message_dict + assert result_message == self.good_message_dict['_source'] assert self.test_connect_called is True assert self.test_result_record is not None result_db_entry = self.test_result_record diff --git a/tests/alerts/alert_test_suite.py b/tests/alerts/alert_test_suite.py index 42704715..954386f1 100644 --- a/tests/alerts/alert_test_suite.py +++ b/tests/alerts/alert_test_suite.py @@ -161,12 +161,16 @@ class AlertTestSuite(UnitTestSuite): rabbitmq_message = self.rabbitmq_alerts_consumer.channel.basic_get() rabbitmq_message.channel.basic_ack(rabbitmq_message.delivery_tag) document = json.loads(rabbitmq_message.body) - assert document['notify_mozdefbot'] is test_case.expected_alert['notify_mozdefbot'], 'Alert from rabbitmq has bad notify_mozdefbot field' - assert document['ircchannel'] == test_case.expected_alert['ircchannel'], 'Alert from rabbitmq has bad ircchannel field' - assert document['summary'] == found_alert['_source']['summary'], 'Alert from rabbitmq has bad summary field' - assert document['utctimestamp'] == found_alert['_source']['utctimestamp'], 'Alert from rabbitmq has bad utctimestamp field' - assert document['category'] == found_alert['_source']['category'], 'Alert from rabbitmq has bad category field' - assert len(document['events']) == len(found_alert['_source']['events']), 'Alert from rabbitmq has bad events field' + assert '_id' in document + assert '_source' in document + assert '_index' in document + alert_body = document['_source'] + assert alert_body['notify_mozdefbot'] is test_case.expected_alert['notify_mozdefbot'], 'Alert from rabbitmq has bad notify_mozdefbot field' + assert alert_body['ircchannel'] == test_case.expected_alert['ircchannel'], 'Alert from rabbitmq has bad ircchannel field' + assert alert_body['summary'] == found_alert['_source']['summary'], 'Alert from rabbitmq has bad summary field' + assert alert_body['utctimestamp'] == found_alert['_source']['utctimestamp'], 'Alert from rabbitmq has bad utctimestamp field' + assert alert_body['category'] == found_alert['_source']['category'], 'Alert from rabbitmq has bad category field' + assert len(alert_body['events']) == len(found_alert['_source']['events']), 'Alert from rabbitmq has bad events field' def verify_saved_events(self, found_alert, test_case): """