Merge pull request #1533 from mozilla/revert-1528-alert-action-invoke

Revert "Alert action invoke"

alerts/actions/dashboard_geomodel.py

@@ -50,11 +50,7 @@ class message(object):
     def write_db_entry(self, alert_record):
         self.dynamodb.put_item(Item=alert_record)
 
-    def onMessage(self, alert):
+    def onMessage(self, message):
-        # As of Dec. 3, 2019, alert actions are given entire alerts rather
-        # than just their source
-        message = alert['_source']
-
         if 'details' not in message:
             return message
 

alerts/actions/pagerDutyTriggerEvent.py

@@ -42,11 +42,7 @@ class message(object):
         except:
             self.options.docs = {}
 
-    def onMessage(self, alert):
+    def onMessage(self, message):
-        # As of Dec. 3, 2019, alert actions are given entire alerts rather
-        # than just their source
-        message = alert['_source']
-
         # here is where you do something with the incoming alert message
         doclink = 'unknown'
         if message['category'] in self.options.docs:

alerts/lib/alerttask.py

@@ -216,7 +216,18 @@ class AlertTask(Task):
         Send alert to the kombu based message queue.  The default is rabbitmq.
         """
         try:
-            self.log.debug(alertDict)
+            # cherry pick items from the alertDict to send to the alerts messageQueue
+            mqAlert = dict(severity="INFO", category="")
+            if "severity" in alertDict:
+                mqAlert["severity"] = alertDict["severity"]
+            if "category" in alertDict:
+                mqAlert["category"] = alertDict["category"]
+            if "utctimestamp" in alertDict:
+                mqAlert["utctimestamp"] = alertDict["utctimestamp"]
+            if "eventtimestamp" in alertDict:
+                mqAlert["eventtimestamp"] = alertDict["eventtimestamp"]
+            mqAlert["summary"] = alertDict["summary"]
+            self.log.debug(mqAlert)
             ensurePublish = self.mqConn.ensure(
                 self.mqproducer, self.mqproducer.publish, max_retries=10
             )
@@ -366,8 +377,7 @@ class AlertTask(Task):
                     alert = self.alertPlugins(alert)
                     alertResultES = self.alertToES(alert)
                     self.tagEventsAlert([i], alertResultES)
-                    full_alert_doc = self.generate_full_doc(alert, alertResultES)
+                    self.alertToMessageQueue(alert)
-                    self.alertToMessageQueue(full_alert_doc)
                     self.hookAfterInsertion(alert)
                     self.saveAlertID(alertResultES)
         # did we not match anything?
@@ -378,8 +388,7 @@ class AlertTask(Task):
                 alert = self.tagBotNotify(alert)
                 self.log.debug(alert)
                 alertResultES = self.alertToES(alert)
-                full_alert_doc = self.generate_full_doc(alert, alertResultES)
+                self.alertToMessageQueue(alert)
-                self.alertToMessageQueue(full_alert_doc)
                 self.hookAfterInsertion(alert)
                 self.saveAlertID(alertResultES)
 
@@ -397,12 +406,11 @@ class AlertTask(Task):
                         self.log.debug(alert)
                         alert = self.alertPlugins(alert)
                         alertResultES = self.alertToES(alert)
-                        full_alert_doc = self.generate_full_doc(alert, alertResultES)
                         # even though we only sample events in the alert
                         # tag all events as alerted to avoid re-alerting
                         # on events we've already processed.
                         self.tagEventsAlert(aggregation["allevents"], alertResultES)
-                        self.alertToMessageQueue(full_alert_doc)
+                        self.alertToMessageQueue(alert)
                         self.saveAlertID(alertResultES)
 
     def alertPlugins(self, alert):
@@ -538,10 +546,3 @@ class AlertTask(Task):
                 logger.error("FAILED to open the configuration file\n")
 
         return json_obj
-
-    def generate_full_doc(self, alert_body, alert_es):
-        return {
-            '_id': alert_es['_id'],
-            '_index': alert_es['_index'],
-            '_source': alert_body
-        }

bot/irc/mozdefbot.py

@@ -289,11 +289,11 @@ class alertConsumer(ConsumerMixin):
         try:
             # just to be safe..check what we were sent.
             if isinstance(body, dict):
-                full_body = body
+                bodyDict = body
             elif isinstance(body, str):
                 try:
-                    full_body = json.loads(body)  # lets assume it's json
+                    bodyDict = json.loads(body)  # lets assume it's json
-                except ValueError:
+                except ValueError as e:
                     # not json..ack but log the message
                     logger.exception(
                         "alertworker exception: unknown body type received %r" % body)
@@ -303,9 +303,7 @@ class alertConsumer(ConsumerMixin):
                     "alertworker exception: unknown body type received %r" % body)
                 return
 
-            body_dict = full_body['_source']
+            if 'notify_mozdefbot' in bodyDict and bodyDict['notify_mozdefbot'] is False:
-
-            if 'notify_mozdefbot' in body_dict and body_dict['notify_mozdefbot'] is False:
                 # If the alert tells us to not notify, then don't post to IRC
                 message.ack()
                 return
@@ -313,9 +311,9 @@ class alertConsumer(ConsumerMixin):
             # process valid message
             # see where we send this alert
             ircchannel = options.alertircchannel
-            if 'ircchannel' in body_dict:
+            if 'ircchannel' in bodyDict:
-                if body_dict['ircchannel'] in options.join.split(","):
+                if bodyDict['ircchannel'] in options.join.split(","):
-                    ircchannel = body_dict['ircchannel']
+                    ircchannel = bodyDict['ircchannel']
 
             # see if we need to delay a bit before sending the alert, to avoid
             # flooding the channel
@@ -326,11 +324,11 @@ class alertConsumer(ConsumerMixin):
                     sys.stdout.write('throttling before writing next alert\n')
                     time.sleep(1)
             self.lastalert = toUTC(datetime.now())
-            if len(body_dict['summary']) > 450:
+            if len(bodyDict['summary']) > 450:
                 sys.stdout.write('alert is more than 450 bytes, truncating\n')
-                body_dict['summary'] = body_dict['summary'][:450] + ' truncated...'
+                bodyDict['summary'] = bodyDict['summary'][:450] + ' truncated...'
 
-            self.ircBot.client.msg(ircchannel, formatAlert(body_dict))
+            self.ircBot.client.msg(ircchannel, formatAlert(bodyDict))
 
             message.ack()
         except ValueError as e:

bot/slack/mozdefbot.py

@@ -48,18 +48,17 @@ class AlertConsumer(ConsumerMixin):
         try:
             # just to be safe..check what we were sent.
             if isinstance(body, dict):
-                full_body = body
+                body_dict = body
             elif isinstance(body, str):
                 try:
-                    full_body = json.loads(body)
+                    body_dict = json.loads(body)  # lets assume it's json
-                except ValueError:
+                except ValueError as e:
                     # not json..ack but log the message
                     logger.exception("mozdefbot_slack exception: unknown body type received %r" % body)
                     return
             else:
                 logger.exception("mozdefbot_slack exception: unknown body type received %r" % body)
                 return
-            body_dict = full_body['_source']
 
             if 'notify_mozdefbot' in body_dict and body_dict['notify_mozdefbot'] is False:
                 # If the alert tells us to not notify, then don't post message

tests/alerts/actions/test_dashboard_geomodel.py

@@ -20,39 +20,37 @@ class TestDashboardGeomodel(object):
 
         self.plugin = message()
         self.good_message_dict = {
-            "_source": {
+            "category": "geomodel",
-                "category": "geomodel",
+            "tags": ['geomodel'],
-                "tags": ['geomodel'],
+            "summary": "ttesterson@mozilla.com NEWCOUNTRY Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before",
-                "summary": "ttesterson@mozilla.com NEWCOUNTRY Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before",
+            "events": [
-                "events": [
+                {
-                    {
+                    'documentsource': {
-                        'documentsource': {
+                        'details': {
-                            'details': {
+                            'event_time': '2018-08-08T02:11:41.85Z',
-                                'event_time': '2018-08-08T02:11:41.85Z',
-                            }
                         }
                     }
-                ],
-                "details": {
-                    "category": "NEWCOUNTRY",
-                    'previous_locality_details': {
-                        'city': 'Oakland',
-                        'country': 'United States'
-                    },
-                    "locality_details": {
-                        "city": "Diamond Bar",
-                        "country": "United States"
-                    },
-                    'source_ip': '1.2.3.4',
-                    "principal": "ttesterson@mozilla.com",
                 }
+            ],
+            "details": {
+                "category": "NEWCOUNTRY",
+                'previous_locality_details': {
+                    'city': 'Oakland',
+                    'country': 'United States'
+                },
+                "locality_details": {
+                    "city": "Diamond Bar",
+                    "country": "United States"
+                },
+                'source_ip': '1.2.3.4',
+                "principal": "ttesterson@mozilla.com",
             }
         }
 
     def test_message_good(self):
         assert self.test_result_record is None
         result_message = self.plugin.onMessage(self.good_message_dict)
-        assert result_message == self.good_message_dict['_source']
+        assert result_message == self.good_message_dict
         assert self.test_connect_called is True
         result_db_entry = self.test_result_record
         assert type(result_db_entry['alert_code']) is str
@@ -68,14 +66,14 @@ class TestDashboardGeomodel(object):
         assert result_db_entry['url'] == 'https://www.mozilla.org'
         assert result_db_entry['url_title'] == 'Get Help'
         assert result_db_entry['user_id'] == 'ttesterson'
-        assert result_db_entry['alert_str_json'] == json.dumps(self.good_message_dict['_source'])
+        assert result_db_entry['alert_str_json'] == json.dumps(self.good_message_dict)
 
     def test_unknown_new_city_message(self):
         message_dict = self.good_message_dict
-        message_dict['_source']['details']['locality_details']['city'] = 'UNKNOWN'
+        message_dict['details']['locality_details']['city'] = 'UNKNOWN'
         assert self.test_result_record is None
         result_message = self.plugin.onMessage(message_dict)
-        assert result_message == self.good_message_dict['_source']
+        assert result_message == self.good_message_dict
         assert self.test_connect_called is True
         result_db_entry = self.test_result_record
         assert type(result_db_entry['alert_code']) is str
@@ -83,34 +81,32 @@ class TestDashboardGeomodel(object):
 
     def test_malformed_message_bad(self):
         message_dict = {
-            "_source": {
+            "category": "geomodel",
-                "category": "geomodel",
+            "tags": ['geomodel'],
-                "tags": ['geomodel'],
+            "summary": "ttesterson@mozilla.com MOVEMENT Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before",
-                "summary": "ttesterson@mozilla.com MOVEMENT Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before",
+            "details": {
-                "details": {
+                "category": "MOVEMENT",
-                    "category": "MOVEMENT",
+                "locality_details": {
-                    "locality_details": {
+                    "city": "Diamond Bar",
-                        "city": "Diamond Bar",
+                    "country": "United States"
-                        "country": "United States"
+                },
-                    },
+                "principal": "ttesterson@mozilla.com",
-                    "principal": "ttesterson@mozilla.com",
-                }
             }
         }
 
         assert self.test_result_record is None
         result_message = self.plugin.onMessage(message_dict)
-        assert result_message == message_dict['_source']
+        assert result_message == message_dict
         assert self.test_connect_called is True
         assert self.test_result_record is None
 
     def test_str_location(self):
-        self.good_message_dict['_source']['summary'] = "ttesterson@mozilla.com NEWCOUNTRY \u0107abcd, \xe4Spain access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before"
+        self.good_message_dict['summary'] = "ttesterson@mozilla.com NEWCOUNTRY \u0107abcd, \xe4Spain access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before"
-        self.good_message_dict['_source']['details']['locality_details']['city'] = '\u0107abcd'
+        self.good_message_dict['details']['locality_details']['city'] = '\u0107abcd'
-        self.good_message_dict['_source']['details']['locality_details']['country'] = '\xe4Spain'
+        self.good_message_dict['details']['locality_details']['country'] = '\xe4Spain'
         assert self.test_result_record is None
         result_message = self.plugin.onMessage(self.good_message_dict)
-        assert result_message == self.good_message_dict['_source']
+        assert result_message == self.good_message_dict
         assert self.test_connect_called is True
         assert self.test_result_record is not None
         assert type(result_message['summary']) is str
@@ -118,10 +114,10 @@ class TestDashboardGeomodel(object):
         assert type(result_message['details']['locality_details']['country']) is str
 
     def test_str_username(self):
-        self.good_message_dict['_source']['details']['principal'] = '\xfcttesterson@mozilla.com'
+        self.good_message_dict['details']['principal'] = '\xfcttesterson@mozilla.com'
         assert self.test_result_record is None
         result_message = self.plugin.onMessage(self.good_message_dict)
-        assert result_message == self.good_message_dict['_source']
+        assert result_message == self.good_message_dict
         assert self.test_connect_called is True
         assert self.test_result_record is not None
         assert type(result_message['summary']) is str
@@ -130,7 +126,7 @@ class TestDashboardGeomodel(object):
     def test_written_details(self):
         assert self.test_result_record is None
         result_message = self.plugin.onMessage(self.good_message_dict)
-        assert result_message == self.good_message_dict['_source']
+        assert result_message == self.good_message_dict
         assert self.test_connect_called is True
         assert self.test_result_record is not None
         result_db_entry = self.test_result_record

tests/alerts/alert_test_suite.py

@@ -161,16 +161,12 @@ class AlertTestSuite(UnitTestSuite):
         rabbitmq_message = self.rabbitmq_alerts_consumer.channel.basic_get()
         rabbitmq_message.channel.basic_ack(rabbitmq_message.delivery_tag)
         document = json.loads(rabbitmq_message.body)
-        assert '_id' in document
+        assert document['notify_mozdefbot'] is test_case.expected_alert['notify_mozdefbot'], 'Alert from rabbitmq has bad notify_mozdefbot field'
-        assert '_source' in document
+        assert document['ircchannel'] == test_case.expected_alert['ircchannel'], 'Alert from rabbitmq has bad ircchannel field'
-        assert '_index' in document
+        assert document['summary'] == found_alert['_source']['summary'], 'Alert from rabbitmq has bad summary field'
-        alert_body = document['_source']
+        assert document['utctimestamp'] == found_alert['_source']['utctimestamp'], 'Alert from rabbitmq has bad utctimestamp field'
-        assert alert_body['notify_mozdefbot'] is test_case.expected_alert['notify_mozdefbot'], 'Alert from rabbitmq has bad notify_mozdefbot field'
+        assert document['category'] == found_alert['_source']['category'], 'Alert from rabbitmq has bad category field'
-        assert alert_body['ircchannel'] == test_case.expected_alert['ircchannel'], 'Alert from rabbitmq has bad ircchannel field'
+        assert len(document['events']) == len(found_alert['_source']['events']), 'Alert from rabbitmq has bad events field'
-        assert alert_body['summary'] == found_alert['_source']['summary'], 'Alert from rabbitmq has bad summary field'
-        assert alert_body['utctimestamp'] == found_alert['_source']['utctimestamp'], 'Alert from rabbitmq has bad utctimestamp field'
-        assert alert_body['category'] == found_alert['_source']['category'], 'Alert from rabbitmq has bad category field'
-        assert len(alert_body['events']) == len(found_alert['_source']['events']), 'Alert from rabbitmq has bad events field'
 
     def verify_saved_events(self, found_alert, test_case):
         """