Add alert for old events

alerts/old_events.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+#
+# Contributors:
+# Brandon Myers bmyers@mozilla.com
+#
+# Looks for events that have an old timestamp
+# which could mean theres something wrong in the event pipeline
+
+from lib.alerttask import AlertTask
+from query_models import SearchQuery, LessThanMatch
+
+from datetime import datetime, timedelta
+from utilities.toUTC import toUTC
+
+
+class OldEvents(AlertTask):
+
+    def main(self):
+        search_query = SearchQuery(hours=6)
+
+        day_old_date = toUTC(datetime.now() - timedelta(days=1)).isoformat()
+        search_query.add_must(LessThanMatch('utctimestamp', day_old_date))
+        self.filtersManual(search_query)
+
+        self.searchEventsAggregated('mozdefhostname', samplesLimit=1000)
+        self.walkAggregations(threshold=1)
+
+    def onAggregation(self, aggreg):
+        category = 'maitenance'
+        tags = ['pipeline']
+        severity = 'ERROR'
+
+        summary = 'Events have an outdated utctimestamp ({0})'.format(aggreg['count'])
+        return self.createAlertDict(summary, category, tags, aggreg['events'], severity)

tests/alerts/test_old_events.py

@@ -0,0 +1,71 @@
+from positive_alert_test_case import PositiveAlertTestCase
+from negative_alert_test_case import NegativeAlertTestCase
+
+from alert_test_suite import AlertTestSuite
+
+
+class TestOldEvents(AlertTestSuite):
+    # We just create a stub, so that we can replace timestamp fields
+    default_event = {
+        "_source": {}
+    }
+
+    default_alert = {
+        "category": "maitenance",
+        "severity": "ERROR",
+        "tags": ['pipeline'],
+        "summary": "Events have an outdated utctimestamp",
+    }
+
+    test_cases = []
+
+    events = AlertTestSuite.create_events(default_event, 100)
+    temp_alert = AlertTestSuite.copy(default_alert)
+    temp_alert['summary'] = 'Events have an outdated utctimestamp (100)'
+    for event in events:
+        event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'hours': 25})
+    test_cases.append(
+        PositiveAlertTestCase(
+            description="Positive test case with good events",
+            events=events,
+            expected_alert=temp_alert
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'hours': 25})
+    temp_alert = AlertTestSuite.copy(default_alert)
+    temp_alert['summary'] = 'Events have an outdated utctimestamp (1)'
+    test_cases.append(
+        PositiveAlertTestCase(
+            description="Positive test case with good event",
+            events=[event],
+            expected_alert=temp_alert
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with event with recent utctimestamp",
+            events=[event],
+        )
+    )
+
+    events = AlertTestSuite.create_events(default_event, 100)
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with events with recent utctimestamp",
+            events=events,
+        )
+    )
+
+    event = AlertTestSuite.create_event(default_event)
+    event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'hours': 25})
+    event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'hours': 25})
+    test_cases.append(
+        NegativeAlertTestCase(
+            description="Negative test case with event with old utctimestamp and receivedtimestamp",
+            events=[event],
+        )
+    )