Update alert specs for new event format

Signed-off-by: Brandon Myers <bmyers@mozilla.com>

alerts/lib/alerttask.py

@@ -92,8 +92,6 @@ class AlertTask(Task):
     def __init__(self):
         self.alert_name = self.__class__.__name__
         self.main_query = None
-        self.begindateUTC = None
-        self.enddateUTC = None
         # List of events
         self.events = None
         # List of aggregations
@@ -219,12 +217,6 @@ class AlertTask(Task):
         see http://pyes.readthedocs.org/en/latest/references/pyes.filters.html
         """
 
-        self.begindateUTC = toUTC(datetime.now() - timedelta(**query.date_timedelta))
-        self.enddateUTC = toUTC(datetime.now())
-
-        range_query = RangeMatch('utctimestamp', self.begindateUTC, self.enddateUTC)
-        query.add_must(range_query)
-
         # Don't fire on already alerted events
         if ExistsMatch('alerttimestamp') not in query.must_not:
             query.add_must_not(ExistsMatch('alerttimestamp'))

lib/elasticsearch_client.py

@@ -98,6 +98,7 @@ class ElasticsearchClient():
                 for field_name in aggregations:
                     query_obj.aggs.bucket(field_name.to_dict()['terms']['field'], field_name)
                 results = query_obj.execute()
+
         result_set = AggregatedResults(results)
         return result_set
 
@@ -112,10 +113,10 @@ class ElasticsearchClient():
         search_query = SearchQuery()
         search_query.add_must(id_match)
         results = search_query.execute(self, indices=['alerts'])
-        if len(results) == 0:
+        if len(results['hits']) == 0:
             return None
         else:
-            return results[0]
+            return results['hits'][0]
 
     def save_dashboard(self, dash_file, dash_name=None):
         f = open(dash_file)

lib/query_models.py

@@ -1,9 +1,13 @@
 from elasticsearch_dsl import Q, Search, A
-from dotmap import DotMap
 
 import pyes
 import pyes_enabled
 
+from utilities.to_utc import toUTC
+
+from datetime import datetime
+from datetime import timedelta
+
 
 def ExistsMatch(field_name):
     if pyes_enabled.pyes_on is True:
@@ -81,7 +85,7 @@ def AggregatedResults(input_results):
                 '_type': hit['_type'],
                 '_index': hit['_index'],
                 '_score': hit['_score'],
-                'data': hit['_source'],
+                '_source': hit['_source'],
             }
             converted_results['hits'].append(hit_dict)
 
@@ -106,7 +110,7 @@ def AggregatedResults(input_results):
                 '_type': hit.meta.doc_type,
                 '_index': hit.meta.index,
                 '_score': hit.meta.score,
-                'data': hit.to_dict()
+                '_source': hit.to_dict()
             }
             converted_results['hits'].append(hit_dict)
 
@@ -119,7 +123,7 @@ def AggregatedResults(input_results):
 
             converted_results['aggregations'][agg_name] = aggregation_dict
 
-    return DotMap(converted_results)
+    return converted_results
 
 
 def SimpleResults(input_results):
@@ -136,7 +140,7 @@ def SimpleResults(input_results):
                 '_type': hit['_type'],
                 '_index': hit['_index'],
                 '_score': hit['_score'],
-                'data': hit['_source'],
+                '_source': hit['_source'],
             }
             converted_results['hits'].append(hit_dict)
     else:
@@ -152,12 +156,12 @@ def SimpleResults(input_results):
                 '_type': hit.meta.doc_type,
                 '_index': hit.meta.index,
                 '_score': hit.meta.score,
-                'data': hit.to_dict()
+                '_source': hit.to_dict()
             }
 
             converted_results['hits'].append(hit_dict)
 
-    return DotMap(converted_results)
+    return converted_results
 
 
 class SearchQuery():
@@ -190,10 +194,17 @@ class SearchQuery():
 
     def add_aggregation(self, input_obj):
         self.append_to_array(self.aggregation, input_obj)
-        # self.aggregatio
-        # self.aggregation[name] = input_obj
 
     def execute(self, elasticsearch_client, indices=['events', 'events-previous']):
+        if self.must == [] and self.must_not == [] and self.should == []:
+            raise AttributeError('Must define a must, must_not, or should query')
+
+        if self.date_timedelta:
+            end_date = toUTC(datetime.now())
+            begin_date = toUTC(datetime.now() - timedelta(**self.date_timedelta))
+            range_query = RangeMatch('utctimestamp', begin_date, end_date)
+            self.add_must(range_query)
+
         search_query = None
         if pyes_enabled.pyes_on is True:
             search_query = pyes.ConstantScoreQuery(pyes.MatchAllQuery())

tests/alerts/alert_test_suite.py

@@ -70,29 +70,24 @@ class AlertTestSuite(UnitTestSuite):
     def verify_alert(self, expected_alert):
         assert len(self.alert_task.alert_ids) != 0
 
+        self.es_client.flush('alerts')
         for alert_id in self.alert_task.alert_ids:
             alert = self.get_alert_by_id(alert_id)
 
             assert alert['_index'] == 'alerts'
             assert alert['_type'] == 'alert'
 
-            assert alert['_source']['category'] == expected_alert[
-                '_source']['category']
-            assert alert['_source']['severity'] == expected_alert[
-                '_source']['severity']
-            assert alert['_source']['summary'] == expected_alert[
-                '_source']['summary']
-            assert alert['_source'][
-                'tags'] == expected_alert['_source']['tags']
+            assert alert['_source']['category'] == expected_alert['_source']['category']
+            assert alert['_source']['severity'] == expected_alert['_source']['severity']
+            assert alert['_source']['summary'] == expected_alert['_source']['summary']
+            assert alert['_source']['tags'] == expected_alert['_source']['tags']
 
-            assert len(alert['_source']['events']) == len(
-                expected_alert['_source']['events'])
+            assert len(alert['_source']['events']) == len( expected_alert['_source']['events'])
 
     def verify_alert_not_fired(self):
         assert len(self.alert_task.alert_ids) == 0
 
     def get_alert_by_id(self, alert_id):
-        self.es_client.flush('alerts')
         return self.es_client.get_alert_by_id(alert_id)
 
     def random_ip(self):

tests/query_models/query_test_suite.py

@@ -17,14 +17,16 @@ class QueryTestSuite(UnitTestSuite):
             # else:
             #     print "\t[ERROR]"
 
-            assert len(query_result) is 1
+            assert query_result['meta']['timed_out'] is False
+            assert len(query_result['hits']) is 1
         else:
             # if len(query_result) is 0:
             #     print "\t[SUCCESS]"
             # else:
             #     print "\t[ERROR]"
 
-            assert len(query_result) is 0
+            assert query_result['meta']['timed_out'] is False
+            assert len(query_result['hits']) is 0
 
     def test_query_class(self):
         # print ""
@@ -36,7 +38,7 @@ class QueryTestSuite(UnitTestSuite):
                 self.populate_test_event(event)
 
                 # Testing must
-                search_query = SearchQuery(minutes=1)
+                search_query = SearchQuery()
                 search_query.add_must(query)
                 query_result = search_query.execute(self.es_client)
                 # replace print statement with a specific py.test unit test, so that it shows up in total tests run
@@ -44,7 +46,7 @@ class QueryTestSuite(UnitTestSuite):
                 self.verify_test(query_result, self.positive_test)
 
                 # Testing must_not
-                search_query = SearchQuery(minutes=1)
+                search_query = SearchQuery()
                 search_query.add_must_not(query)
                 query_result = search_query.execute(self.es_client)
                 # replace print statement with a specific py.test unit test, so that it shows up in total tests run

tests/query_models/test_aggregation.py

@@ -1,31 +1,143 @@
-# from positive_test_suite import PositiveTestSuite
-# from negative_test_suite import NegativeTestSuite
 import os
 import sys
 sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
-from query_models import SearchQuery, Aggregation, ExistsMatch
+from query_models import SearchQuery, Aggregation, TermMatch
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 from unit_test_suite import UnitTestSuite
 
 
 class TestAggregation(UnitTestSuite):
-    def testing(self):
+    def test_simple_aggregation(self):
         events = [
             {"test": "value", "note": "abvc"},
             {"test": "value", "note": "abvc"},
             {"test": "value", "note": "think"},
+            {"test": "value", "summary": "think"},
         ]
         for event in events:
             self.populate_test_event(event)
         search_query = SearchQuery()
-        search_query.add_must(ExistsMatch('note'))
-        aggreg = Aggregation('note')
-        search_query.add_aggregation(aggreg)
+        search_query.add_must(TermMatch('test', 'value'))
+        search_query.add_aggregation(Aggregation('note'))
         results = search_query.execute(self.es_client)
-        assert True is True
-        # import pdb
-        # pdb.set_trace()
-        # print results
-        # results = normalize_results(unformatted_results)
-        # assert results['aggregations']['note_terms']['buckets'][0]['count'] == 2
-        # assert results['aggregations']['note_terms']['buckets'][1]['count'] == 2
+
+        assert results['aggregations'].keys() == ['note']
+
+        assert results['aggregations']['note'].keys() == ['terms']
+        assert len(results['aggregations']['note']['terms']) == 2
+        assert results['aggregations']['note']['terms'][0].keys() == ['count', 'key']
+
+        assert results['aggregations']['note']['terms'][0]['count'] == 2
+        assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
+
+        assert results['aggregations']['note']['terms'][1]['count'] == 1
+        assert results['aggregations']['note']['terms'][1]['key'] == 'think'
+
+    def test_multiple_aggregations(self):
+        events = [
+            {"test": "value", "note": "abvc"},
+            {"test": "value", "note": "abvc"},
+            {"test": "value", "note": "think"},
+            {"test": "value", "summary": "think"},
+        ]
+        for event in events:
+            self.populate_test_event(event)
+        search_query = SearchQuery()
+        search_query.add_must(TermMatch('test', 'value'))
+        search_query.add_aggregation(Aggregation('note'))
+        search_query.add_aggregation(Aggregation('test'))
+        results = search_query.execute(self.es_client)
+
+        assert results['aggregations'].keys() == ['note', 'test']
+
+        assert results['aggregations']['note'].keys() == ['terms']
+        assert len(results['aggregations']['note']['terms']) == 2
+        assert results['aggregations']['note']['terms'][0].keys() == ['count', 'key']
+
+        assert results['aggregations']['note']['terms'][0]['count'] == 2
+        assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
+
+        assert results['aggregations']['note']['terms'][1]['count'] == 1
+        assert results['aggregations']['note']['terms'][1]['key'] == 'think'
+
+        assert results['aggregations']['test'].keys() == ['terms']
+        assert len(results['aggregations']['test']['terms']) == 1
+        assert results['aggregations']['test']['terms'][0].keys() == ['count', 'key']
+
+        assert results['aggregations']['test']['terms'][0]['count'] == 4
+        assert results['aggregations']['test']['terms'][0]['key'] == 'value'
+
+    def test_aggregation_non_existing_term(self):
+        events = [
+            {"test": "value", "note": "abvc"},
+            {"test": "value", "note": "abvc"},
+            {"test": "value", "note": "think"},
+            {"test": "value", "summary": "think"},
+        ]
+        for event in events:
+            self.populate_test_event(event)
+        search_query = SearchQuery()
+        search_query.add_must(TermMatch('test', 'value'))
+        search_query.add_aggregation(Aggregation('example'))
+        results = search_query.execute(self.es_client)
+
+        assert results.keys() == ['hits', 'meta', 'aggregations']
+        assert len(results['hits']) == 4
+        assert results['aggregations'].keys() == ['example']
+
+        assert results['aggregations']['example'].keys() == ['terms']
+        assert results['aggregations']['example']['terms'] == []
+
+    def test_aggregation_multiple_layers(self):
+        events = [
+            {
+                "test": "value",
+                "details": {"ip": "127.0.0.1"},
+            },
+            {
+                "test": "value",
+                "details": {"ip": "127.0.0.1"},
+            },
+            {
+                "test": "value",
+                "details": {"ip": "192.168.1.1"},
+            },
+        ]
+
+        for event in events:
+            self.populate_test_event(event)
+
+        search_query = SearchQuery()
+        search_query.add_must(TermMatch('test', 'value'))
+        search_query.add_aggregation(Aggregation('details.ip'))
+        results = search_query.execute(self.es_client)
+
+        assert results['aggregations'].keys() == ['details.ip']
+        assert results['aggregations']['details.ip'].keys() == ['terms']
+        assert len(results['aggregations']['details.ip']['terms']) == 2
+
+        assert results['aggregations']['details.ip']['terms'][0]['count'] == 2
+        assert results['aggregations']['details.ip']['terms'][0]['key'] == "127.0.0.1"
+
+        assert results['aggregations']['details.ip']['terms'][1]['count'] == 1
+        assert results['aggregations']['details.ip']['terms'][1]['key'] == "192.168.1.1"
+
+    def test_aggregation_non_existing_layers_term(self):
+        events = [
+            {"test": "value", "note": "abvc"},
+            {"test": "value", "note": "abvc"},
+            {"test": "value", "note": "think"},
+            {"test": "value", "summary": "think"},
+        ]
+        for event in events:
+            self.populate_test_event(event)
+        search_query = SearchQuery()
+        search_query.add_must(TermMatch('test', 'value'))
+        search_query.add_aggregation(Aggregation('details.ipinformation'))
+        results = search_query.execute(self.es_client)
+
+        assert results['aggregations'].keys() == ['details.ipinformation']
+        assert results['aggregations']['details.ipinformation'].keys() == ['terms']
+        assert len(results['aggregations']['details.ipinformation']['terms']) == 0
+
+# q2.facet.add_term_facet('details.dn', size=20)

tests/query_models/test_search_query.py

@@ -1,8 +1,9 @@
-import os
-import sys
+import pytest
 
 from datetime import datetime
 
+import os
+import sys
 sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
 from query_models import SearchQuery, ExistsMatch, TermMatch, Aggregation
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
@@ -37,8 +38,7 @@ class TestMustInput(SearchQueryUnitTest):
     def test_populated_array(self):
         self.query.add_must(ExistsMatch('details'))
         self.query.add_must([ExistsMatch('note'), TermMatch('note', 'test')])
-        assert self.query.must == [ExistsMatch(
-            'details'), ExistsMatch('note'), TermMatch('note', 'test')]
+        assert self.query.must == [ExistsMatch('details'), ExistsMatch('note'), TermMatch('note', 'test')]
 
 
 class TestMustNotInput(SearchQueryUnitTest):
@@ -57,10 +57,8 @@ class TestMustNotInput(SearchQueryUnitTest):
 
     def test_populated_array(self):
         self.query.add_must_not(ExistsMatch('details'))
-        self.query.add_must_not(
-            [ExistsMatch('note'), TermMatch('note', 'test')])
-        assert self.query.must_not == [ExistsMatch(
-            'details'), ExistsMatch('note'), TermMatch('note', 'test')]
+        self.query.add_must_not([ExistsMatch('note'), TermMatch('note', 'test')])
+        assert self.query.must_not == [ExistsMatch('details'), ExistsMatch('note'), TermMatch('note', 'test')]
 
 
 class TestShouldInput(SearchQueryUnitTest):
@@ -80,8 +78,7 @@ class TestShouldInput(SearchQueryUnitTest):
     def test_populated_array(self):
         self.query.add_should(ExistsMatch('details'))
         self.query.add_should([ExistsMatch('note'), TermMatch('note', 'test')])
-        assert self.query.should == [ExistsMatch(
-            'details'), ExistsMatch('note'), TermMatch('note', 'test')]
+        assert self.query.should == [ExistsMatch('details'), ExistsMatch('note'), TermMatch('note', 'test')]
 
 
 class TestAggregationInput(SearchQueryUnitTest):
@@ -118,52 +115,51 @@ class TestExecute(SearchQueryUnitTest):
         self.populate_example_event()
         results = query.execute(self.es_client)
         assert results.keys() == ['hits', 'meta', 'aggregations']
-        assert results.meta.keys() == ['timed_out']
-        assert results.meta.timed_out is False
+        assert results['meta'].keys() == ['timed_out']
+        assert results['meta']['timed_out'] is False
 
-        assert len(results.hits) == 2
+        assert len(results['hits']) == 2
 
-        assert results.hits[0].keys() == ['_score', '_type', '_id', 'data', '_index']
-        assert type(results.hits[0]._id) == unicode
-        assert results.hits[0]._type == 'event'
+        assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
+        assert type(results['hits'][0]['_id']) == unicode
+        assert results['hits'][0]['_type'] == 'event'
 
-        assert results.hits[0]._index == datetime.now().strftime("events-%Y%m%d")
-        assert results.hits[0]._score == 1.0
+        assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
+        assert results['hits'][0]['_score'] == 1.0
 
-        assert results.hits[0].data.keys() == ['note', 'details', 'summary']
-        assert results.hits[0].data.note == 'Example note'
-        assert results.hits[0].data.summary == 'Test Summary'
+        assert results['hits'][0]['_source'].keys() == ['note', 'details', 'summary']
+        assert results['hits'][0]['_source']['note'] == 'Example note'
+        assert results['hits'][0]['_source']['summary'] == 'Test Summary'
 
-        assert results.hits[0].data.details.keys() == ['information']
-        assert results.hits[0].data.details.information == 'Example information'
+        assert results['hits'][0]['_source']['details'].keys() == ['information']
+        assert results['hits'][0]['_source']['details']['information'] == 'Example information'
 
-        assert results.hits[1].keys() == ['_score', '_type', '_id', 'data', '_index']
-        assert type(results.hits[1]._id) == unicode
-        assert results.hits[1]._type == 'event'
+        assert results['hits'][1].keys() == ['_score', '_type', '_id', '_source', '_index']
+        assert type(results['hits'][1]['_id']) == unicode
+        assert results['hits'][1]['_type'] == 'event'
 
-        assert results.hits[1]._index == datetime.now().strftime("events-%Y%m%d")
-        assert results.hits[1]._score == 1.0
+        assert results['hits'][1]['_index'] == datetime.now().strftime("events-%Y%m%d")
+        assert results['hits'][1]['_score'] == 1.0
 
-        assert results.hits[1].data.keys() == ['note', 'details', 'summary']
-        assert results.hits[1].data.note == 'Example note'
-        assert results.hits[1].data.summary == 'Test Summary'
+        assert results['hits'][1]['_source'].keys() == ['note', 'details', 'summary']
+        assert results['hits'][1]['_source']['note'] == 'Example note'
+        assert results['hits'][1]['_source']['summary'] == 'Test Summary'
 
-        assert results.hits[1].data.details.keys() == ['information']
-        assert results.hits[1].data.details.information == 'Example information'
+        assert results['hits'][1]['_source']['details'].keys() == ['information']
+        assert results['hits'][1]['_source']['details']['information'] == 'Example information'
 
-        assert results.aggregations.keys() == ['note']
+        assert results['aggregations'].keys() == ['note']
 
-        assert results.aggregations.note.keys() == ['terms']
+        assert results['aggregations']['note'].keys() == ['terms']
 
-        assert len(results.aggregations.note.terms) == 2
+        assert len(results['aggregations']['note']['terms']) == 2
 
-        results.aggregations.note.terms.sort()
-        assert results.aggregations.note.terms[0].count == 2
-        assert results.aggregations.note.terms[0].key == 'example'
-
-        assert results.aggregations.note.terms[1].count == 2
-        assert results.aggregations.note.terms[1].key == 'note'
+        results['aggregations']['note']['terms'].sort()
+        assert results['aggregations']['note']['terms'][0]['count'] == 2
+        assert results['aggregations']['note']['terms'][0]['key'] == 'example'
 
+        assert results['aggregations']['note']['terms'][1]['count'] == 2
+        assert results['aggregations']['note']['terms'][1]['key'] == 'note'
 
     def test_simple_query_execute(self):
         self.setup()
@@ -174,86 +170,181 @@ class TestExecute(SearchQueryUnitTest):
         results = query.execute(self.es_client)
 
         assert results.keys() == ['hits', 'meta']
-        # assert results.meta.shards.successful == 5
-        # assert results.meta.shards.failed == 0
-        # assert results.meta.shards.total == 5
-        assert results.meta.keys() == ['timed_out']
-        assert results.meta.timed_out is False
-        # assert type(results.meta.took) is int
-        assert len(results.hits) == 1
+        assert results['meta'].keys() == ['timed_out']
+        assert results['meta']['timed_out'] is False
+        assert len(results['hits']) == 1
 
-        assert results.hits[0].keys() == ['_score', '_type', '_id', 'data', '_index']
-        assert type(results.hits[0]._id) == unicode
-        assert results.hits[0]._type == 'event'
+        assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
+        assert type(results['hits'][0]['_id']) == unicode
+        assert results['hits'][0]['_type'] == 'event'
 
-        assert results.hits[0]._index == datetime.now().strftime("events-%Y%m%d")
-        assert results.hits[0]._score == 1.0
+        assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
+        assert results['hits'][0]['_score'] == 1.0
 
-        assert results.hits[0].data.keys() == ['note', 'details', 'summary']
-        assert results.hits[0].data.note == 'Example note'
-        assert results.hits[0].data.summary == 'Test Summary'
+        assert results['hits'][0]['_source'].keys() == ['note', 'details', 'summary']
+        assert results['hits'][0]['_source']['note'] == 'Example note'
+        assert results['hits'][0]['_source']['summary'] == 'Test Summary'
 
-        assert results.hits[0].data.details.keys() == ['information']
-        assert results.hits[0].data.details.information == 'Example information'
+        assert results['hits'][0]['_source']['details'].keys() == ['information']
+        assert results['hits'][0]['_source']['details']['information'] == 'Example information'
 
-        # # pyes format
-        # assert results.timed_out is False
-        # # assert type(results.took) is int
-        # assert results._shards == {'successful': 5, 'failed': 0, 'total': 5}
-        # assert len(results.hits) == 3
-        # # assert results.hits.max_score == 1.0
-        # # assert results.hits.total == 1
-        # assert len(results.hits.hits) == 1
-        # assert results.hits.hits[0]._score == 1.0
-        # assert results.hits.hits[0]._type == 'event'
-        # assert type(results.hits.hits[0]._id) is str
-        # assert results.hits.hits[0]._index == 'events-20160824'
-        # assert results.hits.hits[0]._source == {'note': 'Example note', 'details': {
-        #     'information': 'Example information'}, 'summary': 'Test Summary'}
-        # assert results.hits.hits[0]._source.note == 'Example note'
-        # assert results.hits.hits[0]._source.summary == 'Test Summary'
-        # assert results.hits.hits[0]._source.details == {
-        #     'information': 'Example information'}
-        # assert results.hits.hits[
-        #     0]._source.details.information == 'Example information'
+        with pytest.raises(KeyError):
+            results['abcdefg']
 
-        # # elasticsearch_dsl format
-        # assert len(results) == 1
-        # assert results._shards == {'successful': 5, 'failed': 0, 'total': 5}
-        # assert len(results.hits) == 1
-        # assert results.hits[0].meta.doc_type == 'event'
-        # assert type(results.hits[0].meta.id) is unicode
-        # assert results.hits[0].meta.index == 'events-20160824'
-        # assert results.hits[0].meta.score == 1.0
-        # assert results.hits[0].note == 'Example note'
-        # assert results.hits[0].summary == 'Test Summary'
-        # assert results.hits[0].details == {'information': 'Example information'}
-        # assert results.hits[0].details.information == 'Example information'
-        # assert results.timed_out is False
-        # assert type(results.took) == int
+        with pytest.raises(KeyError):
+            results['abcdefg']['test']
 
     def test_beginning_time_seconds(self):
         query = SearchQuery(seconds=10)
         assert query.date_timedelta == {'seconds': 10}
 
+        default_event = {
+            "utctimestamp": self.current_timestamp(),
+            "summary": "Test summary",
+            "details": {
+                "note": "Example note",
+            }
+        }
+        self.populate_test_event(default_event)
+
+        too_old_event = default_event
+        too_old_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'seconds': 11})
+        self.populate_test_event(too_old_event)
+
+        not_old_event = default_event
+        not_old_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'seconds': 9})
+        self.populate_test_event(not_old_event)
+
+        query.add_must(ExistsMatch('summary'))
+
+        results = query.execute(self.es_client)
+        assert len(results['hits']) == 2
+
     def test_beginning_time_minutes(self):
         query = SearchQuery(minutes=10)
         assert query.date_timedelta == {'minutes': 10}
 
+        default_event = {
+            "utctimestamp": self.current_timestamp(),
+            "summary": "Test summary",
+            "details": {
+                "note": "Example note",
+            }
+        }
+
+        self.populate_test_event(default_event)
+        default_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'minutes': 11})
+        self.populate_test_event(default_event)
+
+        not_old_event = default_event
+        not_old_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'minutes': 9})
+        self.populate_test_event(not_old_event)
+
+        query.add_must(ExistsMatch('summary'))
+
+        results = query.execute(self.es_client)
+        assert len(results['hits']) == 2
+
     def test_beginning_time_hours(self):
         query = SearchQuery(hours=10)
         assert query.date_timedelta == {'hours': 10}
 
+        default_event = {
+            "utctimestamp": self.current_timestamp(),
+            "summary": "Test summary",
+            "details": {
+                "note": "Example note",
+            }
+        }
+
+        self.populate_test_event(default_event)
+        default_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'hours': 11})
+        self.populate_test_event(default_event)
+
+        not_old_event = default_event
+        not_old_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'hours': 9})
+        self.populate_test_event(not_old_event)
+
+        query.add_must(ExistsMatch('summary'))
+
+        results = query.execute(self.es_client)
+        assert len(results['hits']) == 2
+
     def test_beginning_time_days(self):
         query = SearchQuery(days=10)
         assert query.date_timedelta == {'days': 10}
 
-    # def test_without_queries(self):
-        # query = SearchQuery()
-        # results = query.execute(self.es_client)
-        # assert results == []
+        default_event = {
+            "utctimestamp": self.current_timestamp(),
+            "summary": "Test summary",
+            "details": {
+                "note": "Example note",
+            }
+        }
 
-    # Test search query without queries verifying that utctimestamp is used
-    # what happens if we don't have a utctimestamp field?
-    # test simple execute format of returned event
-    # test advanced execute format of returned event
+        self.populate_test_event(default_event)
+        default_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'days': 11})
+        self.populate_test_event(default_event)
+
+        not_old_event = default_event
+        not_old_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'days': 9})
+        self.populate_test_event(not_old_event)
+
+        query.add_must(ExistsMatch('summary'))
+
+        results = query.execute(self.es_client)
+        assert len(results['hits']) == 2
+
+    def test_without_time_defined(self):
+        query = SearchQuery()
+        assert query.date_timedelta == {}
+
+        default_event = {
+            "utctimestamp": self.current_timestamp(),
+            "summary": "Test summary",
+            "details": {
+                "note": "Example note",
+            }
+        }
+
+        self.populate_test_event(default_event)
+        default_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'days': 11})
+        self.populate_test_event(default_event)
+
+        not_old_event = default_event
+        not_old_event['utctimestamp'] = self.subtract_from_timestamp(self.current_timestamp(), {'days': 9})
+        self.populate_test_event(not_old_event)
+
+        query.add_must(ExistsMatch('summary'))
+
+        results = query.execute(self.es_client)
+        assert len(results['hits']) == 3
+
+    def test_without_utctimestamp(self):
+        query = SearchQuery(days=10)
+        assert query.date_timedelta == {'days': 10}
+
+        default_event = {
+            "timestamp": self.current_timestamp(),
+            "summary": "Test summary",
+            "details": {
+                "note": "Example note",
+            }
+        }
+
+        self.populate_test_event(default_event)
+
+        query.add_must(ExistsMatch('summary'))
+
+        results = query.execute(self.es_client)
+        assert len(results['hits']) == 0
+
+    def test_without_queries_and_timestamp(self):
+        query = SearchQuery()
+        with pytest.raises(AttributeError):
+            query.execute(self.es_client)
+
+    def test_without_queries(self):
+        query = SearchQuery(minutes=10)
+        with pytest.raises(AttributeError):
+            query.execute(self.es_client)

tests/test_elasticsearch_client.py

@@ -1,12 +1,9 @@
-import pytest
 import os
 import sys
 sys.path.append(os.path.join(os.path.dirname(__file__), "../lib"))
 sys.path.append(os.path.join(os.path.dirname(__file__), "../alerts/lib"))
 
 from unit_test_suite import UnitTestSuite
-from elasticsearch_client import ElasticsearchClient
-from config import ES
 
 
 class ElasticsearchClientTest(UnitTestSuite):
@@ -51,7 +48,7 @@ class TestWriteWithRead(ElasticsearchClientTest):
     def test_saved_index(self):
         assert self.saved_alert['_index'] == 'alerts'
 
-    def test_alert_data(self):
+    def test_alert_source(self):
         self.fetched_alert = self.es_client.get_alert_by_id(self.saved_alert['_id'])
         assert self.fetched_alert['_source'] == self.alert
 

tests/unit_test_suite.py

@@ -11,10 +11,6 @@ from dateutil.parser import parse
 import pytz
 
 
-class ElasticsearchException(Exception):
-    pass
-
-
 class UnitTestSuite(object):
     def setup(self):
         self.index_name = datetime.now().strftime("events-%Y%m%d")