mozilla
/
MozDef
зеркало из https://github.com/mozilla/MozDef.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add travisci to project and stabalize tests

This commit is contained in:
Brandon Myers 2017-07-05 15:34:32 -05:00 коммит произвёл Brandon Myers
Родитель c26b2aee29
Коммит ad64804e32
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 8AA79AD83045BBC7
20 изменённых файлов: 139 добавлений и 177 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

16
.travis.yml Normal file
Просмотреть файл

@ -0,0 +1,16 @@
language: python
python:
- '2.7.11'
before_install:
- ES_VERSION=2.4.4; curl -O https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/${ES_VERSION}/elasticsearch-${ES_VERSION}.deb && sudo dpkg -i --force-confnew elasticsearch-${ES_VERSION}.deb && sudo service elasticsearch restart
- sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
- sudo dpkg-reconfigure --frontend noninteractive tzdata
services:
- rabbitmq
install:
- "pip install -r requirements.txt"
- "pip install -r requirements_tests.txt"
before_script:
- sleep 5
script:
- py.test --delete_indexes --ignore tests/loginput --ignore tests/rest tests

1
README.md
Просмотреть файл

@ -1,3 +1,4 @@
[![Build Status](https://travis-ci.org/pwnbus/MozDef.svg?branch=master)](https://travis-ci.org/pwnbus/MozDef)
MozDef: The Mozilla Defense Platform
=====================================

4
alerts/lib/config.py
Просмотреть файл

@ -20,8 +20,8 @@ ALERTS={
RABBITMQ = {
'mqserver': 'localhost',
'mquser': 'mozdef',
'mqpassword': 'mozdef',
'mquser': 'guest',
'mqpassword': 'guest',
'mqport': 5672,
'alertexchange': 'alerts',
'alertqueue': 'mozdef.alert'

1
counter Normal file
Просмотреть файл

@ -0,0 +1 @@
20

0
mq/__init__.py Normal file
Просмотреть файл

2
requirements_tests.txt Normal file
Просмотреть файл

@ -0,0 +1,2 @@
pytest==3.1.1
WebTest==2.0.27

1
tests/alerts/alert_test_case.py
Просмотреть файл

@ -17,7 +17,6 @@ class AlertTestCase(object):
self.full_events = []
def run(self, alert_filename, alert_classname):
print '\n\tTesting {} '.format(self.description),
alert_file_module = __import__(alert_filename)
alert_class_attr = getattr(alert_file_module, alert_classname)

4
tests/alerts/alert_test_suite.py
Просмотреть файл

@ -103,7 +103,7 @@ class AlertTestSuite(UnitTestSuite):
test_case.full_events.append(merged_event)
self.populate_test_event(merged_event['_source'], merged_event['_type'])
self.es_client.flush('events')
self.flush('events')
alert_task = test_case.run(alert_filename=self.alert_filename, alert_classname=self.alert_classname)
self.verify_alert_task(alert_task, test_case)
@ -138,7 +138,7 @@ class AlertTestSuite(UnitTestSuite):
def verify_alert_task(self, alert_task, test_case):
if test_case.expected_test_result is True:
assert len(alert_task.alert_ids) is not 0, 'Alert did not fire as expected'
self.es_client.flush('alerts')
self.flush('alerts')
for alert_id in alert_task.alert_ids:
found_alert = self.es_client.get_alert_by_id(alert_id)
self.verify_expected_alert(found_alert, test_case)

2
tests/alerts/test_cloudtrail_logging_disabled.py
Просмотреть файл

@ -24,7 +24,7 @@ class TestAlertCloudtrailLoggingDisabled(AlertTestSuite):
"category": "AWSCloudtrail",
"severity": "CRITICAL",
"summary": "Cloudtrail Logging Disabled: cloudtrail_example_name",
"tags": ['cloudtrail', 'aws'],
"tags": ['cloudtrail', 'aws', 'cloudtrailpagerduty'],
}
test_cases = []

2
tests/alerts/test_open_port_violation.py
Просмотреть файл

@ -31,7 +31,7 @@ class TestAlertOpenPortViolation(AlertTestSuite):
# This alert is the expected result from running this task
default_alert = {
"category": "open_port_policy_violation",
"tags": ['open_port_policy_violation'],
"tags": ['open_port_policy_violation', 'openportpagerduty'],
"severity": "CRITICAL",
"summary": '10 unauthorized open port(s) on 1.2.3.4 (25 25 25 25 25 )',
}

1
tests/lib/query_models/query_test_suite.py
Просмотреть файл

@ -36,6 +36,7 @@ class QueryTestSuite(UnitTestSuite):
self.setup_elasticsearch()
self.populate_test_event(event)
self.flush(self.event_index_name)
# Testing must
search_query = SearchQuery()

19
tests/lib/query_models/test_aggregation.py
Просмотреть файл

@ -28,6 +28,8 @@ class TestAggregation(UnitTestSuite):
]
for event in events:
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
@ -60,6 +62,9 @@ class TestAggregation(UnitTestSuite):
]
for event in events:
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
@ -96,6 +101,9 @@ class TestAggregation(UnitTestSuite):
]
for event in events:
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('example'))
@ -127,6 +135,8 @@ class TestAggregation(UnitTestSuite):
for event in events:
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ip'))
@ -151,6 +161,9 @@ class TestAggregation(UnitTestSuite):
]
for event in events:
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ipinformation'))
@ -164,6 +177,9 @@ class TestAggregation(UnitTestSuite):
for num in range(0, 100):
event = {'keyname': 'value' + str(num)}
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
search_query.add_aggregation(Aggregation('keyname'))
@ -174,6 +190,9 @@ class TestAggregation(UnitTestSuite):
for num in range(0, 100):
event = {'keyname': 'value' + str(num)}
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
search_query.add_aggregation(Aggregation('keyname', 2))

43
tests/lib/query_models/test_search_query.py
Просмотреть файл

@ -138,6 +138,9 @@ class TestExecute(SearchQueryUnitTest):
}
}
)
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
@ -203,6 +206,8 @@ class TestExecute(SearchQueryUnitTest):
event = self.generate_default_event()
event['_source']['utctimestamp'] = event['_source']['utctimestamp']()
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery(minutes=10)
search_query.add_aggregation(Aggregation('summary'))
@ -211,11 +216,14 @@ class TestExecute(SearchQueryUnitTest):
def test_aggregation_query_execute(self):
query = SearchQuery()
assert query.date_timedelta == {}
query.add_must(ExistsMatch('note'))
query.add_aggregation(Aggregation('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.populate_example_event()
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
@ -264,9 +272,12 @@ class TestExecute(SearchQueryUnitTest):
def test_simple_query_execute(self):
query = SearchQuery()
assert query.date_timedelta == {}
query.add_must(ExistsMatch('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta']
@ -295,6 +306,7 @@ class TestExecute(SearchQueryUnitTest):
def test_beginning_time_seconds(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
default_event = {
@ -314,13 +326,14 @@ class TestExecute(SearchQueryUnitTest):
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
self.populate_test_event(not_old_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_beginning_time_minutes(self):
query = SearchQuery(minutes=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'minutes': 10}
default_event = {
@ -339,13 +352,14 @@ class TestExecute(SearchQueryUnitTest):
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'minutes': 9})
self.populate_test_event(not_old_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_beginning_time_hours(self):
query = SearchQuery(hours=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'hours': 10}
default_event = {
@ -364,13 +378,14 @@ class TestExecute(SearchQueryUnitTest):
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'hours': 9})
self.populate_test_event(not_old_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_beginning_time_days(self):
query = SearchQuery(days=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'days': 10}
default_event = {
@ -389,13 +404,14 @@ class TestExecute(SearchQueryUnitTest):
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 9})
self.populate_test_event(not_old_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_without_time_defined(self):
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {}
default_event = {
@ -414,13 +430,14 @@ class TestExecute(SearchQueryUnitTest):
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 9})
self.populate_test_event(not_old_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 3
def test_without_utctimestamp(self):
query = SearchQuery(days=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'days': 10}
default_event = {
@ -432,8 +449,7 @@ class TestExecute(SearchQueryUnitTest):
}
self.populate_test_event(default_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 0
@ -451,6 +467,7 @@ class TestExecute(SearchQueryUnitTest):
def test_execute_with_size(self):
for num in range(0, 30):
self.populate_example_event()
self.flush(self.event_index_name)
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client, size=12)
@ -459,6 +476,7 @@ class TestExecute(SearchQueryUnitTest):
def test_execute_without_size(self):
for num in range(0, 1200):
self.populate_example_event()
self.flush(self.event_index_name)
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
@ -466,6 +484,7 @@ class TestExecute(SearchQueryUnitTest):
def test_execute_with_should(self):
self.populate_example_event()
self.flush(self.event_index_name)
self.query.add_should(ExistsMatch('summary'))
self.query.add_should(ExistsMatch('nonexistentfield'))
results = self.query.execute(self.es_client)
@ -473,6 +492,7 @@ class TestExecute(SearchQueryUnitTest):
def test_beginning_time_seconds_received_timestamp(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
default_event = {
@ -492,13 +512,14 @@ class TestExecute(SearchQueryUnitTest):
not_old_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
self.populate_test_event(not_old_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_time_received_timestamp(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
received_timestamp_default_event = {
@ -537,7 +558,7 @@ class TestExecute(SearchQueryUnitTest):
modified_utc_timestamp_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
self.populate_test_event(modified_utc_timestamp_event)
query.add_must(ExistsMatch('summary'))
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 5

6
tests/lib/test_bulk_queue.py
Просмотреть файл

@ -16,7 +16,7 @@ class BulkQueueTest(UnitTestSuite):
super(BulkQueueTest, self).setup()
def num_objects_saved(self):
self.es_client.flush('events')
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
results = search_query.execute(self.es_client)
@ -113,13 +113,13 @@ class TestTimer(BulkQueueTest):
queue.add(index='events', doc_type='event', body={'keyname': 'value' + str(num)})
assert self.num_objects_saved() == 200
assert queue.size() == 1
time.sleep(5)
time.sleep(3)
assert self.num_objects_saved() == 201
assert queue.size() == 0
for num in range(0, 201):
queue.add(index='events', doc_type='event', body={'keyname': 'value' + str(num)})
assert self.num_objects_saved() == 401
time.sleep(5)
time.sleep(3)
assert self.num_objects_saved() == 402
queue.stop_timer()

75
tests/lib/test_elasticsearch_client.py
Просмотреть файл

@ -32,7 +32,7 @@ class ElasticsearchClientTest(UnitTestSuite):
self.es_client = ElasticsearchClient(ES['servers'], bulk_refresh_time=3)
def get_num_events(self):
self.es_client.flush('events')
self.flush('events')
search_query = SearchQuery()
search_query.add_must(TermMatch('_type', 'event'))
search_query.add_aggregation(Aggregation('_type'))
@ -48,20 +48,12 @@ class MockTransportClass:
def __init__(self):
self.request_counts = 0
self.original_function = None
# Exclude certain paths/urls so that we only
# count requests that were made to ADD events
self.exclude_paths = [
"/events,events-previous/_search",
"/events/_flush",
"/_all/_flush",
"/events%2Cevents-previous/_search"
]
def backup_function(self, orig_function):
self.original_function = orig_function
def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()):
if url not in self.exclude_paths:
if url == '/_bulk' or url == '/events/event':
self.request_counts += 1
return self.original_function(method, url, params=params, body=body)
@ -95,7 +87,7 @@ class TestWriteWithRead(ElasticsearchClientTest):
'url': 'https://mozilla.org',
'utctimestamp': '2016-08-19T16:40:57.851092+00:00'}
self.saved_alert = self.es_client.save_alert(body=self.alert)
self.es_client.flush('alerts')
self.flush('alerts')
def test_saved_type(self):
assert self.saved_alert['_type'] == 'alert'
@ -143,9 +135,9 @@ class TestSimpleWrites(ElasticsearchClientTest):
for event in events:
self.es_client.save_event(body=event)
self.es_client.flush('events')
assert mock_class.request_counts == 100
self.flush(self.event_index_name)
num_events = self.get_num_events()
assert num_events == 100
@ -162,6 +154,7 @@ class TestSimpleWrites(ElasticsearchClientTest):
}
}
self.populate_test_event(default_event)
self.flush(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
@ -181,6 +174,7 @@ class TestSimpleWrites(ElasticsearchClientTest):
}
}
self.populate_test_event(default_event)
self.flush(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
@ -197,14 +191,14 @@ class BulkTest(ElasticsearchClientTest):
self.es_client.es_connection.transport.perform_request = self.mock_class.perform_request
def teardown(self):
super(BulkTest, self).teardown()
self.es_client.finish_bulk()
super(BulkTest, self).teardown()
class TestBulkWrites(BulkTest):
def test_bulk_writing(self):
event_length = 10000
def test_bulk_writing_simple(self):
event_length = 2000
events = []
for num in range(event_length):
events.append({"key": "value" + str(num)})
@ -212,36 +206,52 @@ class TestBulkWrites(BulkTest):
assert self.mock_class.request_counts == 0
for event in events:
self.es_client.save_event(body=event, bulk=True)
self.es_client.flush('events')
assert self.mock_class.request_counts == 100
self.flush(self.event_index_name)
time.sleep(1)
# We encountered a weird bug in travis
# that would sometimes cause the number
# of requests sent to ES to fluctuate.
# As a result, we're checking within 5 requests
# from 20, to verify we are still using bulk
assert self.mock_class.request_counts <= 25 and self.mock_class.request_counts >= 15
num_events = self.get_num_events()
assert num_events == 10000
assert num_events == 2000
class TestBulkWritesWithMoreThanThreshold(BulkTest):
def test_bulk_writing(self):
event_length = 9995
def test_bulk_writing_more_threshold(self):
event_length = 1995
events = []
for num in range(event_length):
events.append({"key": "value" + str(num)})
for event in events:
self.es_client.save_object(index='events', doc_type='event', body=event, bulk=True)
self.es_client.flush('events')
assert self.mock_class.request_counts == 99
assert self.get_num_events() == 9900
time.sleep(3)
self.es_client.flush('events')
assert self.mock_class.request_counts == 100
assert self.get_num_events() == 9995
self.flush(self.event_index_name)
# We encountered a weird bug in travis
# that would sometimes cause the number
# of requests sent to ES to fluctuate.
# As a result, we're checking within 5 requests
# from 20, to verify we are still using bulk
non_flushed_request_count = self.mock_class.request_counts
assert self.mock_class.request_counts <= 25 and self.mock_class.request_counts >= 15
assert self.get_num_events() == 1900
time.sleep(5)
# All we want to check here is that during the sleep
# we purged the queue and sent the remaining events to ES
assert self.mock_class.request_counts > non_flushed_request_count
self.flush(self.event_index_name)
assert self.get_num_events() == 1995
class TestBulkWritesWithLessThanThreshold(BulkTest):
def test_bulk_writing(self):
def test_bulk_writing_less_threshold(self):
self.es_client.save_event(body={'key': 'value'}, bulk=True)
assert self.get_num_events() == 0
assert self.mock_class.request_counts == 0
@ -251,7 +261,9 @@ class TestBulkWritesWithLessThanThreshold(BulkTest):
self.es_client.save_event(body={"key": "value" + str(num)}, bulk=True)
assert self.get_num_events() == 0
time.sleep(3)
self.flush(self.event_index_name)
time.sleep(5)
assert self.get_num_events() == 6
@ -273,7 +285,7 @@ class TestWriteWithIDExists(ElasticsearchClientTest):
event['new_key'] = 'updated_value'
saved_event = self.es_client.save_event(body=event, doc_id=event_id)
assert saved_event['_id'] == event_id
self.es_client.flush('events')
self.flush(self.event_index_name)
fetched_event = self.es_client.get_event_by_id(event_id)
assert fetched_event['_source'] == event
@ -288,7 +300,7 @@ class TestGetIndices(ElasticsearchClientTest):
def test_get_indices(self):
if pytest.config.option.delete_indexes:
self.es_client.create_index('test_index')
time.sleep(0.5)
time.sleep(1)
indices = self.es_client.get_indices()
indices.sort()
assert indices == [self.alert_index_name, self.previous_event_index_name, self.event_index_name, 'test_index']
@ -400,5 +412,6 @@ class TestBulkInvalidFormatProblem(BulkTest):
self.es_client.save_object(index='events', doc_type='event', body=event, bulk=True)
self.es_client.save_object(index='events', doc_type='event', body=malformed_event, bulk=True)
self.flush(self.event_index_name)
time.sleep(5)
assert self.get_num_events() == 1

19
tests/lib/utilities/test_toUTC.py
Просмотреть файл

@ -1,11 +1,10 @@
from datetime import datetime
from datetime import datetime, date
from dateutil.parser import parse
import sys
import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../lib'))
from utilities.toUTC import toUTC
import pytest
class TestToUTC():
@ -31,7 +30,7 @@ class TestToUTC():
def test_abnormal_date_str_without_timezone(self):
result = toUTC("Jan 2 08:01:57")
self.result_is_datetime(result)
assert str(result) == '2016-01-02 08:01:57+00:00'
assert str(result) == str(date.today().year) + '-01-02 08:01:57+00:00'
def test_abnormal_date_obj_with_timezone_in_date(self):
result = toUTC(parse("2016-01-02 08:01:57+06:00"))
@ -41,28 +40,24 @@ class TestToUTC():
def test_long_epoch_without_timezone(self):
result = toUTC(1468443523000000000)
self.result_is_datetime(result)
assert str(result) == '2016-07-13 15:58:43+00:00'
assert str(result) == '2016-07-13 20:58:43+00:00'
def test_short_epoch_without_timezone(self):
result = toUTC(1468443523)
self.result_is_datetime(result)
assert str(result) == '2016-07-13 15:58:43+00:00'
assert str(result) == '2016-07-13 20:58:43+00:00'
def test_float_epoch(self):
result = toUTC(1468443523.0)
self.result_is_datetime(result)
assert str(result) == '2016-07-13 15:58:43+00:00'
assert str(result) == '2016-07-13 20:58:43+00:00'
def test_long_float_epoch(self):
result = toUTC(1.468443523e+18)
self.result_is_datetime(result)
assert str(result) == '2016-07-13 15:58:43+00:00'
assert str(result) == '2016-07-13 20:58:43+00:00'
def test_float_epoch_milliseconds(self):
result = toUTC(1.468443523e+11)
self.result_is_datetime(result)
assert str(result) == '2016-07-13 15:58:43+00:00'
def test_unparseable_suspectedDate(self):
with pytest.raises(ValueError):
toUTC("This is not a date")
assert str(result) == '2016-07-13 20:58:43+00:00'

108
tests/mq/plugins/test_fluentdSqsFixup.py
Просмотреть файл

@ -1,108 +0,0 @@
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../../../mq/plugins"))
from fluentdSqsFixup import message
import json
import pytest
class TestMessageFunctions():
def setup(self):
self.message = message()
self.msg = json.loads("""
{
"_index": "events-20151022",
"_type": "event",
"_id": "_KJo6K-dTk2MFeKK-dUKZw",
"_score": null,
"_source": {
"receivedtimestamp": "2015-10-22T04:57:33.752446+00:00",
"utctimestamp": "2015-10-22T04:57:00+00:00",
"tags": [
"nubis_events_non_prod"
],
"timestamp": "2015-10-22T04:57:00+00:00",
"mozdefhostname": "mozdef.hostname",
"summary": "Connection closed by 10.10.10.10 [preauth]",
"details": {
"ident": "sshd",
"__tag": "ec2.forward.system.secure",
"region": "us-east-1",
"pid": "24710",
"instance_id": "i-b0a7de10",
"instance_type": "t2.micro",
"host": "ip-11-11-11-11",
"sourceipgeolocation": {
"city": null,
"region_code": null,
"area_code": 0,
"time_zone": "Asia/Seoul",
"dma_code": 0,
"metro_code": null,
"country_code3": "KOR",
"latitude": 37.56999999999999,
"postal_code": null,
"longitude": 126.98000000000002,
"country_code": "KR",
"country_name": "Korea, Republic of",
"continent": "AS"
},
"time": "2015-10-22T04:57:00Z",
"message": "Connection closed by 10.10.10.10 [preauth]",
"az": "us-east-1a"
}
},
"sort": [
1445489820000
]
}
""")
self.msg2 =json.loads("""
{
"_index": "events-20151022",
"_type": "event",
"_id": "3eQPX3MMRLOnGQBuX9NQiA",
"_score": null,
"_source": {
"receivedtimestamp": "2015-10-22T05:24:41.721237+00:00",
"utctimestamp": "2015-10-22T05:24:26+00:00",
"tags": [
"nubis_events_non_prod"
],
"timestamp": "2015-10-22T05:24:26+00:00",
"mozdefhostname": "mozdef.hostname",
"summary": "INFO (transaction.py:150): Flushing 1 transaction during flush #377900",
"details": {
"ident": "dd.forwarder",
"__tag": "ec2.forward.system.syslog",
"region": "us-east-1",
"pid": "1969",
"instance_id": "i-965f8f42",
"instance_type": "m3.medium",
"host": "ip-10-162-17-177",
"time": "2015-10-22T05:24:26Z",
"message": "INFO (transaction.py:150): Flushing 1 transaction during flush #377900",
"az": "us-east-1d"
}
},
"sort": [
1445491466000
]
}
""")
def test_onMessageSSH(self):
metadata = {}
(retmessage, retmeta) = self.message.onMessage(self.msg['_source'], metadata)
assert retmessage['category'] == 'syslog'
assert retmessage['details']['program'] == 'sshd'
with pytest.raises(KeyError):
retmessage['details']['time']
def test_onMessageGeneric(self):
metadata = {}
(retmessage, retmeta) = self.message.onMessage(self.msg2['_source'], metadata)
assert retmessage['category'] == 'syslog'
assert retmessage['hostname'] == 'ip-10-162-17-177'
with pytest.raises(KeyError):
retmessage['details']['time']

4
tests/mq/test_esworker_sns_sqs.py
Просмотреть файл

@ -12,7 +12,7 @@
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../../mq"))
from esworker_sns_sqs import taskConsumer
from mq.esworker_sns_sqs import taskConsumer
sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
from utilities.dot_dict import DotDict
@ -39,7 +39,7 @@ class TestEsworkerSNSSQS(UnitTestSuite):
self.consumer = taskConsumer(mq_conn, task_queue, es_connection, options)
def search_and_verify_event(self, expected_event):
self.es_client.flush('events')
self.flush('events')
search_query = SearchQuery(minutes=5)
search_query.add_must(ExistsMatch('tags'))
results = search_query.execute(self.es_client)

4
tests/rest/test_rest_index.py
Просмотреть файл

@ -52,7 +52,7 @@ class TestKibanaDashboardsRoute(RestTestSuite):
json_dashboard_location = os.path.join(os.path.dirname(__file__), "ssh_dashboard.json")
self.es_client.save_dashboard(json_dashboard_location, "Example SSH Dashboard")
self.es_client.save_dashboard(json_dashboard_location, "Example FTP Dashboard")
self.es_client.flush('.kibana')
self.flush('.kibana')
def test_route_endpoints(self):
for route in self.routes:
@ -226,7 +226,7 @@ class TestLdapLoginsRoute(RestTestSuite):
}
self.populate_test_event(event)
self.es_client.flush('events')
self.flush('events')
def test_route_endpoints(self):
for route in self.routes:

4
tests/unit_test_suite.py
Просмотреть файл

@ -57,7 +57,6 @@ class UnitTestSuite(object):
def populate_test_event(self, event, event_type='event'):
self.es_client.save_event(body=event, doc_type=event_type)
self.es_client.flush(self.event_index_name)
def setup_elasticsearch(self):
self.es_client.create_index(self.event_index_name)
@ -75,6 +74,9 @@ class UnitTestSuite(object):
self.es_client.delete_index(self.alert_index_name, True)
self.es_client.delete_index('alerts', True)
def flush(self, index_name):
self.es_client.flush(index_name)
def random_ip(self):
return str(random.randint(1, 255)) + "." + str(random.randint(1, 255)) + "." + str(random.randint(1, 255)) + "." + str(random.randint(1, 255))