зеркало из https://github.com/mozilla/MozDef.git
netantho-105-ttl: refactor setupIndexTemplates.py and es-docs/inject.py to use a common module
This commit is contained in:
Anthony Verez
Родитель
2a9bf02f44
Коммит
1e2ec563ba
|
|
@ -1,2 +1,4 @@
|
|||
.DS_Store
|
||||
meteor/packages
|
||||
*.pyc
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#
|
||||
# Contributors:
|
||||
# Jeff Bryner jbryner@mozilla.com
|
||||
# Anthony Verez averez@mozilla.com
|
||||
|
||||
# Use this to setup the index templates for mozdef
|
||||
# You only need to run it once, it will setup the templates
|
||||
|
|
@ -14,192 +15,11 @@
|
|||
|
||||
import requests
|
||||
import sys
|
||||
import os
|
||||
from configlib import getConfig, OptionParser
|
||||
|
||||
|
||||
def esPutTemplates():
|
||||
eventstemplate = r'''
|
||||
{
|
||||
"template" : "events*",
|
||||
"mappings" : {
|
||||
"event" : {
|
||||
"_ttl" : { "enabled" : true },
|
||||
"properties" : {
|
||||
"category" : {
|
||||
"index" : "not_analyzed",
|
||||
"type" : "string"
|
||||
},
|
||||
"details" : {
|
||||
"properties" : {
|
||||
"destinationipaddress" : {
|
||||
"type" : "ip"
|
||||
},
|
||||
"destinationport" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"email" : {
|
||||
"index" : "not_analyzed",
|
||||
"type" : "string"
|
||||
},
|
||||
"dn" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"hostname" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"msg" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"note" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"processid" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"program" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"protocol" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"result" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"source" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"sourceipaddress" : {
|
||||
"type" : "ip"
|
||||
},
|
||||
"sourceport" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"srcip" : {
|
||||
"type" : "ip"
|
||||
},
|
||||
"sub" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"success" : {
|
||||
"type" : "boolean"
|
||||
},
|
||||
"timestamp" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"ts" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"uid" : {
|
||||
"type" : "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"eventsource" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"hostname" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"processid" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"receivedtimestatmp" : {
|
||||
"type" : "date",
|
||||
"format" : "dateOptionalTime"
|
||||
},
|
||||
"severity" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"summary" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"tags" : {
|
||||
"index" : "not_analyzed",
|
||||
"type" : "string"
|
||||
},
|
||||
"timestamp" : {
|
||||
"type" : "date",
|
||||
"format" : "dateOptionalTime"
|
||||
},
|
||||
"utctimestamp" : {
|
||||
"type" : "date",
|
||||
"format" : "dateOptionalTime"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
url = '{0}/_template/eventstemplate'.format(options.esservers[0])
|
||||
r = requests.put(url=url, data=eventstemplate)
|
||||
if r.status_code == 200:
|
||||
print('Successfully put events template')
|
||||
else:
|
||||
print('Problem putting events template %r' % r)
|
||||
|
||||
alertstemplate = r'''
|
||||
{
|
||||
"template" : "alerts*",
|
||||
"mappings" : {
|
||||
"alert" : {
|
||||
"properties" : {
|
||||
"timestamp" : {
|
||||
"format" : "dateOptionalTime",
|
||||
"type" : "date"
|
||||
},
|
||||
"tags" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"summary" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"receivedtimestatmp" : {
|
||||
"format" : "dateOptionalTime",
|
||||
"type" : "date"
|
||||
},
|
||||
"category" : {
|
||||
"index" : "not_analyzed",
|
||||
"type" : "string"
|
||||
},
|
||||
"events" : {
|
||||
"properties" : {
|
||||
"id" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"index" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"type" : {
|
||||
"type" : "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"eventsource" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"hostname" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"severity" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"utctimestamp" : {
|
||||
"format" : "dateOptionalTime",
|
||||
"type" : "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
url = '{0}/_template/alertstemplate'.format(options.esservers[0])
|
||||
r = requests.put(url=url, data=alertstemplate)
|
||||
if r.status_code == 200:
|
||||
print('Successfully put alerts template')
|
||||
else:
|
||||
print('Problem putting alerts template %r' % r)
|
||||
|
||||
sys.path.insert(1, os.path.join(sys.path[0], '..'))
|
||||
from utils import es as es_module
|
||||
|
||||
def initConfig():
|
||||
options.esservers = list(getConfig(
|
||||
|
|
@ -216,4 +36,6 @@ if __name__ == '__main__':
|
|||
help="configuration file to use")
|
||||
(options, args) = parser.parse_args()
|
||||
initConfig()
|
||||
esPutTemplates()
|
||||
es = es_module.Elasticsearch(options.esservers[0])
|
||||
es.setupIndexTemplate('eventstemplate', '../examples/es-docs/events_template.json')
|
||||
es.setupIndexTemplate('alertstemplate', '../examples/es-docs/alerts_template.json')
|
||||
|
|
|
|||
|
|
@ -8,120 +8,13 @@
|
|||
# Contributors:
|
||||
# Anthony Verez averez@mozilla.com
|
||||
|
||||
import json
|
||||
import pyes
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import inspect
|
||||
from configlib import getConfig, OptionParser
|
||||
|
||||
|
||||
def deleteIndices(conn):
|
||||
print('Deleting alerts and events indices...')
|
||||
try:
|
||||
conn.indices.delete_index("alerts")
|
||||
conn.indices.delete_index("events")
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def setupTemplates(options):
|
||||
f = open('events_template.json')
|
||||
eventsTemplate = f.read()
|
||||
url = '{0}/_template/eventstemplate'.format(options.esservers[0])
|
||||
r = requests.put(url=url, data=eventsTemplate)
|
||||
if r.status_code == 200:
|
||||
print('Successfully put events template')
|
||||
else:
|
||||
print('Problem putting events template %r' % r)
|
||||
f.close()
|
||||
|
||||
f = open('alerts_template.json')
|
||||
alertsTemplate = f.read()
|
||||
url = '{0}/_template/alertstemplate'.format(options.esservers[0])
|
||||
r = requests.put(url=url, data=alertsTemplate)
|
||||
if r.status_code == 200:
|
||||
print('Successfully put alerts template')
|
||||
else:
|
||||
print('Problem putting alerts template %r' % r)
|
||||
f.close()
|
||||
|
||||
|
||||
def createIndices(conn):
|
||||
print('Creating alerts and events indices...')
|
||||
try:
|
||||
conn.indices.create_index("alerts")
|
||||
conn.indices.delete_index("events")
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def loadDocs(conn):
|
||||
print('Loading sample docs...')
|
||||
f = open('alerts.json')
|
||||
data = json.load(f)
|
||||
for l in data:
|
||||
conn.index(l, "alerts", "alert")
|
||||
f.close()
|
||||
|
||||
f = open('events-auditd.json')
|
||||
data = json.load(f)
|
||||
for l in data:
|
||||
conn.index(l, "events", "auditd")
|
||||
f.close()
|
||||
|
||||
f = open('events-event.json')
|
||||
data = json.load(f)
|
||||
for l in data:
|
||||
conn.index(l, "events", "event")
|
||||
f.close()
|
||||
|
||||
f = open('events-cloudtrail.json')
|
||||
data = json.load(f)
|
||||
for l in data:
|
||||
conn.index(l, "events", "cloudtrail")
|
||||
f.close()
|
||||
|
||||
|
||||
def loadDashboards(conn):
|
||||
print('Loading sample dashboards...')
|
||||
|
||||
f = open('events-kibana.json')
|
||||
dashboardjson = json.load(f)
|
||||
url = '{0}/kibana-int/dashboard/{1}'.format(
|
||||
options.esservers[0],
|
||||
dashboardjson['title'])
|
||||
dashboarddata = {
|
||||
"user": "guest",
|
||||
"group": "guest",
|
||||
"title": dashboardjson['title'],
|
||||
"dashboard": json.dumps(dashboardjson)
|
||||
}
|
||||
r = requests.put(url=url, data=json.dumps(dashboarddata))
|
||||
if r.status_code < 220:
|
||||
print('Successfully put events kibana dashboard')
|
||||
else:
|
||||
print r.json()
|
||||
print('Problem putting events kibana dashboard %r' % r)
|
||||
f.close()
|
||||
|
||||
f = open('alerts-kibana.json')
|
||||
dashboardjson = json.load(f)
|
||||
url = '{0}/kibana-int/dashboard/{1}'.format(
|
||||
options.esservers[0],
|
||||
dashboardjson['title'])
|
||||
dashboarddata = {
|
||||
"user": "guest",
|
||||
"group": "guest",
|
||||
"title": dashboardjson['title'],
|
||||
"dashboard": json.dumps(dashboardjson)
|
||||
}
|
||||
r = requests.put(url=url, data=json.dumps(dashboarddata))
|
||||
if r.status_code < 220:
|
||||
print('Successfully put alerts kibana dashboard')
|
||||
else:
|
||||
print r.json()
|
||||
print('Problem putting alerts kibana dashboard %r' % r)
|
||||
f.close()
|
||||
sys.path.insert(1, os.path.join(sys.path[0], '../..'))
|
||||
from utils import es as es_module
|
||||
|
||||
|
||||
def initConfig():
|
||||
|
|
@ -140,9 +33,17 @@ if __name__ == '__main__':
|
|||
help="configuration file to use")
|
||||
(options, args) = parser.parse_args()
|
||||
initConfig()
|
||||
conn = pyes.ES(options.esservers[0])
|
||||
deleteIndices(conn)
|
||||
setupTemplates(options)
|
||||
createIndices(conn)
|
||||
loadDocs(conn)
|
||||
loadDashboards(conn)
|
||||
es = es_module.Elasticsearch(options.esservers[0])
|
||||
es.deleteIndex('events')
|
||||
es.deleteIndex('alerts')
|
||||
es.deleteIndex('kibana-int')
|
||||
es.setupIndexTemplate('eventstemplate', 'events_template.json')
|
||||
es.setupIndexTemplate('alertstemplate', 'alerts_template.json')
|
||||
es.createIndex('alerts')
|
||||
es.createIndex('events')
|
||||
es.loadDocs('alerts', 'alert', 'alerts.json')
|
||||
es.loadDocs('events', 'auditd', 'events-auditd.json')
|
||||
es.loadDocs('events', 'event', 'events-event.json')
|
||||
es.loadDocs('events', 'cloudtrail', 'events-cloudtrail.json')
|
||||
es.loadDashboard('events', 'events-kibana.json')
|
||||
es.loadDashboard('alerts', 'alerts-kibana.json')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,78 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# Copyright (c) 2014 Mozilla Corporation
|
||||
#
|
||||
# Contributors:
|
||||
# Anthony Verez averez@mozilla.com
|
||||
|
||||
import pyes
|
||||
import json
|
||||
import requests
|
||||
|
||||
|
||||
class Elasticsearch(object):
|
||||
def __init__(self, esserver):
|
||||
"""
|
||||
Class used for ES features not supported by pyes
|
||||
and for shortcuts when using pyes
|
||||
"""
|
||||
self.esserver = esserver
|
||||
self.conn = pyes.ES(esserver)
|
||||
|
||||
def deleteIndex(self, index):
|
||||
print('Deleting %s index...' % index)
|
||||
try:
|
||||
self.conn.indices.delete_index(index)
|
||||
except:
|
||||
pass
|
||||
|
||||
def setupIndexTemplate(self, template_name, template_file):
|
||||
f = open(template_file)
|
||||
templateData = f.read()
|
||||
url = '{0}/_template/{1}'.format(self.esserver, template_name)
|
||||
r = requests.put(url=url, data=templateData)
|
||||
if r.status_code == 200:
|
||||
print('Successfully put %s template' % template_name)
|
||||
else:
|
||||
print('Problem putting %s template %r' % (template_name, r))
|
||||
f.close()
|
||||
|
||||
def createIndex(self, index):
|
||||
print('Creating %s index...' % index)
|
||||
try:
|
||||
self.conn.indices.create_index(index)
|
||||
except:
|
||||
pass
|
||||
|
||||
def loadDocs(self, index, docs_type, docs_file):
|
||||
print('Loading docs from %s...' % docs_file)
|
||||
f = open(docs_file)
|
||||
data = json.load(f)
|
||||
for l in data:
|
||||
self.conn.index(l, index, docs_type)
|
||||
f.close()
|
||||
|
||||
def loadDashboard(self, dash_name, dash_file):
|
||||
print('Loading %s dashboard...' % dash_name)
|
||||
|
||||
f = open(dash_file)
|
||||
dashboardjson = json.load(f)
|
||||
url = '{0}/kibana-int/dashboard/{1}'.format(
|
||||
self.esserver,
|
||||
dashboardjson['title'])
|
||||
dashboarddata = {
|
||||
"user": "guest",
|
||||
"group": "guest",
|
||||
"title": dashboardjson['title'],
|
||||
"dashboard": json.dumps(dashboardjson)
|
||||
}
|
||||
r = requests.put(url=url, data=json.dumps(dashboarddata))
|
||||
if r.status_code < 220:
|
||||
print('Successfully put %s kibana dashboard' % dash_name)
|
||||
else:
|
||||
print(r.json())
|
||||
print('Problem putting %s kibana dashboard %r' % (dash_name, r))
|
||||
f.close()
|
||||
Загрузка…
Ссылка в новой задаче