зеркало из https://github.com/mozilla/MozDef.git
add veris visualization for incident stats, closes #64
This commit is contained in:
Jeff Bryner
Родитель
b8975f28c0
Коммит
e2d7cb049d
|
|
@ -0,0 +1,18 @@
|
|||
<!--
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
Copyright (c) 2014 Mozilla Corporation
|
||||
|
||||
Contributors:
|
||||
Jeff Bryner jbryner@mozilla.com
|
||||
-->
|
||||
|
||||
<template name="incidentsveris">
|
||||
<style>
|
||||
|
||||
</style>
|
||||
<div class="veris-wrapper" id="veris-wrapper">
|
||||
|
||||
</div>
|
||||
</template>
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
/*
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
Copyright (c) 2014 Mozilla Corporation
|
||||
|
||||
Contributors:
|
||||
Jeff Bryner jbryner@mozilla.com
|
||||
*/
|
||||
|
||||
if (Meteor.isClient) {
|
||||
|
||||
Template.incidentsveris.rendered = function () {
|
||||
var ndx = crossfilter();
|
||||
var container=document.getElementById('veris-wrapper')
|
||||
var margin = {top: 20, right: 20, bottom: 20, left: 20},
|
||||
width = window.innerWidth - margin.left - margin.right,
|
||||
height = window.innerHeight - margin.top - margin.bottom,
|
||||
minRadius=3,
|
||||
maxRadius=40,
|
||||
clipPadding=4;
|
||||
|
||||
var fill = d3.scale.category10();
|
||||
|
||||
var nodes = [],
|
||||
links = [],
|
||||
foci = [{x: width/2, y: height/2}];
|
||||
|
||||
var svg = d3.select(".veris-wrapper").append("svg")
|
||||
.attr("width", width)
|
||||
.attr("height", height)
|
||||
.append("g")
|
||||
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
var force = d3.layout.force()
|
||||
.nodes(nodes)
|
||||
.links([])
|
||||
.gravity(.1)
|
||||
.charge(-1000)
|
||||
.size([width, height])
|
||||
.on("tick", tick);
|
||||
|
||||
var node = svg.selectAll(".node"),
|
||||
link = svg.selectAll(".link");
|
||||
|
||||
//setup the radius scale
|
||||
var r = d3.scale.sqrt()
|
||||
.range([0, maxRadius]);
|
||||
|
||||
container.style.cursor='wait'
|
||||
d3.json(getSetting('rootAPI') + '/veris' , function(error, jsondata) {
|
||||
//console.log(jsondata)
|
||||
//jsondata.forEach(function(d){
|
||||
// console.log(d);
|
||||
//});
|
||||
ndx.add(jsondata);
|
||||
container.style.cursor='auto';
|
||||
if ( ndx.size() >0 ){
|
||||
var all = ndx.groupAll();
|
||||
var tagsDim = ndx.dimension(function(d) {return d.tags;});
|
||||
var phaseDim = ndx.dimension(function(d) {return d.phase});
|
||||
}
|
||||
r.domain([0, d3.max(tagsDim.group().all(), function(d) { return d.value; })]);
|
||||
tagsDim.group().all().forEach(function(d){
|
||||
d.r = r(d.value);
|
||||
d.cr = Math.max(minRadius, d.r);
|
||||
nodes.push(d);
|
||||
});
|
||||
start();
|
||||
});
|
||||
|
||||
container.style.cursor='auto';
|
||||
|
||||
function start() {
|
||||
node = node.data(force.nodes(), function(d) { return d.key;});
|
||||
//make a node for each entry
|
||||
node.enter()
|
||||
.append("a")
|
||||
.attr("class", function(d) { return "node " + d.key; })
|
||||
.attr("class", "node")
|
||||
.call(force.drag);
|
||||
|
||||
// setp the node body:
|
||||
var nodeBody = node.append("g")
|
||||
.attr("class", "g-success");
|
||||
|
||||
nodeBody.append("clipPath")
|
||||
.attr("id", function(d) { return "g-clip-success-" + d.key; })
|
||||
.append("rect");
|
||||
|
||||
nodeBody.append("circle")
|
||||
.attr("r", function(d) {return d.cr;})
|
||||
.attr("class","successcircle");
|
||||
|
||||
node.append("svg:text")
|
||||
.attr("x", "-3em")
|
||||
.attr("y", ".3em")
|
||||
.attr("class","textlabel")
|
||||
.text(function(d) { return d.key; });
|
||||
|
||||
//make a mouse over
|
||||
node.append("title")
|
||||
.text(function(d) { return d.key + ": " + d.value});
|
||||
|
||||
//size circle clips
|
||||
node.selectAll("rect")
|
||||
.attr("y", function(d) { return -d.r - clipPadding; })
|
||||
.attr("height", function(d) { return 2 * d.r + 2 * clipPadding; });
|
||||
node.exit().remove();
|
||||
force.start();
|
||||
}
|
||||
|
||||
function tick(e) {
|
||||
var k = .1 * e.alpha;
|
||||
// Push nodes toward their focus.
|
||||
nodes.forEach(function(o, i) {
|
||||
o.y += (foci[0].y - o.y) * k;
|
||||
o.x += (foci[0].x - o.x) * k;
|
||||
});
|
||||
|
||||
svg.selectAll(".node")
|
||||
.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
|
||||
}
|
||||
};
|
||||
|
||||
Template.incidentsveris.destroyed = function () {
|
||||
debugLog('destroyed');
|
||||
};
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ Anthony Verez averez@mozilla.com
|
|||
<li><a href="/Incidents">Incidents</a>
|
||||
<ul>
|
||||
<li class="first"><a href="/incidents/new/">new incident</a></li>
|
||||
<li class="last"><a href="/incidents/veris/">veris stats</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="first"><a href="#">Visualizations</a>
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ Router.map(function () {
|
|||
layoutTemplate: 'layout'
|
||||
});
|
||||
|
||||
this.route('incidentsveris', {
|
||||
path: '/incidents/veris',
|
||||
template: 'incidentsveris',
|
||||
layoutTemplate: 'layout'
|
||||
});
|
||||
|
||||
this.route('incidentedit', {
|
||||
path: '/incident/:_id/edit',
|
||||
waitOn: function() {
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ from datetime import datetime
|
|||
from datetime import timedelta
|
||||
from dateutil.parser import parse
|
||||
from ipwhois import IPWhois
|
||||
from bson.son import SON
|
||||
from pymongo import MongoClient
|
||||
from bson import json_util
|
||||
|
||||
options = None
|
||||
dbcursor = None
|
||||
|
|
@ -74,15 +77,15 @@ def index():
|
|||
return(esLdapResults())
|
||||
|
||||
|
||||
@route('/alerts')
|
||||
@route('/alerts/')
|
||||
@route('/veris')
|
||||
@route('/veris/')
|
||||
@enable_cors
|
||||
def index():
|
||||
if request.body:
|
||||
request.body.read()
|
||||
request.body.close()
|
||||
response.content_type = "application/json"
|
||||
return(esAlertsSummary())
|
||||
return(verisSummary())
|
||||
|
||||
|
||||
@route('/kibanadashboards')
|
||||
|
|
@ -181,12 +184,10 @@ def index():
|
|||
return
|
||||
|
||||
|
||||
def toUTC(suspectedDate, localTimeZone=None):
|
||||
def toUTC(suspectedDate, localTimeZone="US/Pacific"):
|
||||
'''make a UTC date out of almost anything'''
|
||||
utc = pytz.UTC
|
||||
objDate = None
|
||||
if localTimeZone is None:
|
||||
localTimeZone=options.defaulttimezone
|
||||
if type(suspectedDate) == str:
|
||||
objDate = parse(suspectedDate, fuzzy=True)
|
||||
elif type(suspectedDate) == datetime:
|
||||
|
|
@ -217,36 +218,6 @@ def isIPv4(ip):
|
|||
except:
|
||||
return False
|
||||
|
||||
|
||||
def esAlertsSummary(begindateUTC=None, enddateUTC=None):
|
||||
resultsList = list()
|
||||
if begindateUTC is None:
|
||||
begindateUTC = datetime.now() - timedelta(hours=12)
|
||||
begindateUTC = toUTC(begindateUTC)
|
||||
if enddateUTC is None:
|
||||
enddateUTC = datetime.now()
|
||||
enddateUTC = toUTC(enddateUTC)
|
||||
try:
|
||||
|
||||
#q=S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert').filter(utctimestamp__range=[begindateUTC.isoformat(),enddateUTC.isoformat()])
|
||||
#f=q.facet_raw(alerttype={"terms" : {"script_field" : "_source.type","size" : 500}})
|
||||
|
||||
#get all alerts
|
||||
#q= S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert')
|
||||
q= S().es(urls=list('{0}'.format(s) for s in options.esservers)).query(_type='alert')
|
||||
#create a facet field using the entire 'category' field (not the sub terms) and filter it by date.
|
||||
f=q.facet_raw(\
|
||||
alerttype={"terms" : {"script_field" : "_source.category"},\
|
||||
"facet_filter":{'range': {'utctimestamp': \
|
||||
{'gte': begindateUTC.isoformat(), 'lte': enddateUTC.isoformat()}}}\
|
||||
|
||||
})
|
||||
return(json.dumps(f.facet_counts()['alerttype']))
|
||||
|
||||
except Exception as e:
|
||||
sys.stderr.write('%r' % e)
|
||||
|
||||
|
||||
def esLdapResults(begindateUTC=None, enddateUTC=None):
|
||||
resultsList = list()
|
||||
if begindateUTC is None:
|
||||
|
|
@ -269,8 +240,8 @@ def esLdapResults(begindateUTC=None, enddateUTC=None):
|
|||
q2.facet.add_term_facet('details.dn', size=20)
|
||||
results = es.search(q2, indices='events')
|
||||
|
||||
stoplist = ('o', 'example', 'dc', 'com', 'mozilla.com',
|
||||
'example.com', 'org')
|
||||
stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com',
|
||||
'mozillafoundation.org', 'org')
|
||||
for t in results.facets['details.dn'].terms:
|
||||
if t['term'] in stoplist:
|
||||
continue
|
||||
|
|
@ -425,10 +396,42 @@ def checkBlockIPService():
|
|||
sys.stderr.write('Failed to connect to the Banhammer DB\n')
|
||||
|
||||
|
||||
def verisSummary(verisRegex=None):
|
||||
try:
|
||||
# aggregate the veris tags from the incidents collection and return as json
|
||||
client = MongoClient(options.mongohost, options.mongoport)
|
||||
# use meteor db
|
||||
incidents= client.meteor['incidents']
|
||||
#iveris=incidents.aggregate([
|
||||
#{"$match":{"tags":{"$exists":True}}},
|
||||
#{"$unwind" : "$tags" },
|
||||
#{"$match":{"tags":{"$regex":''}}}, #regex for tag querying
|
||||
#{"$group": {"_id": "$tags", "hitcount": {"$sum": 1}}}, # count by tag
|
||||
#{"$sort": SON([("hitcount", -1), ("_id", -1)])}, #sort
|
||||
#])
|
||||
|
||||
iveris=incidents.aggregate([
|
||||
|
||||
{"$match":{"tags":{"$exists":True}}},
|
||||
{"$unwind" : "$tags" },
|
||||
{"$match":{"tags":{"$regex":''}}}, #regex for tag querying
|
||||
{ "$project" : { "dateOpened" : 1 ,
|
||||
"tags" : 1 ,
|
||||
"phase": 1,
|
||||
"_id": 0
|
||||
} }
|
||||
])
|
||||
if 'ok' in iveris.keys() and 'result' in iveris.keys():
|
||||
return json.dumps(iveris['result'], default=json_util.default)
|
||||
else:
|
||||
return json.dumps(list())
|
||||
except Exception as e:
|
||||
sys.stderr.write('Exception while aggregating veris summary: {0}\n'.format(e))
|
||||
|
||||
def initConfig():
|
||||
#change this to your default zone for when it's not specified
|
||||
options.defaulttimezone = getConfig('defaulttimezone',
|
||||
'UTC',
|
||||
options.defaultTimeZone = getConfig('defaulttimezone',
|
||||
'US/Pacific',
|
||||
options.configfile)
|
||||
options.esservers = list(getConfig('esservers',
|
||||
'http://localhost:9200',
|
||||
|
|
@ -462,7 +465,9 @@ def initConfig():
|
|||
options.cifhosturl = getConfig('cifhosturl',
|
||||
'http://localhost/',
|
||||
options.configfile)
|
||||
|
||||
# mongo connectivity options
|
||||
options.mongohost = getConfig('mongohost', 'localhost', options.configfile)
|
||||
options.mongoport = getConfig('mongoport', 3001, options.configfile)
|
||||
|
||||
# check any service you'd like at startup rather than waiting
|
||||
# for a client request.
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче