From e2d7cb049d273b812e7f345b731893741a9ee6a3 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Tue, 14 Oct 2014 17:10:40 -0700 Subject: [PATCH] add veris visualization for incident stats, closes #64 --- meteor/app/client/incidentsveris.html | 18 ++++ meteor/app/client/incidentsveris.js | 129 ++++++++++++++++++++++++++ meteor/app/client/menu.html | 1 + meteor/app/router/router.js | 6 ++ rest/index.py | 87 +++++++++-------- 5 files changed, 200 insertions(+), 41 deletions(-) create mode 100644 meteor/app/client/incidentsveris.html create mode 100644 meteor/app/client/incidentsveris.js diff --git a/meteor/app/client/incidentsveris.html b/meteor/app/client/incidentsveris.html new file mode 100644 index 00000000..ecd55587 --- /dev/null +++ b/meteor/app/client/incidentsveris.html @@ -0,0 +1,18 @@ + + + diff --git a/meteor/app/client/incidentsveris.js b/meteor/app/client/incidentsveris.js new file mode 100644 index 00000000..dd6183da --- /dev/null +++ b/meteor/app/client/incidentsveris.js @@ -0,0 +1,129 @@ + /* +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. +Copyright (c) 2014 Mozilla Corporation + +Contributors: +Jeff Bryner jbryner@mozilla.com + */ + +if (Meteor.isClient) { + + Template.incidentsveris.rendered = function () { + var ndx = crossfilter(); + var container=document.getElementById('veris-wrapper') + var margin = {top: 20, right: 20, bottom: 20, left: 20}, + width = window.innerWidth - margin.left - margin.right, + height = window.innerHeight - margin.top - margin.bottom, + minRadius=3, + maxRadius=40, + clipPadding=4; + + var fill = d3.scale.category10(); + + var nodes = [], + links = [], + foci = [{x: width/2, y: height/2}]; + + var svg = d3.select(".veris-wrapper").append("svg") + .attr("width", width) + .attr("height", height) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + var force = d3.layout.force() + .nodes(nodes) + .links([]) + .gravity(.1) + .charge(-1000) + .size([width, height]) + .on("tick", tick); + + var node = svg.selectAll(".node"), + link = svg.selectAll(".link"); + + //setup the radius scale + var r = d3.scale.sqrt() + .range([0, maxRadius]); + + container.style.cursor='wait' + d3.json(getSetting('rootAPI') + '/veris' , function(error, jsondata) { + //console.log(jsondata) + //jsondata.forEach(function(d){ + // console.log(d); + //}); + ndx.add(jsondata); + container.style.cursor='auto'; + if ( ndx.size() >0 ){ + var all = ndx.groupAll(); + var tagsDim = ndx.dimension(function(d) {return d.tags;}); + var phaseDim = ndx.dimension(function(d) {return d.phase}); + } + r.domain([0, d3.max(tagsDim.group().all(), function(d) { return d.value; })]); + tagsDim.group().all().forEach(function(d){ + d.r = r(d.value); + d.cr = Math.max(minRadius, d.r); + nodes.push(d); + }); + start(); + }); + + container.style.cursor='auto'; + + function start() { + node = node.data(force.nodes(), function(d) { return d.key;}); + //make a node for each entry + node.enter() + .append("a") + .attr("class", function(d) { return "node " + d.key; }) + .attr("class", "node") + .call(force.drag); + + // setp the node body: + var nodeBody = node.append("g") + .attr("class", "g-success"); + + nodeBody.append("clipPath") + .attr("id", function(d) { return "g-clip-success-" + d.key; }) + .append("rect"); + + nodeBody.append("circle") + .attr("r", function(d) {return d.cr;}) + .attr("class","successcircle"); + + node.append("svg:text") + .attr("x", "-3em") + .attr("y", ".3em") + .attr("class","textlabel") + .text(function(d) { return d.key; }); + + //make a mouse over + node.append("title") + .text(function(d) { return d.key + ": " + d.value}); + + //size circle clips + node.selectAll("rect") + .attr("y", function(d) { return -d.r - clipPadding; }) + .attr("height", function(d) { return 2 * d.r + 2 * clipPadding; }); + node.exit().remove(); + force.start(); + } + + function tick(e) { + var k = .1 * e.alpha; + // Push nodes toward their focus. + nodes.forEach(function(o, i) { + o.y += (foci[0].y - o.y) * k; + o.x += (foci[0].x - o.x) * k; + }); + + svg.selectAll(".node") + .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; }); + } + }; + + Template.incidentsveris.destroyed = function () { + debugLog('destroyed'); + }; +} \ No newline at end of file diff --git a/meteor/app/client/menu.html b/meteor/app/client/menu.html index 5693e546..3bae7562 100644 --- a/meteor/app/client/menu.html +++ b/meteor/app/client/menu.html @@ -29,6 +29,7 @@ Anthony Verez averez@mozilla.com
  • Incidents
  • Visualizations diff --git a/meteor/app/router/router.js b/meteor/app/router/router.js index 9aeb4336..09f023d3 100644 --- a/meteor/app/router/router.js +++ b/meteor/app/router/router.js @@ -39,6 +39,12 @@ Router.map(function () { layoutTemplate: 'layout' }); + this.route('incidentsveris', { + path: '/incidents/veris', + template: 'incidentsveris', + layoutTemplate: 'layout' + }); + this.route('incidentedit', { path: '/incident/:_id/edit', waitOn: function() { diff --git a/rest/index.py b/rest/index.py index a7fc5fc2..f52a44f3 100644 --- a/rest/index.py +++ b/rest/index.py @@ -23,6 +23,9 @@ from datetime import datetime from datetime import timedelta from dateutil.parser import parse from ipwhois import IPWhois +from bson.son import SON +from pymongo import MongoClient +from bson import json_util options = None dbcursor = None @@ -74,15 +77,15 @@ def index(): return(esLdapResults()) -@route('/alerts') -@route('/alerts/') +@route('/veris') +@route('/veris/') @enable_cors def index(): if request.body: request.body.read() request.body.close() response.content_type = "application/json" - return(esAlertsSummary()) + return(verisSummary()) @route('/kibanadashboards') @@ -181,12 +184,10 @@ def index(): return -def toUTC(suspectedDate, localTimeZone=None): +def toUTC(suspectedDate, localTimeZone="US/Pacific"): '''make a UTC date out of almost anything''' utc = pytz.UTC objDate = None - if localTimeZone is None: - localTimeZone=options.defaulttimezone if type(suspectedDate) == str: objDate = parse(suspectedDate, fuzzy=True) elif type(suspectedDate) == datetime: @@ -217,36 +218,6 @@ def isIPv4(ip): except: return False - -def esAlertsSummary(begindateUTC=None, enddateUTC=None): - resultsList = list() - if begindateUTC is None: - begindateUTC = datetime.now() - timedelta(hours=12) - begindateUTC = toUTC(begindateUTC) - if enddateUTC is None: - enddateUTC = datetime.now() - enddateUTC = toUTC(enddateUTC) - try: - - #q=S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert').filter(utctimestamp__range=[begindateUTC.isoformat(),enddateUTC.isoformat()]) - #f=q.facet_raw(alerttype={"terms" : {"script_field" : "_source.type","size" : 500}}) - - #get all alerts - #q= S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert') - q= S().es(urls=list('{0}'.format(s) for s in options.esservers)).query(_type='alert') - #create a facet field using the entire 'category' field (not the sub terms) and filter it by date. - f=q.facet_raw(\ - alerttype={"terms" : {"script_field" : "_source.category"},\ - "facet_filter":{'range': {'utctimestamp': \ - {'gte': begindateUTC.isoformat(), 'lte': enddateUTC.isoformat()}}}\ - - }) - return(json.dumps(f.facet_counts()['alerttype'])) - - except Exception as e: - sys.stderr.write('%r' % e) - - def esLdapResults(begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: @@ -269,8 +240,8 @@ def esLdapResults(begindateUTC=None, enddateUTC=None): q2.facet.add_term_facet('details.dn', size=20) results = es.search(q2, indices='events') - stoplist = ('o', 'example', 'dc', 'com', 'mozilla.com', - 'example.com', 'org') + stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com', + 'mozillafoundation.org', 'org') for t in results.facets['details.dn'].terms: if t['term'] in stoplist: continue @@ -425,10 +396,42 @@ def checkBlockIPService(): sys.stderr.write('Failed to connect to the Banhammer DB\n') +def verisSummary(verisRegex=None): + try: + # aggregate the veris tags from the incidents collection and return as json + client = MongoClient(options.mongohost, options.mongoport) + # use meteor db + incidents= client.meteor['incidents'] + #iveris=incidents.aggregate([ + #{"$match":{"tags":{"$exists":True}}}, + #{"$unwind" : "$tags" }, + #{"$match":{"tags":{"$regex":''}}}, #regex for tag querying + #{"$group": {"_id": "$tags", "hitcount": {"$sum": 1}}}, # count by tag + #{"$sort": SON([("hitcount", -1), ("_id", -1)])}, #sort + #]) + + iveris=incidents.aggregate([ + + {"$match":{"tags":{"$exists":True}}}, + {"$unwind" : "$tags" }, + {"$match":{"tags":{"$regex":''}}}, #regex for tag querying + { "$project" : { "dateOpened" : 1 , + "tags" : 1 , + "phase": 1, + "_id": 0 + } } + ]) + if 'ok' in iveris.keys() and 'result' in iveris.keys(): + return json.dumps(iveris['result'], default=json_util.default) + else: + return json.dumps(list()) + except Exception as e: + sys.stderr.write('Exception while aggregating veris summary: {0}\n'.format(e)) + def initConfig(): #change this to your default zone for when it's not specified - options.defaulttimezone = getConfig('defaulttimezone', - 'UTC', + options.defaultTimeZone = getConfig('defaulttimezone', + 'US/Pacific', options.configfile) options.esservers = list(getConfig('esservers', 'http://localhost:9200', @@ -462,7 +465,9 @@ def initConfig(): options.cifhosturl = getConfig('cifhosturl', 'http://localhost/', options.configfile) - + # mongo connectivity options + options.mongohost = getConfig('mongohost', 'localhost', options.configfile) + options.mongoport = getConfig('mongoport', 3001, options.configfile) # check any service you'd like at startup rather than waiting # for a client request.