correlate and track ssh fingerprints

cron/collectSSHFingerprints.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+#
+# Contributors:
+# Jeff Bryner jbryner@mozilla.com
+
+import logging
+import pyes
+import pytz
+import random
+import netaddr
+import sys
+from datetime import datetime
+from datetime import timedelta
+from configlib import getConfig, OptionParser
+from logging.handlers import SysLogHandler
+from dateutil.parser import parse
+from pymongo import MongoClient
+from pymongo import collection
+import re
+userre = re.compile(r'''Accepted publickey for (.*?) from''', re.IGNORECASE)
+
+
+logger = logging.getLogger(sys.argv[0])
+
+
+def loggerTimeStamp(self, record, datefmt=None):
+    return toUTC(datetime.now()).isoformat()
+
+
+def initLogger():
+    logger.level = logging.INFO
+    formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    formatter.formatTime = loggerTimeStamp
+    if options.output == 'syslog':
+        logger.addHandler(
+            SysLogHandler(
+                address=(options.sysloghostname, options.syslogport)))
+    else:
+        sh = logging.StreamHandler(sys.stderr)
+        sh.setFormatter(formatter)
+        logger.addHandler(sh)
+
+
+def toUTC(suspectedDate, localTimeZone="US/Pacific"):
+    '''make a UTC date out of almost anything'''
+    utc = pytz.UTC
+    objDate = None
+    if type(suspectedDate) == str:
+        objDate = parse(suspectedDate, fuzzy=True)
+    elif type(suspectedDate) == datetime:
+        objDate = suspectedDate
+
+    if objDate.tzinfo is None:
+        objDate = pytz.timezone(localTimeZone).localize(objDate)
+        objDate = utc.normalize(objDate)
+    else:
+        objDate = utc.normalize(objDate)
+    if objDate is not None:
+        objDate = utc.normalize(objDate)
+
+    return objDate
+
+
+def genMeteorID():
+    return('%024x' % random.randrange(16**24))
+
+
+def searchForSSHKeys(es):
+    begindateUTC = toUTC(datetime.now() - timedelta(minutes=5))
+    enddateUTC = toUTC(datetime.now())
+    qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
+                                                from_value=begindateUTC,
+                                                to_value=enddateUTC))
+    qType = pyes.TermFilter('_type', 'event')
+    qEvents = pyes.TermFilter("program", "sshd")
+    q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
+    q.filters.append(
+        pyes.BoolFilter(must=[qType,
+                              qDate,
+                              qEvents,
+                              pyes.QueryFilter(
+                                  pyes.MatchQuery("summary",
+                                                  "found matching key accepted publickey",
+                                                  "boolean"))
+                              ]))
+
+    results = es.search(q, size=10000, indices='events')
+    # return raw search to avoid pyes iteration bug
+    return results._search_raw()
+
+
+def correlateSSHKeys(esResults):
+    # correlate ssh key to userid by hostname and processid
+    # dict to populate hits we find
+    # will be a dict with hostname:processid as they key and sshkey and username as the dict items.
+    correlations = {}
+    # a list for the final dicts containing keys: username and key
+    uniqueCorrelations = []
+
+    # first find the keys
+    for r in esResults['hits']['hits']:
+        if 'found matching' in r['_source']['summary'].lower():
+            hostname = r['_source']['details']['hostname']
+            processid = r['_source']['details']['processid']
+            sshkey = r['_source']['summary'].split('key:')[1].strip()
+            if '{0}:{1}'.format(hostname, processid) not in correlations.keys():
+                correlations['{0}:{1}'.format(hostname, processid)] = dict(sshkey=sshkey)
+    # find the users and match on host:processid
+    for r in esResults['hits']['hits']:
+        if 'accepted publickey' in r['_source']['summary'].lower():
+            hostname = r['_source']['details']['hostname']
+            processid = r['_source']['details']['processid']
+            username = userre.split(r['_source']['summary'])[1]
+            if '{0}:{1}'.format(hostname, processid) in correlations.keys() and 'username' not in correlations['{0}:{1}'.format(hostname, processid)].keys():
+                correlations['{0}:{1}'.format(hostname, processid)]['username'] = username
+
+    for c in correlations:
+        if 'username' in correlations[c].keys():
+            if correlations[c] not in uniqueCorrelations:
+                uniqueCorrelations.append(correlations[c])
+    return uniqueCorrelations
+
+
+def updateMongo(mozdefdb, correlations):
+    sshkeys = mozdefdb['sshkeys']
+    for c in correlations:
+        keyrecord = sshkeys.find_one({'sshkey': c['sshkey']})
+        if keyrecord is None:
+            # new record
+            # generate a meteor-compatible ID
+            c['_id'] = genMeteorID()
+            c['utctimestamp'] = toUTC(datetime.now()).isoformat()
+            logger.debug(c)
+            sshkeys.insert(c)
+
+
+def main():
+    logger.debug('starting')
+    logger.debug(options)
+    try:
+        es = pyes.ES(server=(list('{0}'.format(s) for s in options.esservers)))
+        client = MongoClient(options.mongohost, options.mongoport)
+        # use meteor db
+        mozdefdb = client.meteor
+        esResults = searchForSSHKeys(es)
+        correlations = correlateSSHKeys(esResults)
+        if len(correlations) > 0:
+            updateMongo(mozdefdb, correlations)
+
+    except Exception as e:
+        logger.error("Exception %r sending health to mongo" % e)
+
+
+def initConfig():
+    # output our log to stdout or syslog
+    options.output = getConfig('output', 'stdout', options.configfile)
+    # syslog hostname
+    options.sysloghostname = getConfig('sysloghostname',
+                                       'localhost',
+                                       options.configfile)
+    # syslog port
+    options.syslogport = getConfig('syslogport', 514, options.configfile)
+
+    # elastic search server settings
+    options.esservers = list(getConfig('esservers',
+                                       'http://localhost:9200',
+                                       options.configfile).split(','))
+    options.mongohost = getConfig('mongohost', 'localhost', options.configfile)
+    options.mongoport = getConfig('mongoport', 3001, options.configfile)
+
+
+if __name__ == '__main__':
+    parser = OptionParser()
+    parser.add_option(
+        "-c",
+        dest='configfile',
+        default=sys.argv[0].replace('.py', '.conf'),
+        help="configuration file to use")
+    (options, args) = parser.parse_args()
+    initConfig()
+    initLogger()
+    main()