MozDef/cron/collectSSHFingerprints.py

#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
#
# Contributors:
# Jeff Bryner jbryner@mozilla.com

import logging
import pyes
import pytz
import random
import netaddr
import sys
from datetime import datetime
from datetime import timedelta
from configlib import getConfig, OptionParser
from logging.handlers import SysLogHandler
from dateutil.parser import parse
from pymongo import MongoClient
from pymongo import collection
import re
userre = re.compile(r'''Accepted publickey for (.*?) from''', re.IGNORECASE)


logger = logging.getLogger(sys.argv[0])


def loggerTimeStamp(self, record, datefmt=None):
    return toUTC(datetime.now()).isoformat()


def initLogger():
    logger.level = logging.INFO
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    formatter.formatTime = loggerTimeStamp
    if options.output == 'syslog':
        logger.addHandler(
            SysLogHandler(
                address=(options.sysloghostname, options.syslogport)))
    else:
        sh = logging.StreamHandler(sys.stderr)
        sh.setFormatter(formatter)
        logger.addHandler(sh)


def toUTC(suspectedDate, localTimeZone="US/Pacific"):
    '''make a UTC date out of almost anything'''
    utc = pytz.UTC
    objDate = None
    if type(suspectedDate) == str:
        objDate = parse(suspectedDate, fuzzy=True)
    elif type(suspectedDate) == datetime:
        objDate = suspectedDate

    if objDate.tzinfo is None:
        objDate = pytz.timezone(localTimeZone).localize(objDate)
        objDate = utc.normalize(objDate)
    else:
        objDate = utc.normalize(objDate)
    if objDate is not None:
        objDate = utc.normalize(objDate)

    return objDate


def genMeteorID():
    return('%024x' % random.randrange(16**24))


def searchForSSHKeys(es):
    begindateUTC = toUTC(datetime.now() - timedelta(minutes=5))
    enddateUTC = toUTC(datetime.now())
    qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
                                                from_value=begindateUTC,
                                                to_value=enddateUTC))
    qType = pyes.TermFilter('_type', 'event')
    qEvents = pyes.TermFilter("program", "sshd")
    q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
    q.filters.append(
        pyes.BoolFilter(must=[qType,
                              qDate,
                              qEvents,
                              pyes.QueryFilter(
                                  pyes.MatchQuery("summary",
                                                  "found matching key accepted publickey",
                                                  "boolean"))
                              ]))

    results = es.search(q, size=10000, indices='events')
    # return raw search to avoid pyes iteration bug
    return results._search_raw()


def correlateSSHKeys(esResults):
    # correlate ssh key to userid by hostname and processid
    # dict to populate hits we find
    # will be a dict with hostname:processid as they key and sshkey and username as the dict items.
    correlations = {}
    # a list for the final dicts containing keys: username and key
    uniqueCorrelations = []

    # first find the keys
    for r in esResults['hits']['hits']:
        if 'found matching' in r['_source']['summary'].lower():
            hostname = r['_source']['details']['hostname']
            processid = r['_source']['details']['processid']
            sshkey = r['_source']['summary'].split('key:')[1].strip()
            if '{0}:{1}'.format(hostname, processid) not in correlations.keys():
                correlations['{0}:{1}'.format(hostname, processid)] = dict(sshkey=sshkey)
    # find the users and match on host:processid
    for r in esResults['hits']['hits']:
        if 'accepted publickey' in r['_source']['summary'].lower():
            hostname = r['_source']['details']['hostname']
            processid = r['_source']['details']['processid']
            username = userre.split(r['_source']['summary'])[1]
            if '{0}:{1}'.format(hostname, processid) in correlations.keys() and 'username' not in correlations['{0}:{1}'.format(hostname, processid)].keys():
                correlations['{0}:{1}'.format(hostname, processid)]['username'] = username

    for c in correlations:
        if 'username' in correlations[c].keys():
            if correlations[c] not in uniqueCorrelations:
                uniqueCorrelations.append(correlations[c])
    return uniqueCorrelations


def updateMongo(mozdefdb, correlations):
    sshkeys = mozdefdb['sshkeys']
    for c in correlations:
        keyrecord = sshkeys.find_one({'sshkey': c['sshkey']})
        if keyrecord is None:
            # new record
            # generate a meteor-compatible ID
            c['_id'] = genMeteorID()
            c['utctimestamp'] = toUTC(datetime.now()).isoformat()
            logger.debug(c)
            sshkeys.insert(c)


def main():
    logger.debug('starting')
    logger.debug(options)
    try:
        es = pyes.ES(server=(list('{0}'.format(s) for s in options.esservers)))
        client = MongoClient(options.mongohost, options.mongoport)
        # use meteor db
        mozdefdb = client.meteor
        esResults = searchForSSHKeys(es)
        correlations = correlateSSHKeys(esResults)
        if len(correlations) > 0:
            updateMongo(mozdefdb, correlations)

    except Exception as e:
        logger.error("Exception %r sending health to mongo" % e)


def initConfig():
    # output our log to stdout or syslog
    options.output = getConfig('output', 'stdout', options.configfile)
    # syslog hostname
    options.sysloghostname = getConfig('sysloghostname',
                                       'localhost',
                                       options.configfile)
    # syslog port
    options.syslogport = getConfig('syslogport', 514, options.configfile)

    # elastic search server settings
    options.esservers = list(getConfig('esservers',
                                       'http://localhost:9200',
                                       options.configfile).split(','))
    options.mongohost = getConfig('mongohost', 'localhost', options.configfile)
    options.mongoport = getConfig('mongoport', 3001, options.configfile)


if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option(
        "-c",
        dest='configfile',
        default=sys.argv[0].replace('.py', '.conf'),
        help="configuration file to use")
    (options, args) = parser.parse_args()
    initConfig()
    initLogger()
    main()