From 3bd574571cb8f8786825056f5954b04c558c10ab Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Mon, 3 Jun 2019 13:14:40 -0500 Subject: [PATCH] Move vidyo cron script and dependencies into mozdef-deprecated --- cloudy_mozdef/packer/packer.json | 2 +- cron/vidyo2MozDef.conf | 6 - cron/vidyo2MozDef.py | 198 -------------------------- cron/vidyo2MozDef.sh | 10 -- docker/compose/mozdef_base/Dockerfile | 5 +- docs/source/installation.rst | 4 +- mq/plugins/vidyoCallID.py | 25 ---- requirements.txt | 1 - 8 files changed, 5 insertions(+), 246 deletions(-) delete mode 100644 cron/vidyo2MozDef.conf delete mode 100644 cron/vidyo2MozDef.py delete mode 100755 cron/vidyo2MozDef.sh delete mode 100644 mq/plugins/vidyoCallID.py diff --git a/cloudy_mozdef/packer/packer.json b/cloudy_mozdef/packer/packer.json index b4771c72..41e345c5 100644 --- a/cloudy_mozdef/packer/packer.json +++ b/cloudy_mozdef/packer/packer.json @@ -50,7 +50,7 @@ "set -e", "sudo yum update -y", "sudo yum makecache fast", - "sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make mysql-devel python python-devel python-pip git docker python3", + "sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make python python-devel python-pip git docker python3", "sudo pip install virtualenv docker-compose", "sudo systemctl enable docker", "sudo systemctl start docker", diff --git a/cron/vidyo2MozDef.conf b/cron/vidyo2MozDef.conf deleted file mode 100644 index 3c41bb6c..00000000 --- a/cron/vidyo2MozDef.conf +++ /dev/null @@ -1,6 +0,0 @@ -[options] -hostname= -username= -password= -database= -url=http://localhost:8080/events diff --git a/cron/vidyo2MozDef.py b/cron/vidyo2MozDef.py deleted file mode 100644 index 6258e740..00000000 --- a/cron/vidyo2MozDef.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Copyright (c) 2014 Mozilla Corporation - -import copy -import os -import sys -import json -import ConfigParser -import socket -import MySQLdb -from requests import Session -from optparse import OptionParser -from datetime import datetime - - -class MozDefError(Exception): - def __init__(self, msg): - self.msg = msg - - def __str__(self): - return repr(self.msg) - - -class MozDefEvent(): - # create requests session to allow for keep alives - httpsession = Session() - # Turns off needless and repetitive .netrc check for creds - httpsession.trust_env = False - debug = False - verify_certificate = False - # Never fail (ie no unexcepted exceptions sent to user, such as server/network not responding) - fire_and_forget_mode = True - log = {} - log['timestamp'] = datetime.isoformat(datetime.now()) - log['hostname'] = socket.getfqdn() - log['processid'] = os.getpid() - log['processname'] = sys.argv[0] - log['severity'] = 'INFO' - log['summary'] = None - log['category'] = 'event' - log['tags'] = list() - log['details'] = dict() - - def __init__(self, url='http://localhost/events', summary=None, category='event', severity='INFO', tags=[], details={}): - self.summary = summary - self.category = category - self.severity = severity - self.tags = tags - self.details = details - self.url = url - - def send(self, timestamp=None, summary=None, category=None, severity=None, tags=None, details=None, hostname=None): - log_msg = copy.copy(self.log) - - if timestamp is None: - log_msg['timestamp'] = self.timestamp - - else: - log_msg['timestamp'] = timestamp - - if summary is None: - log_msg['summary'] = self.summary - else: - log_msg['summary'] = summary - - if category is None: - log_msg['category'] = self.category - else: - log_msg['category'] = category - - if severity is None: - log_msg['severity'] = self.severity - else: - log_msg['severity'] = severity - - if tags is None: - log_msg['tags'] = self.tags - else: - log_msg['tags'] = tags - - if details is None: - log_msg['details'] = self.details - else: - log_msg['details'] = details - - if hostname is None: - log_msg['hostname'] = self.hostname - else: - log_msg['hostname'] = hostname - - if type(log_msg['details']) != dict: - raise MozDefError('details must be a dict') - elif type(log_msg['tags']) != list: - raise MozDefError('tags must be a list') - elif summary is None: - raise MozDefError('Summary is a required field') - - try: - self.httpsession.post(self.url, json.dumps(log_msg, encoding='utf-8'), verify=self.verify_certificate) - - except Exception as e: - if not self.fire_and_forget_mode: - raise e - - -def main(): - ''' - connect to vidyo's mysql, read in calls and write to mozdef - ''' - mdEvent = MozDefEvent(options.url) - mdEvent.debug = True - mdEvent.fire_and_forget_mode = False - - # connect to mysql - db=MySQLdb.connect(host=options.hostname, user=options.username,passwd=options.password,db=options.database) - c=db.cursor(MySQLdb.cursors.DictCursor) - - c.execute("select * from ConferenceCall2 where JoinTime between NOW() - INTERVAL 30 MINUTE and NOW() or LeaveTime between NOW() - INTERVAL 30 MINUTE and NOW()") - rows=c.fetchall() - c.close() - - # Build dictionary of calls in order to consolidate multiple rows for a single call - calls = {} - for row in rows: - id = row['UniqueCallID'] - # Copy the row's info if we don't already have the final completed call state - if id not in calls or (id in calls and calls[id]['CallState'] != 'COMPLETED'): - calls[id] = row - - # Massage call data and send to MozDef - for key in calls.keys(): - call = calls[key] - if call['LeaveTime'] is not None: - duration = call['LeaveTime'] - call['JoinTime'] - call['CallDuration'] = duration.seconds - - # fix up the data for json - for k in call.keys(): - # convert datetime objects to isoformat for json serialization - if isinstance(call[k], datetime): - call[k] = call[k].isoformat() - # make sure it's a string, not unicode forced into a string - if isinstance(call[k],str): - # db has unicode stored as string, so decode, then encode - call[k] = call[k].decode('utf-8','ignore').encode('ascii','ignore') - - mdEvent.send(timestamp=call['JoinTime'], - summary='Vidyo call status for ' + call['UniqueCallID'].encode('ascii', 'ignore'), - tags=['vidyo'], - details=call, - category='vidyo', - hostname=socket.gethostname() - ) - - -def getConfig(optionname, thedefault, configfile): - """read an option from a config file or set a default - send 'thedefault' as the data class you want to get a string back - i.e. 'True' will return a string - True will return a bool - 1 will return an int - """ - retvalue = thedefault - opttype = type(thedefault) - if os.path.isfile(configfile): - config = ConfigParser.ConfigParser() - config.readfp(open(configfile)) - if config.has_option('options', optionname): - if opttype == bool: - retvalue = config.getboolean('options', optionname) - elif opttype == int: - retvalue = config.getint('options', optionname) - elif opttype == float: - retvalue = config.getfloat('options', optionname) - else: - retvalue = config.get('options', optionname) - return retvalue - - -def initConfig(configfile): - # default options - options.url = getConfig('url', 'http://localhost:8080/events', configfile) - options.username = getConfig('username', '', configfile) - options.password = getConfig('password', '', configfile) - options.database = getConfig('database', '', configfile) - options.hostname = getConfig('hostname', '', configfile) - - -if __name__ == '__main__': - parser = OptionParser() - parser.add_option("-c", dest='configfile', default=sys.argv[0].replace('.py', '.conf'), help="configuration file to use") - (options, args) = parser.parse_args() - initConfig(options.configfile) - main() diff --git a/cron/vidyo2MozDef.sh b/cron/vidyo2MozDef.sh deleted file mode 100755 index 7de0c15a..00000000 --- a/cron/vidyo2MozDef.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Copyright (c) 2014 Mozilla Corporation - -source /opt/mozdef/envs/python/bin/activate -/opt/mozdef/envs/mozdef/cron/vidyo2MozDef.py -c /opt/mozdef/envs/mozdef/cron/vidyo2MozDef.conf - diff --git a/docker/compose/mozdef_base/Dockerfile b/docker/compose/mozdef_base/Dockerfile index 0701a8d9..7df5c549 100644 --- a/docker/compose/mozdef_base/Dockerfile +++ b/docker/compose/mozdef_base/Dockerfile @@ -14,15 +14,14 @@ RUN \ libffi-devel \ zlib-devel \ libcurl-devel \ - openssl \ + openssl \ openssl-devel \ git \ make && \ useradd -ms /bin/bash -d /opt/mozdef -m mozdef && \ mkdir /opt/mozdef/envs && \ cd /opt/mozdef && \ - yum install -y mysql-devel \ - python \ + yum install -y python \ python-devel \ python-pip && \ yum clean all && \ diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 6f8d9057..ed202d6b 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -70,11 +70,11 @@ We need to install a python2.7 virtualenv. On Yum-based systems:: - sudo yum install make zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel pcre-devel gcc gcc-c++ mysql-devel + sudo yum install make zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel pcre-devel gcc gcc-c++ On APT-based systems:: - sudo apt-get install make zlib1g-dev libbz2-dev libssl-dev libncurses5-dev libsqlite3-dev libreadline-dev tk-dev libpcre3-dev libpcre++-dev build-essential g++ libmysqlclient-dev + sudo apt-get install make zlib1g-dev libbz2-dev libssl-dev libncurses5-dev libsqlite3-dev libreadline-dev tk-dev libpcre3-dev libpcre++-dev build-essential g++ Then:: diff --git a/mq/plugins/vidyoCallID.py b/mq/plugins/vidyoCallID.py deleted file mode 100644 index dac34e7a..00000000 --- a/mq/plugins/vidyoCallID.py +++ /dev/null @@ -1,25 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Copyright (c) 2014 Mozilla Corporation - -import hashlib - - -class message(object): - def __init__(self): - ''' - takes an incoming vidyo call record and assigns a static ID - so we always update the same doc for current status. - ''' - - # this plugin - # sets the type field - self.registration = ['uniquecallid'] - self.priority = 5 - - def onMessage(self, message, metadata): - docid = hashlib.md5('vidyouniquecallid' + message['details']['uniquecallid']).hexdigest() - metadata['id'] = docid - message['type'] = 'vidyo' - return (message, metadata) diff --git a/requirements.txt b/requirements.txt index 4c620ec1..7d752437 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,6 @@ kombu==4.1.0 meld3==1.0.2 mozdef-client==1.0.11 mozdef-util==1.0.8 -MySQL-python==1.2.5 netaddr==0.7.1 nose==1.3.7 oauth2client==1.4.12