diff --git a/.gitignore b/.gitignore index 818c7ac5..bf0591e2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ cron/ipblocklist.txt alerts/generic_alerts /.project /data +.vscode diff --git a/alerts/alertPlugins.ini b/alerts/alertPlugins.ini deleted file mode 100644 index 71f4272f..00000000 --- a/alerts/alertPlugins.ini +++ /dev/null @@ -1,16 +0,0 @@ -[uwsgi] -chdir = /opt/mozdef/envs/mozdef/alerts/ -uid = mozdef -mule = alertWorker.py -pyargv = -c /opt/mozdef/envs/mozdef/alerts/alertWorker.conf -log-syslog = alertplugins-worker -log-drain = generated 0 bytes -socket = /opt/mozdef/envs/mozdef/alerts/alertPlugins.socket -virtualenv = /opt/mozdef/envs/mozdef/ -master-fifo = /opt/mozdef/envs/mozdef/alerts/alertPlugins.fifo -procname-master = [m] -procname-prefix = [alertPlugins] -never-swap -pidfile= /var/run/mozdef-alerts/alertPlugins.pid -vacuum = true -enable-threads diff --git a/alerts/alert_plugins.ini b/alerts/alert_plugins.ini new file mode 100644 index 00000000..e0ea1626 --- /dev/null +++ b/alerts/alert_plugins.ini @@ -0,0 +1,16 @@ +[uwsgi] +chdir = /opt/mozdef/envs/mozdef/alerts/ +uid = mozdef +mule = alert_worker.py +pyargv = -c /opt/mozdef/envs/mozdef/alerts/alert_worker.conf +log-syslog = alertplugins-worker +log-drain = generated 0 bytes +socket = /opt/mozdef/envs/mozdef/alerts/alert_plugins.socket +virtualenv = /opt/mozdef/envs/mozdef/ +master-fifo = /opt/mozdef/envs/mozdef/alerts/alert_plugins.fifo +procname-master = [m] +procname-prefix = [alertplugins] +never-swap +pidfile= /var/run/mozdef-alerts/alert_plugins.pid +vacuum = true +enable-threads diff --git a/alerts/alertWorker.conf b/alerts/alert_worker.conf similarity index 100% rename from alerts/alertWorker.conf rename to alerts/alert_worker.conf diff --git a/alerts/alertWorker.py b/alerts/alert_worker.py similarity index 100% rename from alerts/alertWorker.py rename to alerts/alert_worker.py diff --git a/alerts/bugzillaauthbruteforce.conf b/alerts/bugzilla_auth_bruteforce.conf similarity index 100% rename from alerts/bugzillaauthbruteforce.conf rename to alerts/bugzilla_auth_bruteforce.conf diff --git a/alerts/bugzillaauthbruteforce.py b/alerts/bugzilla_auth_bruteforce.py similarity index 95% rename from alerts/bugzillaauthbruteforce.py rename to alerts/bugzilla_auth_bruteforce.py index 7d47de28..41a8ce52 100644 --- a/alerts/bugzillaauthbruteforce.py +++ b/alerts/bugzilla_auth_bruteforce.py @@ -15,7 +15,7 @@ from query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch class AlertBugzillaPBruteforce(AlertTask): def main(self): - self.parse_config('bugzillaauthbruteforce.conf', ['url']) + self.parse_config('bugzilla_auth_bruteforce.conf', ['url']) search_query = SearchQuery(minutes=15) search_query.add_must([ diff --git a/alerts/httpauthbruteforce.conf b/alerts/http_auth_bruteforce.conf similarity index 100% rename from alerts/httpauthbruteforce.conf rename to alerts/http_auth_bruteforce.conf diff --git a/alerts/httpauthbruteforce.py b/alerts/http_auth_bruteforce.py similarity index 95% rename from alerts/httpauthbruteforce.py rename to alerts/http_auth_bruteforce.py index dc2d1845..cf812f92 100644 --- a/alerts/httpauthbruteforce.py +++ b/alerts/http_auth_bruteforce.py @@ -15,7 +15,7 @@ from query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch class AlertHTTPBruteforce(AlertTask): def main(self): - self.parse_config('httpauthbruteforce.conf', ['url']) + self.parse_config('http_auth_bruteforce.conf', ['url']) search_query = SearchQuery(minutes=15) search_query.add_must([ diff --git a/alerts/httperrors.conf b/alerts/http_errors.conf similarity index 100% rename from alerts/httperrors.conf rename to alerts/http_errors.conf diff --git a/alerts/httperrors.py b/alerts/http_errors.py similarity index 96% rename from alerts/httperrors.py rename to alerts/http_errors.py index 74298675..17427ea3 100644 --- a/alerts/httperrors.py +++ b/alerts/http_errors.py @@ -15,7 +15,7 @@ from query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch class AlertHTTPErrors(AlertTask): def main(self): - self.parse_config('httperrors.conf', ['url']) + self.parse_config('http_errors.conf', ['url']) search_query = SearchQuery(minutes=15) diff --git a/alerts/sshbruteforce_bro.conf b/alerts/ssh_bruteforce_bro.conf similarity index 100% rename from alerts/sshbruteforce_bro.conf rename to alerts/ssh_bruteforce_bro.conf diff --git a/alerts/sshbruteforce_bro.py b/alerts/ssh_bruteforce_bro.py similarity index 96% rename from alerts/sshbruteforce_bro.py rename to alerts/ssh_bruteforce_bro.py index c737b80c..37a0d398 100644 --- a/alerts/sshbruteforce_bro.py +++ b/alerts/ssh_bruteforce_bro.py @@ -15,7 +15,7 @@ from query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch class AlertSSHManyConns(AlertTask): def main(self): - self.parse_config('sshbruteforce_bro.conf', ['url']) + self.parse_config('ssh_bruteforce_bro.conf', ['url']) search_query = SearchQuery(minutes=15) diff --git a/alerts/supervisord.alerts.ini b/alerts/supervisord_alerts.ini similarity index 100% rename from alerts/supervisord.alerts.ini rename to alerts/supervisord_alerts.ini diff --git a/config/50-mozdef-filter.conf b/config/50-mozdef-filter.conf index 0df34c9b..e3154036 100644 --- a/config/50-mozdef-filter.conf +++ b/config/50-mozdef-filter.conf @@ -2,7 +2,7 @@ if $programname == 'mozdefbot-worker' then /var/log/mozdef/mozdefbot.log if $programname == 'loginput-worker' then /var/log/mozdef/loginput.log if $programname == 'infosecsqs-worker' then /var/log/mozdef/infosecsqs.log if $programname == 'restapi-worker' then /var/log/mozdef/restapi.log -if $programname == 'syslog-worker' then /var/log/mozdef/syslog.log +if $programname == 'eventtask-worker' then /var/log/mozdef/eventtask.log if $programname == 'nubis-worker' then /var/log/mozdef/nubis.log if $programname == 'bro-worker' then /var/log/mozdef/bro.log if $programname == 'migsqs-worker' then /var/log/mozdef/migsqs.log diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 583408f4..466d0bb9 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -395,7 +395,7 @@ Manual Installation *Use sudo whereever required* -**(Currently only for apt-based systems)** +**(Currently only for apt-based systems using Docker)** 1. Cloning repository :: diff --git a/mq/esworker.cloudtrail.conf b/mq/esworker_cloudtrail.conf similarity index 100% rename from mq/esworker.cloudtrail.conf rename to mq/esworker_cloudtrail.conf diff --git a/mq/esworker.cloudtrail.py b/mq/esworker_cloudtrail.py similarity index 100% rename from mq/esworker.cloudtrail.py rename to mq/esworker_cloudtrail.py diff --git a/mq/esworker.conf b/mq/esworker_eventtask.conf similarity index 100% rename from mq/esworker.conf rename to mq/esworker_eventtask.conf diff --git a/mq/esworker.py b/mq/esworker_eventtask.py similarity index 100% rename from mq/esworker.py rename to mq/esworker_eventtask.py diff --git a/mq/esworker.papertrail.conf b/mq/esworker_papertrail.conf similarity index 100% rename from mq/esworker.papertrail.conf rename to mq/esworker_papertrail.conf diff --git a/mq/esworker.papertrail.py b/mq/esworker_papertrail.py similarity index 100% rename from mq/esworker.papertrail.py rename to mq/esworker_papertrail.py diff --git a/mq/esworker.sqs.conf b/mq/esworker_sqs.conf similarity index 100% rename from mq/esworker.sqs.conf rename to mq/esworker_sqs.conf diff --git a/mq/esworker.sqs.py b/mq/esworker_sqs.py similarity index 99% rename from mq/esworker.sqs.py rename to mq/esworker_sqs.py index 487e19b2..5d191a36 100755 --- a/mq/esworker.sqs.py +++ b/mq/esworker_sqs.py @@ -11,7 +11,7 @@ # kombu's support for SQS is buggy # so this version uses boto # to read an SQS queue and put events into elastic search -# in the same manner as esworker.py +# in the same manner as esworker_eventtask.py import json diff --git a/mq/eventtask.ini b/mq/eventtask.ini new file mode 100644 index 00000000..493ccf51 --- /dev/null +++ b/mq/eventtask.ini @@ -0,0 +1,25 @@ +[uwsgi] +chdir = /opt/mozdef/envs/mozdef/mq/ +uid = mozdef +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +mule = esworker_eventtask.py +pyargv = -c /opt/mozdef/envs/mozdef/mq/esworker_eventtask.conf +log-syslog = eventtask-worker +log-drain = generated 0 bytes +socket = /opt/mozdef/envs/mozdef/mq/eventtask.socket +virtualenv = /opt/mozdef/envs/mozdef/ +procname-master = [m] +procname-prefix = [eventtask] +master-fifo = /opt/mozdef/envs/mozdef/mq/eventtask.fifo +never-swap +pidfile = /var/run/mozdefeventtask/eventtask.pid +vacuum = true +enable-threads diff --git a/mq/mqwSyslog.ini b/mq/mqwSyslog.ini deleted file mode 100644 index fe69348e..00000000 --- a/mq/mqwSyslog.ini +++ /dev/null @@ -1,25 +0,0 @@ -[uwsgi] -chdir = /opt/mozdef/envs/mozdef/mq/ -uid = mozdef -mule = esworker.py -mule = esworker.py -mule = esworker.py -mule = esworker.py -mule = esworker.py -mule = esworker.py -mule = esworker.py -mule = esworker.py -mule = esworker.py -mule = esworker.py -pyargv = -c /opt/mozdef/envs/mozdef/mq/esworker.conf -log-syslog = syslog-worker -log-drain = generated 0 bytes -socket = /opt/mozdef/envs/mozdef/mq/mqwSyslog.socket -virtualenv = /opt/mozdef/envs/mozdef/ -procname-master = [m] -procname-prefix = [mqwSyslog] -master-fifo = /opt/mozdef/envs/mozdef/mq/mqwSyslog.fifo -never-swap -pidfile = /var/run/mozdefmqwSyslog/mqwSyslog.pid -vacuum = true -enable-threads diff --git a/systemdfiles/alert/mozdefalertplugins.service b/systemdfiles/alert/mozdefalertplugins.service index 233a9843..3e6a45c4 100644 --- a/systemdfiles/alert/mozdefalertplugins.service +++ b/systemdfiles/alert/mozdefalertplugins.service @@ -1,5 +1,5 @@ [Unit] -Description=uWSGI mozdef Alert Plugins +Description=uWSGI MozDef Alert Plugins Service After=rabbitmq-server.service [Service] @@ -9,7 +9,7 @@ ExecStartPre=-/usr/bin/mkdir -p /var/run/mozdef-alerts ExecStartPre=/usr/bin/chown -R mozdef:mozdef /var/run/mozdef-alerts User=mozdef Group=mozdef -ExecStart=/bin/bash -c 'cd /opt/mozdef/envs/mozdef/alerts; source /opt/mozdef/envs/mozdef/bin/activate; uwsgi --ini alertPlugins.ini' +ExecStart=/bin/bash -c 'cd /opt/mozdef/envs/mozdef/alerts; source /opt/mozdef/envs/mozdef/bin/activate; uwsgi --ini alert_plugins.ini' Restart=always KillSignal=SIGQUIT Type=notify @@ -18,4 +18,4 @@ NotifyAccess=all [Install] WantedBy=multi-user.target -Alias=alertPlugins +Alias=alertplugins diff --git a/systemdfiles/alert/mozdefalerts.service b/systemdfiles/alert/mozdefalerts.service index 6a00b0b2..ea377e90 100644 --- a/systemdfiles/alert/mozdefalerts.service +++ b/systemdfiles/alert/mozdefalerts.service @@ -1,5 +1,5 @@ [Unit] -Description=uWSGI mozdef Alert Service +Description=uWSGI MozDef Alert Service After=rabbitmq-server.service [Service] @@ -10,7 +10,7 @@ ExecStartPre=/usr/bin/chown -R mozdef:mozdef /var/run/mozdef-alerts PIDFile=/var/run/mozdef-alerts/supervisord.pid User=mozdef Group=mozdef -ExecStart=/bin/bash -c 'cd /opt/mozdef/envs/mozdef/alerts; source /opt/mozdef/envs/mozdef/bin/activate; /opt/mozdef/envs/mozdef/bin/supervisord -c /opt/mozdef/envs/mozdef/alerts/supervisord.alerts.ini' +ExecStart=/bin/bash -c 'cd /opt/mozdef/envs/mozdef/alerts; source /opt/mozdef/envs/mozdef/bin/activate; /opt/mozdef/envs/mozdef/bin/supervisord -c /opt/mozdef/envs/mozdef/alerts/supervisord_alerts.ini' Restart=always KillSignal=SIGQUIT Type=forking diff --git a/systemdfiles/alert/mozdefbot.service b/systemdfiles/alert/mozdefbot.service index b3ecbbb0..a2ffaec3 100644 --- a/systemdfiles/alert/mozdefbot.service +++ b/systemdfiles/alert/mozdefbot.service @@ -1,5 +1,5 @@ [Unit] -Description=uWSGI mozdef bot +Description=uWSGI MozDef Bot Service After=rabbitmq-server.service [Service] diff --git a/systemdfiles/consumer/mozdefloginput.service b/systemdfiles/consumer/mozdefloginput.service index cb9a79a5..55690c99 100644 --- a/systemdfiles/consumer/mozdefloginput.service +++ b/systemdfiles/consumer/mozdefloginput.service @@ -1,5 +1,5 @@ [Unit] -Description=uWSGI mozdefloginput +Description=uWSGI MozDef Log Input Service After=rabbitmq-server.service [Service] diff --git a/systemdfiles/consumer/mozdefmqwsyslog.service b/systemdfiles/consumer/mozdefmqwsyslog.service deleted file mode 100644 index 38f27f11..00000000 --- a/systemdfiles/consumer/mozdefmqwsyslog.service +++ /dev/null @@ -1,21 +0,0 @@ -[Unit] -Description=uWSGI mozdef mqwSyslog -After=rabbitmq-server.service - -[Service] -# Requires systemd version 211 or newer -PermissionsStartOnly=true -ExecStartPre=-/usr/bin/mkdir -p /var/run/mozdefmqwSyslog -ExecStartPre=/usr/bin/chown -R mozdef:mozdef /var/run/mozdefmqwSyslog -User=mozdef -Group=mozdef -ExecStart=/bin/bash -c 'cd /opt/mozdef/envs/mozdef/mq; source /opt/mozdef/envs/mozdef/bin/activate; uwsgi --ini mqwSyslog.ini' -Restart=always -KillSignal=SIGQUIT -Type=notify -StandardError=syslog -NotifyAccess=all - -[Install] -WantedBy=multi-user.target -Alias=mqwsyslog diff --git a/systemdfiles/consumer/mworker-eventtask.service b/systemdfiles/consumer/mworker-eventtask.service new file mode 100644 index 00000000..a6e8eeb9 --- /dev/null +++ b/systemdfiles/consumer/mworker-eventtask.service @@ -0,0 +1,21 @@ +[Unit] +Description=uWSGI eventtask mworker +After=rabbitmq-server.service + +[Service] +# Requires systemd version 211 or newer +PermissionsStartOnly=true +ExecStartPre=-/usr/bin/mkdir -p /var/run/mozdef_mq_worker_pids +ExecStartPre=/usr/bin/chown -R mozdef:mozdef /var/run/mozdef_mq_worker_pids +User=mozdef +Group=mozdef +ExecStart=/bin/bash -c 'cd /opt/mozdef/envs/mozdef/mq/; source /opt/mozdef/envs/mozdef/bin/activate; uwsgi --ini eventtask.ini' +Restart=always +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all + +[Install] +WantedBy=multi-user.target +Alias=eventtask \ No newline at end of file