зеркало из https://github.com/mozilla/MozDef.git
Merge pull request #1436 from mozilla/revert-1420-alerts_mongodb_scheduler
Revert "Turn on/off alerts via Web UI"
This commit is contained in:
Brandon Myers
GitHub
Коммит
e5f455429d
|
|
@ -0,0 +1,83 @@
|
|||
import os
|
||||
from celery import Celery
|
||||
from importlib import import_module
|
||||
from lib.config import ALERTS, LOGGING, RABBITMQ
|
||||
from logging.config import dictConfig
|
||||
|
||||
# Alert files to include
|
||||
alerts_include = []
|
||||
for alert in ALERTS.keys():
|
||||
alerts_include.append(".".join((alert).split(".")[:-1]))
|
||||
alerts_include = list(set(alerts_include))
|
||||
|
||||
# XXX TBD this should get wrapped into an object that provides pyconfig
|
||||
if os.getenv("OPTIONS_MQPROTOCOL", "amqp") == "sqs":
|
||||
BROKER_URL = "sqs://@"
|
||||
BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1], 'is_secure': True, 'port': 443}
|
||||
CELERY_RESULT_BACKEND = None
|
||||
alert_queue_name = os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4]
|
||||
CELERY_DEFAULT_QUEUE = alert_queue_name
|
||||
CELERY_QUEUES = {
|
||||
alert_queue_name: {"exchange": alert_queue_name, "binding_key": alert_queue_name}
|
||||
}
|
||||
else:
|
||||
BROKER_URL = "amqp://{0}:{1}@{2}:{3}//".format(
|
||||
RABBITMQ["mquser"], RABBITMQ["mqpassword"], RABBITMQ["mqserver"], RABBITMQ["mqport"]
|
||||
)
|
||||
CELERY_QUEUES = {
|
||||
"celery-default": {"exchange": "celery-default", "binding_key": "celery-default"}
|
||||
}
|
||||
CELERY_DEFAULT_QUEUE = 'celery-default'
|
||||
|
||||
CELERY_DISABLE_RATE_LIMITS = True
|
||||
CELERYD_CONCURRENCY = 1
|
||||
CELERY_IGNORE_RESULT = True
|
||||
CELERY_ACCEPT_CONTENT = ["json"]
|
||||
CELERY_TASK_SERIALIZER = "json"
|
||||
CELERYBEAT_SCHEDULE = {}
|
||||
|
||||
# Register frequency of the tasks in the scheduler
|
||||
for alert in ALERTS.keys():
|
||||
CELERYBEAT_SCHEDULE[alert] = {
|
||||
"task": alert,
|
||||
"schedule": ALERTS[alert]["schedule"],
|
||||
"options": {"queue": CELERY_DEFAULT_QUEUE, "exchange": CELERY_DEFAULT_QUEUE},
|
||||
}
|
||||
# add optional parameters:
|
||||
if "args" in ALERTS[alert]:
|
||||
CELERYBEAT_SCHEDULE[alert]["args"] = ALERTS[alert]["args"]
|
||||
if "kwargs" in ALERTS[alert]:
|
||||
CELERYBEAT_SCHEDULE[alert]["kwargs"] = ALERTS[alert]["kwargs"]
|
||||
|
||||
# Load logging config
|
||||
dictConfig(LOGGING)
|
||||
|
||||
# print(CELERYBEAT_SCHEDULE)
|
||||
|
||||
# Optional configuration, see the application user guide.
|
||||
# app.conf.update(
|
||||
# CELERY_TASK_RESULT_EXPIRES=3600,
|
||||
# )
|
||||
app = Celery("alerts", include=alerts_include)
|
||||
app.config_from_object("celeryconfig", force=True)
|
||||
|
||||
# As a result of celery 3 to celery 4, we need to dynamically
|
||||
# register all of the alert tasks specifically
|
||||
for alert_namespace in CELERYBEAT_SCHEDULE:
|
||||
try:
|
||||
alert_tokens = alert_namespace.split(".")
|
||||
alert_module_name = alert_tokens[0]
|
||||
alert_classname = alert_tokens[1]
|
||||
alert_module = import_module(alert_module_name)
|
||||
alert_class = getattr(alert_module, alert_classname)
|
||||
app.register_task(alert_class())
|
||||
except ImportError as e:
|
||||
print("Error importing {}".format(alert_namespace))
|
||||
print(e)
|
||||
pass
|
||||
except Exception as e:
|
||||
print("Error addding alert")
|
||||
print(e)
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.start()
|
||||
|
|
@ -1,177 +0,0 @@
|
|||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
# Copyright 2018 Regents of the University of Michigan
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Vendored and modified from https://github.com/zmap/celerybeat-mongo
|
||||
|
||||
import datetime
|
||||
|
||||
from mozdef_util.utilities.toUTC import toUTC
|
||||
from celery.beat import Scheduler, ScheduleEntry
|
||||
from celery import current_app
|
||||
import celery.schedules
|
||||
|
||||
|
||||
class AlertScheduleEntry(ScheduleEntry):
|
||||
|
||||
def __init__(self, task):
|
||||
self._task = task
|
||||
|
||||
self.app = current_app._get_current_object()
|
||||
self.name = self._task['name']
|
||||
self.task = self._task['name']
|
||||
|
||||
# Fill out schedule
|
||||
if self._task['schedule_type'] == 'crontab':
|
||||
self.schedule = celery.schedules.crontab(
|
||||
minute=self._task['crontab']['minute'],
|
||||
hour=self._task['crontab']['hour'],
|
||||
day_of_week=self._task['crontab']['day_of_week'],
|
||||
day_of_month=self._task['crontab']['day_of_month'],
|
||||
month_of_year=self._task['crontab']['month_of_year']
|
||||
)
|
||||
elif self._task['schedule_type'] == 'interval':
|
||||
self.schedule = celery.schedules.schedule(datetime.timedelta(**{self._task['interval']['period']: self._task['interval']['every']}))
|
||||
|
||||
self.args = self._task['args']
|
||||
self.kwargs = self._task['kwargs']
|
||||
self.options = {
|
||||
'enabled': self._task['enabled']
|
||||
}
|
||||
if 'last_run_at' not in self._task:
|
||||
self._task['last_run_at'] = self._default_now()
|
||||
self.last_run_at = toUTC(self._task['last_run_at'])
|
||||
if 'run_immediately' not in self._task:
|
||||
self._task['run_immediately'] = False
|
||||
|
||||
def _default_now(self):
|
||||
return self.app.now()
|
||||
|
||||
def next(self):
|
||||
self._task['last_run_at'] = self.app.now()
|
||||
self._task['run_immediately'] = False
|
||||
return self.__class__(self._task)
|
||||
|
||||
__next__ = next
|
||||
|
||||
def is_due(self):
|
||||
if not self._task['enabled']:
|
||||
return False, 5.0 # 5 second delay for re-enable.
|
||||
if 'start_after' in self._task and self._task['start_after']:
|
||||
if datetime.datetime.now() < self._task['start_after']:
|
||||
return False, 5.0
|
||||
if self._task['run_immediately']:
|
||||
# figure out when the schedule would run next anyway
|
||||
_, n = self.schedule.is_due(self.last_run_at)
|
||||
return True, n
|
||||
return self.schedule.is_due(self.last_run_at)
|
||||
|
||||
def __repr__(self):
|
||||
return (u'<{0} ({1} {2}(*{3}, **{4}) {{5}})>'.format(
|
||||
self.__class__.__name__,
|
||||
self.name, self.task, self.args,
|
||||
self.kwargs, self.schedule,
|
||||
))
|
||||
|
||||
def reserve(self, entry):
|
||||
new_entry = Scheduler.reserve(self, entry)
|
||||
return new_entry
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# Copyright 2018 Regents of the University of Michigan
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Vendored and modified from https://github.com/zmap/celerybeat-mongo
|
||||
|
||||
import datetime
|
||||
|
||||
from .periodic_task import PeriodicTask
|
||||
from .alert_schedule_entry import AlertScheduleEntry
|
||||
from .celery_rest_client import CeleryRestClient
|
||||
|
||||
from celery.beat import Scheduler
|
||||
|
||||
|
||||
class AlertScheduler(Scheduler):
|
||||
|
||||
#: how often should we sync in schedule information
|
||||
#: from the backend mongo database
|
||||
UPDATE_INTERVAL = datetime.timedelta(seconds=5)
|
||||
|
||||
Entry = AlertScheduleEntry
|
||||
|
||||
Model = PeriodicTask
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._schedule = {}
|
||||
self._last_updated = None
|
||||
Scheduler.__init__(self, *args, **kwargs)
|
||||
self.max_interval = (kwargs.get('max_interval') or self.app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or 5)
|
||||
self.celery_rest = CeleryRestClient()
|
||||
self.celery_rest.print_schedule()
|
||||
self._schedule = self.fetch_schedule()
|
||||
|
||||
def setup_schedule(self):
|
||||
pass
|
||||
|
||||
def requires_update(self):
|
||||
"""check whether we should pull an updated schedule
|
||||
from the backend database"""
|
||||
if not self._last_updated:
|
||||
return True
|
||||
return self._last_updated + self.UPDATE_INTERVAL < datetime.datetime.now()
|
||||
|
||||
@property
|
||||
def schedule(self):
|
||||
if self.requires_update():
|
||||
self.update_schedule()
|
||||
self._last_updated = datetime.datetime.now()
|
||||
return self._schedule
|
||||
|
||||
def update_schedule(self):
|
||||
api_results = self.celery_rest.fetch_schedule_dict()
|
||||
for name, entry in self._schedule.items():
|
||||
if name in api_results:
|
||||
entry._task['enabled'] = api_results[name]['enabled']
|
||||
|
||||
def fetch_schedule(self):
|
||||
api_results = self.celery_rest.fetch_schedule_dict()
|
||||
schedule = {}
|
||||
for name, doc in api_results.items():
|
||||
schedule[name] = self.Entry(doc)
|
||||
return schedule
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
import os
|
||||
from logging.config import dictConfig
|
||||
|
||||
from lib.config import LOGGING, RABBITMQ, RESTAPI_URL, RESTAPI_TOKEN
|
||||
|
||||
|
||||
# XXX TBD this should get wrapped into an object that provides pyconfig
|
||||
if os.getenv("OPTIONS_MQPROTOCOL", "amqp") == "sqs":
|
||||
BROKER_URL = "sqs://@"
|
||||
BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1], 'is_secure': True, 'port': 443}
|
||||
CELERY_RESULT_BACKEND = None
|
||||
alert_queue_name = os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4]
|
||||
CELERY_DEFAULT_QUEUE = alert_queue_name
|
||||
CELERY_QUEUES = {
|
||||
alert_queue_name: {"exchange": alert_queue_name, "binding_key": alert_queue_name}
|
||||
}
|
||||
else:
|
||||
BROKER_URL = "amqp://{0}:{1}@{2}:{3}//".format(
|
||||
RABBITMQ["mquser"], RABBITMQ["mqpassword"], RABBITMQ["mqserver"], RABBITMQ["mqport"]
|
||||
)
|
||||
CELERY_QUEUES = {
|
||||
"celery-default": {"exchange": "celery-default", "binding_key": "celery-default"}
|
||||
}
|
||||
CELERY_DEFAULT_QUEUE = 'celery-default'
|
||||
|
||||
CELERY_DISABLE_RATE_LIMITS = True
|
||||
CELERYD_CONCURRENCY = 1
|
||||
CELERY_IGNORE_RESULT = True
|
||||
CELERY_ACCEPT_CONTENT = ["json"]
|
||||
CELERY_TASK_SERIALIZER = "json"
|
||||
|
||||
# Custom Alert Scheduler
|
||||
CELERYBEAT_SCHEDULER = "lib.celery_scheduler.alerts_scheduler.AlertScheduler"
|
||||
CELERY_RESTAPI_URL = RESTAPI_URL
|
||||
CELERY_RESTAPI_JWT = RESTAPI_TOKEN
|
||||
|
||||
# Load logging config
|
||||
dictConfig(LOGGING)
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
from celery.utils.log import get_logger
|
||||
from celery import current_app
|
||||
from requests_jwt import JWTAuth
|
||||
import requests
|
||||
import json
|
||||
|
||||
from bson.objectid import ObjectId
|
||||
from importlib import import_module
|
||||
from celery.schedules import crontab, timedelta
|
||||
from mozdef_util.utilities.logger import logger
|
||||
from lib.config import ALERTS, RESTAPI_URL
|
||||
|
||||
|
||||
class CeleryRestClient(object):
|
||||
def __init__(self):
|
||||
if hasattr(current_app.conf, "CELERY_RESTAPI_JWT") and current_app.conf.CELERY_RESTAPI_JWT != "":
|
||||
self._restapi_jwt = JWTAuth(current_app.conf.CELERY_RESTAPI_JWT)
|
||||
self._restapi_jwt.set_header_format('Bearer %s')
|
||||
get_logger(__name__).info("setting JWT value")
|
||||
else:
|
||||
self._restapi_jwt = None
|
||||
|
||||
if hasattr(current_app.conf, "CELERY_RESTAPI_URL"):
|
||||
self._restapi_url = current_app.conf.CELERY_RESTAPI_URL
|
||||
get_logger(__name__).info("alert scheduler using {0}".format(self._restapi_url))
|
||||
else:
|
||||
raise Exception("Need to define CELERY_RESTAPI_URL")
|
||||
|
||||
def fetch_schedule_dict(self):
|
||||
resp = requests.get(self._restapi_url + "/alertschedules", auth=self._restapi_jwt)
|
||||
if not resp.ok:
|
||||
raise Exception("Received error {0} from rest api when fetching alert schedules".format(resp.status_code))
|
||||
api_results = json.loads(resp.text)
|
||||
return api_results
|
||||
|
||||
def print_schedule(self):
|
||||
schedule = self.fetch_schedule_dict()
|
||||
get_logger(__name__).info("**** Current Alert Schedule ****")
|
||||
for alert_name, details in schedule.items():
|
||||
get_logger(__name__).info("\t{0}: {1} (enabled={2})".format(alert_name, details['schedule_string'], details['enabled']))
|
||||
|
||||
def load_and_register_alerts(self):
|
||||
existing_alert_schedules = self.fetch_schedule_dict()
|
||||
alert_schedules = {}
|
||||
for alert_name, params in ALERTS.items():
|
||||
# Register alerts in celery
|
||||
try:
|
||||
alert_tokens = alert_name.split(".")
|
||||
alert_module_name = alert_tokens[0]
|
||||
alert_classname = alert_tokens[-1]
|
||||
alert_module = import_module(alert_module_name)
|
||||
alert_class = getattr(alert_module, alert_classname)
|
||||
current_app.register_task(alert_class())
|
||||
except ImportError as e:
|
||||
logger.exception("Error importing {0}: {1}".format(alert_name, e))
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.exception("Generic error registering {0}: {1}".format(alert_name, e))
|
||||
pass
|
||||
|
||||
full_path_name = "{0}.{1}".format(alert_module_name, alert_classname)
|
||||
alert_schedule = {
|
||||
"_id": str(ObjectId()),
|
||||
"_cls": "PeriodicTask",
|
||||
"name": full_path_name,
|
||||
"task": full_path_name,
|
||||
"enabled": True,
|
||||
"args": [],
|
||||
"kwargs": {},
|
||||
}
|
||||
if 'args' in params:
|
||||
alert_schedule['args'] = params['args']
|
||||
if 'kwargs' in params:
|
||||
alert_schedule['kwargs'] = params['kwargs']
|
||||
|
||||
if isinstance(params['schedule'], timedelta):
|
||||
alert_schedule['schedule_type'] = 'interval'
|
||||
alert_schedule['interval'] = {
|
||||
"every": params['schedule'].total_seconds(),
|
||||
"period": "seconds"
|
||||
}
|
||||
alert_schedule['schedule_string'] = "{0} {1}".format(
|
||||
params['schedule'].total_seconds(),
|
||||
"seconds"
|
||||
)
|
||||
elif isinstance(params['schedule'], crontab):
|
||||
alert_schedule['schedule_type'] = 'crontab'
|
||||
alert_schedule['crontab'] = {
|
||||
"minute": params['schedule']._orig_minute,
|
||||
"hour": params['schedule']._orig_hour,
|
||||
"day_of_week": params['schedule']._orig_day_of_week,
|
||||
"day_of_month": params['schedule']._orig_day_of_month,
|
||||
"month_of_year": params['schedule']._orig_month_of_year,
|
||||
}
|
||||
alert_schedule['schedule_string'] = "{0} {1} {2} {3} {4}".format(
|
||||
params['schedule']._orig_minute,
|
||||
params['schedule']._orig_hour,
|
||||
params['schedule']._orig_day_of_week,
|
||||
params['schedule']._orig_day_of_month,
|
||||
params['schedule']._orig_month_of_year,
|
||||
)
|
||||
|
||||
if alert_name not in existing_alert_schedules:
|
||||
logger.debug("Inserting schedule for {0} into mongodb".format(full_path_name))
|
||||
updated_alert_schedule = alert_schedule
|
||||
else:
|
||||
# Update schedule if it differs from file to api
|
||||
del existing_alert_schedules[alert_name][existing_alert_schedules[alert_name]['schedule_type']]
|
||||
existing_alert_schedules[alert_name]['schedule_type'] = alert_schedule['schedule_type']
|
||||
existing_alert_schedules[alert_name][alert_schedule['schedule_type']] = alert_schedule[alert_schedule['schedule_type']]
|
||||
existing_alert_schedules[alert_name]['schedule_string'] = alert_schedule['schedule_string']
|
||||
updated_alert_schedule = existing_alert_schedules[alert_name]
|
||||
|
||||
alert_schedules[alert_name] = updated_alert_schedule
|
||||
resp = requests.post(url=RESTAPI_URL + "/updatealertschedules", data=json.dumps(alert_schedules), auth=self._restapi_jwt)
|
||||
if not resp.ok:
|
||||
raise Exception("Received error {0} from rest api when updating alerts schedules".format(resp.status_code))
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
# Copyright 2018 Regents of the University of Michigan
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Vendored and modified from https://github.com/zmap/celerybeat-mongo
|
||||
|
||||
|
||||
class PeriodicTask():
|
||||
@property
|
||||
def schedule(self):
|
||||
if self.interval:
|
||||
return self.interval.schedule
|
||||
elif self.crontab:
|
||||
return self.crontab.schedule
|
||||
else:
|
||||
raise Exception("must define interval or crontab schedule")
|
||||
|
||||
def __unicode__(self):
|
||||
fmt = '{0.name}: {{no schedule}}'
|
||||
if self.interval:
|
||||
fmt = '{0.name}: {0.interval}'
|
||||
elif self.crontab:
|
||||
fmt = '{0.name}: {0.crontab}'
|
||||
else:
|
||||
raise Exception("must define interval or crontab schedule")
|
||||
return fmt.format(self)
|
||||
|
|
@ -41,10 +41,6 @@ ES = {
|
|||
'servers': [es_server]
|
||||
}
|
||||
|
||||
RESTAPI_URL = "http://localhost:8081"
|
||||
# Leave empty for no auth
|
||||
RESTAPI_TOKEN = ""
|
||||
|
||||
LOGGING = {
|
||||
'version': 1,
|
||||
'disable_existing_loggers': True,
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
from celery import Celery
|
||||
|
||||
from lib.celery_scheduler import celery_config
|
||||
from lib.celery_scheduler.celery_rest_client import CeleryRestClient
|
||||
|
||||
app = Celery("alerts")
|
||||
app.config_from_object(celery_config, force=True)
|
||||
|
||||
celery_rest = CeleryRestClient()
|
||||
celery_rest.load_and_register_alerts()
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.start()
|
||||
|
|
@ -18,7 +18,7 @@ serverurl=unix:///opt/mozdef/envs/mozdef/alerts/supervisorctl.sock
|
|||
|
||||
[program:alerts]
|
||||
priority=2
|
||||
command=celery -A lib.tasks worker --loglevel=info --beat
|
||||
command=celery -A celeryconfig worker --loglevel=info --beat
|
||||
user=mozdef
|
||||
group=mozdef
|
||||
directory=/opt/mozdef/envs/mozdef/alerts
|
||||
|
|
|
|||
|
|
@ -41,10 +41,6 @@ else:
|
|||
'servers': ['http://localhost:9200']
|
||||
}
|
||||
|
||||
RESTAPI_URL = "http://rest:8081"
|
||||
# Leave empty for no auth
|
||||
RESTAPI_TOKEN = ""
|
||||
|
||||
LOGGING = {
|
||||
'version': 1,
|
||||
'disable_existing_loggers': True,
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ services:
|
|||
volumes:
|
||||
- /opt/mozdef/docker/compose/mozdef_alerts/files/config.py:/opt/mozdef/envs/mozdef/alerts/lib/config.py
|
||||
restart: always
|
||||
command: bash -c 'celery -A lib.tasks worker --loglevel=info --beat'
|
||||
command: bash -c 'celery -A celeryconfig worker --loglevel=info --beat'
|
||||
depends_on:
|
||||
- base
|
||||
- bootstrap
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ services:
|
|||
- mozdef/mozdef_alerts
|
||||
- mozdef_alerts:latest
|
||||
restart: always
|
||||
command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && celery -A lib.tasks worker --loglevel=info --beat'
|
||||
command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && celery -A celeryconfig worker --loglevel=info --beat'
|
||||
depends_on:
|
||||
- base
|
||||
- elasticsearch
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
from celery.schedules import crontab, timedelta
|
||||
import time
|
||||
import logging
|
||||
import os
|
||||
|
||||
ALERTS = {
|
||||
'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')},
|
||||
|
|
@ -33,18 +32,13 @@ RABBITMQ = {
|
|||
'alertqueue': 'mozdef.alert'
|
||||
}
|
||||
|
||||
es_server = "http://elasticsearch:9200"
|
||||
|
||||
if os.getenv('OPTIONS_ESSERVERS'):
|
||||
es_server = os.getenv('OPTIONS_ESSERVERS')
|
||||
|
||||
ES = {
|
||||
'servers': [es_server]
|
||||
'servers': ['http://elasticsearch:9200']
|
||||
}
|
||||
|
||||
RESTAPI_URL = "http://rest:8081"
|
||||
# Leave empty for no auth
|
||||
RESTAPI_TOKEN = ""
|
||||
OPTIONS = {
|
||||
'defaulttimezone': 'UTC',
|
||||
}
|
||||
|
||||
LOGGING = {
|
||||
'version': 1,
|
||||
|
|
@ -75,7 +69,7 @@ LOGGING = {
|
|||
'loggers': {
|
||||
'celery': {
|
||||
'handlers': ['celery', 'console'],
|
||||
'level': 'INFO',
|
||||
'level': 'DEBUG',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ Once you've reference the containers, you can shell into the alerts container:
|
|||
Next, start celery
|
||||
::
|
||||
|
||||
celery -A lib.tasks worker --loglevel=info --beat
|
||||
celery -A celeryconfig worker --loglevel=info --beat
|
||||
|
||||
If you need to send in adhoc events you can usually do it via curl as follows:
|
||||
::
|
||||
|
|
|
|||
|
|
@ -530,7 +530,7 @@ Alternatively you can start the following services manually in this way from ins
|
|||
|
||||
# alert worker
|
||||
(mozdef) [mozdef@mozdev mozdef]$ cd ~/mozdef/alerts
|
||||
(mozdef) [mozdef@mozdev alerts]$ celery -A lib.tasks worker --loglevel=info --beat
|
||||
(mozdef) [mozdef@mozdev alerts]$ celery -A celeryconfig worker --loglevel=info --beat
|
||||
|
||||
To initialize elasticsearch indices and load some sample data::
|
||||
|
||||
|
|
|
|||
|
|
@ -33,8 +33,6 @@ import '/client/watchItem.js';
|
|||
import '/client/eventdetails.html';
|
||||
import '/client/fqdnBlocklistTable.html';
|
||||
import '/client/fqdnBlocklistTable.js';
|
||||
import '/client/manageAlertsTable.html';
|
||||
import '/client/manageAlertsTable.js';
|
||||
import '/client/globe.html';
|
||||
import '/client/globe.js';
|
||||
import '/client/incidentAdd.html';
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
<!--
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
Copyright (c) 2014 Mozilla Corporation
|
||||
-->
|
||||
|
||||
<template name="alertschedules">
|
||||
<div class="alertschedules container">
|
||||
<div class="fluid center">
|
||||
<table class="table table-striped table-hover table-condensed">
|
||||
<caption><p class="lead">Alert Schedules</p>
|
||||
</caption>
|
||||
{{ #if isReady }}
|
||||
<thead>
|
||||
<tr>
|
||||
<td>NAME</td>
|
||||
<td>SCHEDULE TYPE</td>
|
||||
<td>SCHEDULE</td>
|
||||
<td>ENABLED</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{#each documents}}
|
||||
{{>each_alert_schedule}}
|
||||
{{else}}
|
||||
{{#if query}}
|
||||
<tr><td colspan="7"><p class="alert alert-warning">Nothing found for {{query}}..</p></td></tr>
|
||||
{{/if}}
|
||||
{{/each}}
|
||||
</tbody>
|
||||
{{else}}
|
||||
{{>loading}}
|
||||
{{/if}}
|
||||
</table>
|
||||
{{> defaultBootstrapPaginator pagination=templatePagination limit=5 containerClass="text-center"}}
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<template name="each_alert_schedule">
|
||||
<tr class="tooltip-wrapper info-row" data-toggle="tooltip">
|
||||
<td>{{name}}</td>
|
||||
<td>{{schedule_type}}</td>
|
||||
<td>{{schedule_string}}</td>
|
||||
{{#if enabled}}
|
||||
<td>Yes</td>
|
||||
<td>
|
||||
<span class="tooltip-wrapper" title="{{#if modifiedby}}Enabled by: {{modifiedby}} at {{modifiedat}}{{/if}}" data-toggle="tooltip">
|
||||
<button class="btn btn-xs btn-warning" data-toggle="collapse" data-target="#disable{{_id}}">arm disable</button>
|
||||
<div id="disable{{_id}}" class="collapse">
|
||||
<button class="btn btn-xs btn-warning btnAlertAck" data-target="{{_id}}">Disable</button>
|
||||
</div>
|
||||
</span>
|
||||
</td>
|
||||
{{else}}
|
||||
<td>No</td>
|
||||
<td>
|
||||
<span class="tooltip-wrapper" title="{{#if modifiedby}}Disabled by: {{modifiedby}} at {{modifiedat}}{{/if}}" data-toggle="tooltip">
|
||||
<button class="btn btn-xs btn-warning" data-toggle="collapse" data-target="#enable{{_id}}">arm enable</button>
|
||||
<div id="enable{{_id}}" class="collapse">
|
||||
<button class="btn btn-xs btn-warning btnAlertAcked" data-target="{{_id}}">Enable</button>
|
||||
</div>
|
||||
</span>
|
||||
</td>
|
||||
{{/if}}
|
||||
</tr>
|
||||
|
||||
</template>
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
/*
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
Copyright (c) 2014 Mozilla Corporation
|
||||
*/
|
||||
import { Template } from 'meteor/templating';
|
||||
|
||||
if ( Meteor.isClient ) {
|
||||
Template.alertschedules.helpers( {
|
||||
isReady: function() {
|
||||
return Template.instance().pagination.ready();
|
||||
},
|
||||
|
||||
templatePagination: function() {
|
||||
return Template.instance().pagination;
|
||||
},
|
||||
|
||||
documents: function() {
|
||||
return Template.instance().pagination.getPage();
|
||||
},
|
||||
|
||||
query() {
|
||||
return Template.instance().searchQuery.get();
|
||||
}
|
||||
} );
|
||||
|
||||
Template.alertschedules.events( {
|
||||
"click .btnAlertAcked": function( e, t ) {
|
||||
id = $( e.target ).attr( 'data-target' );
|
||||
alertschedules.update( {_id: id}, { $set: { 'enabled': true } } );
|
||||
// tag the alert with info about user who interacted
|
||||
alertschedules.update( {_id: id}, { $set: { 'modifiedat': new Date() } } );
|
||||
alertschedules.update( {_id: id}, { $set: { 'modifiedby': Meteor.user().profile.email } } );
|
||||
},
|
||||
|
||||
"click .btnAlertAck": function( e, t ) {
|
||||
id = $( e.target ).attr( 'data-target' );
|
||||
alertschedules.update( {_id: id}, { $set: { 'enabled': false } } );
|
||||
// tag the alert with info about user who interacted
|
||||
alertschedules.update( {_id: id}, { $set: { 'modifiedat': new Date() } } );
|
||||
alertschedules.update( {_id: id}, { $set: { 'modifiedby': Meteor.user().profile.email } } );
|
||||
},
|
||||
|
||||
"keyup #search"( event, template ) {
|
||||
let value = event.target.value.trim();
|
||||
|
||||
if ( value !== '' && event.keyCode === 13 ) {
|
||||
template.searchQuery.set( value );
|
||||
}
|
||||
|
||||
if ( value === '' || event.keyCode == 27 ) {
|
||||
template.searchQuery.set( '' );
|
||||
event.target.value = '';
|
||||
}
|
||||
}
|
||||
} );
|
||||
|
||||
Template.alertschedules.onCreated( function() {
|
||||
this.pagination = new Meteor.Pagination( alertschedules, {
|
||||
sort: {
|
||||
name: 1
|
||||
},
|
||||
perPage: prefs().pageSize,
|
||||
} );
|
||||
|
||||
Template.instance().searchQuery = new ReactiveVar();
|
||||
Tracker.autorun( () => {
|
||||
const filter_Text = this.searchQuery.get();
|
||||
|
||||
if ( filter_Text && filter_Text.length > 0 ) {
|
||||
this.pagination.filters( { $text: { $search: filter_Text } } );
|
||||
} else {
|
||||
this.pagination.filters( {} );
|
||||
}
|
||||
|
||||
} );
|
||||
} );
|
||||
}
|
||||
|
|
@ -35,9 +35,6 @@ Copyright (c) 2014 Mozilla Corporation
|
|||
{{#if isFeature "fqdnblocklist"}}
|
||||
<li><a href="/fqdnblocklist">fqdn blocklist</a></li>
|
||||
{{/if}}
|
||||
{{#if isFeature "manage-alerts"}}
|
||||
<li><a href="/manage-alerts">manage alerts</a></li>
|
||||
{{/if}}
|
||||
</ul>
|
||||
</li>
|
||||
{{/if}}
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ Router.map(function() {
|
|||
path: '/watchlist',
|
||||
template: 'watchlist',
|
||||
layoutTemplate: 'layout'
|
||||
});
|
||||
});
|
||||
|
||||
this.route('investigations', {
|
||||
path: '/investigations',
|
||||
|
|
@ -201,12 +201,6 @@ Router.map(function() {
|
|||
layoutTemplate: 'layout'
|
||||
});
|
||||
|
||||
this.route('manage-alerts', {
|
||||
path: '/manage-alerts',
|
||||
template: 'alertschedules',
|
||||
layoutTemplate: 'layout'
|
||||
});
|
||||
|
||||
this.route('ipwhois', {
|
||||
path: '/ipwhois/:_ipaddress',
|
||||
template: 'ipwhois',
|
||||
|
|
|
|||
|
|
@ -35,9 +35,6 @@ Copyright (c) 2014 Mozilla Corporation
|
|||
{{#if isFeature "fqdnblocklist"}}
|
||||
<li><a href="/fqdnblocklist">fqdn blocklist</a></li>
|
||||
{{/if}}
|
||||
{{#if isFeature "manage-alerts"}}
|
||||
<li><a href="/manage-alerts">manage alerts</a></li>
|
||||
{{/if}}
|
||||
</ul>
|
||||
</li>
|
||||
{{/if}}
|
||||
|
|
|
|||
|
|
@ -71,11 +71,6 @@ Copyright (c) 2014 Mozilla Corporation
|
|||
<a href="/fqdnblocklist">fqdn blocklist</a>
|
||||
</li>
|
||||
{{/if}}
|
||||
{{#if isFeature "manage-alerts"}}
|
||||
<li>
|
||||
<a href="/manage-alerts">manage alerts</a>
|
||||
</li>
|
||||
{{/if}}
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ Meteor.startup( () => {
|
|||
fqdnblocklist = new Mongo.Collection( "fqdnblocklist" );
|
||||
watchlist = new Mongo.Collection( "watchlist" );
|
||||
preferences = new Mongo.Collection( "preferences" );
|
||||
alertschedules = new Mongo.Collection( "alertschedules" );
|
||||
|
||||
|
||||
if ( Meteor.isServer ) {
|
||||
|
|
@ -292,16 +291,11 @@ Meteor.startup( () => {
|
|||
publishPagination( fqdnblocklist );
|
||||
publishPagination( watchlist );
|
||||
publishPagination( alerts );
|
||||
publishPagination( alertschedules );
|
||||
|
||||
Meteor.publish( "preferences", function() {
|
||||
return preferences.find( {}, { limit: 0 } );
|
||||
} )
|
||||
|
||||
Meteor.publish( "alertschedules", function() {
|
||||
return alertschedules.find( {}, { limit: 0 } );
|
||||
} );
|
||||
|
||||
//access rules from clients
|
||||
//barebones to allow you to specify rules
|
||||
|
||||
|
|
@ -410,12 +404,6 @@ Meteor.startup( () => {
|
|||
fetch: ['creator']
|
||||
} );
|
||||
|
||||
alertschedules.allow( {
|
||||
update: function( docId, doc, fields, modifier ) {
|
||||
return ( docId );
|
||||
}
|
||||
} );
|
||||
|
||||
// since we store email from oidc calls in the profile
|
||||
// deny updates to the profile which is writeable by default
|
||||
// https://docs.meteor.com/api/accounts.html#Meteor-users
|
||||
|
|
|
|||
|
|
@ -10,5 +10,4 @@ logincounts /logincounts
|
|||
attackers /attackers
|
||||
globe /globe
|
||||
about /about
|
||||
blockip /blockip
|
||||
manage-alerts /manage-alerts
|
||||
blockip /blockip
|
||||
|
|
@ -238,54 +238,6 @@ def index():
|
|||
return response
|
||||
|
||||
|
||||
@route('/alertschedules')
|
||||
@route('/alertschedules/')
|
||||
@enable_cors
|
||||
def index():
|
||||
'''an endpoint to return alert schedules'''
|
||||
if request.body:
|
||||
request.body.read()
|
||||
request.body.close()
|
||||
response.content_type = "application/json"
|
||||
mongoclient = MongoClient(options.mongohost, options.mongoport)
|
||||
schedulers_db = mongoclient.meteor['alertschedules']
|
||||
|
||||
mongodb_alerts = schedulers_db.find()
|
||||
alert_schedules_dict = {}
|
||||
for mongodb_alert in mongodb_alerts:
|
||||
mongodb_alert['_id'] = str(mongodb_alert['_id'])
|
||||
alert_schedules_dict[mongodb_alert['name']] = mongodb_alert
|
||||
|
||||
response.body = json.dumps(alert_schedules_dict)
|
||||
response.status = 200
|
||||
return response
|
||||
|
||||
|
||||
@post('/updatealertschedules', methods=['POST'])
|
||||
@post('/updatealertschedules/', methods=['POST'])
|
||||
@enable_cors
|
||||
def update_alert_schedules():
|
||||
'''an endpoint to return alerts schedules'''
|
||||
if not request.body:
|
||||
response.status = 503
|
||||
return response
|
||||
|
||||
alert_schedules = json.loads(request.body.read())
|
||||
request.body.close()
|
||||
|
||||
response.content_type = "application/json"
|
||||
mongoclient = MongoClient(options.mongohost, options.mongoport)
|
||||
schedulers_db = mongoclient.meteor['alertschedules']
|
||||
schedulers_db.remove()
|
||||
|
||||
for alert_name, alert_schedule in alert_schedules.items():
|
||||
logger.debug("Inserting schedule for {0} into mongodb".format(alert_name))
|
||||
schedulers_db.insert(alert_schedule)
|
||||
|
||||
response.status = 200
|
||||
return response
|
||||
|
||||
|
||||
@route('/plugins', methods=['GET'])
|
||||
@route('/plugins/', methods=['GET'])
|
||||
@route('/plugins/<endpoint>', methods=['GET'])
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче