зеркало из https://github.com/mozilla/MozDef.git
Merge pull request #1481 from mpurzynski/stackdriver_final
Stackdriver final
This commit is contained in:
Michal Purzynski
GitHub
Коммит
cce6d78095
|
|
@ -0,0 +1,8 @@
|
|||
[options]
|
||||
prefetch=10
|
||||
esbulksize=10
|
||||
esservers=http://localhost:9200
|
||||
mqprotocol = pubsub
|
||||
credentials_file = "<service credentials file>.json"
|
||||
project_id = "your project id"
|
||||
resource_name = "projects/<your project id>/subscriptions/<sub name>"
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# Copyright (c) 2017 Mozilla Corporation
|
||||
|
||||
import json
|
||||
import sys
|
||||
import socket
|
||||
import time
|
||||
from configlib import getConfig, OptionParser
|
||||
from datetime import datetime
|
||||
from mozdef_util.utilities.toUTC import toUTC
|
||||
from mozdef_util.utilities.logger import logger, initLogger
|
||||
from mozdef_util.elasticsearch_client import (
|
||||
ElasticsearchClient,
|
||||
ElasticsearchBadServer,
|
||||
ElasticsearchInvalidIndex,
|
||||
ElasticsearchException,
|
||||
)
|
||||
from google.cloud import pubsub
|
||||
from lib.plugins import sendEventToPlugins, registerPlugins
|
||||
|
||||
# running under uwsgi?
|
||||
try:
|
||||
import uwsgi
|
||||
|
||||
hasUWSGI = True
|
||||
except ImportError as e:
|
||||
hasUWSGI = False
|
||||
|
||||
|
||||
class PubSubtaskConsumer(object):
|
||||
def __init__(self, esConnection, options):
|
||||
self.esConnection = esConnection
|
||||
self.pluginList = registerPlugins()
|
||||
self.options = options
|
||||
self.scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/pubsub "]
|
||||
self.credentials_file = options.credentials_file
|
||||
|
||||
def run(self):
|
||||
# XXX: fetch from the config file
|
||||
subscriber = pubsub.SubscriberClient.from_service_account_file(self.options.credentials_file)
|
||||
res = subscriber.subscribe(self.options.resource_name, callback=self.onMessage)
|
||||
try:
|
||||
res.result()
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
logger.error(
|
||||
"Received error during subscribing - killing self and my background thread in 5 seconds for uwsgi to bring me back"
|
||||
)
|
||||
time.sleep(5)
|
||||
res.cancel()
|
||||
sys.exit(1)
|
||||
|
||||
def onMessage(self, message):
|
||||
try:
|
||||
# default elastic search metadata for an event
|
||||
metadata = {"index": "events", "id": None}
|
||||
event = {}
|
||||
|
||||
event["receivedtimestamp"] = toUTC(datetime.now()).isoformat()
|
||||
event["mozdefhostname"] = self.options.mozdefhostname
|
||||
|
||||
event["details"] = json.loads(message.data.decode("UTF-8"))
|
||||
|
||||
if "tags" in event["details"]:
|
||||
event["tags"] = event["details"]["tags"].extend([self.options.resource_name])
|
||||
else:
|
||||
event["tags"] = [self.options.resource_name]
|
||||
event["tags"].extend(["pubsub"])
|
||||
|
||||
(event, metadata) = sendEventToPlugins(event, metadata, self.pluginList)
|
||||
# Drop message if plugins set to None
|
||||
if event is None:
|
||||
message.ack()
|
||||
return
|
||||
self.save_event(event, metadata)
|
||||
message.ack()
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
logger.error("Malformed message: %r" % message)
|
||||
message.ack()
|
||||
|
||||
def save_event(self, event, metadata):
|
||||
try:
|
||||
# drop the message if a plug in set it to None
|
||||
# signaling a discard
|
||||
if event is None:
|
||||
return
|
||||
|
||||
# make a json version for posting to elastic search
|
||||
jbody = json.JSONEncoder().encode(event)
|
||||
|
||||
try:
|
||||
bulk = False
|
||||
if self.options.esbulksize != 0:
|
||||
bulk = True
|
||||
|
||||
self.esConnection.save_event(index=metadata["index"], doc_id=metadata["id"], body=jbody, bulk=bulk)
|
||||
|
||||
except (ElasticsearchBadServer, ElasticsearchInvalidIndex) as e:
|
||||
# handle loss of server or race condition with index rotation/creation/aliasing
|
||||
try:
|
||||
self.esConnection = esConnect()
|
||||
return
|
||||
except (ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException) as e:
|
||||
logger.exception("ElasticSearchException: {0} reported while indexing event".format(e))
|
||||
return
|
||||
except ElasticsearchException as e:
|
||||
logger.exception("ElasticSearchException: {0} reported while indexing event".format(e))
|
||||
logger.error("Malformed jbody: %r" % jbody)
|
||||
return
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
logger.error("Malformed message: %r" % event)
|
||||
|
||||
|
||||
def esConnect():
|
||||
"""open or re-open a connection to elastic search"""
|
||||
return ElasticsearchClient((list("{0}".format(s) for s in options.esservers)), options.esbulksize)
|
||||
|
||||
|
||||
def initConfig():
|
||||
# capture the hostname
|
||||
options.mozdefhostname = getConfig("mozdefhostname", socket.gethostname(), options.configfile)
|
||||
|
||||
# elastic search options. set esbulksize to a non-zero value to enable bulk posting, set timeout to post no matter how many events after X seconds.
|
||||
options.esservers = list(getConfig("esservers", "http://localhost:9200", options.configfile).split(","))
|
||||
options.esbulksize = getConfig("esbulksize", 0, options.configfile)
|
||||
options.esbulktimeout = getConfig("esbulktimeout", 30, options.configfile)
|
||||
|
||||
# GCP PubSub options
|
||||
options.resource_name = getConfig("resource_name", "", options.configfile)
|
||||
options.credentials_file = getConfig("credentials_file", "", options.configfile)
|
||||
|
||||
options.mqprotocol = getConfig("mqprotocol", "pubsub", options.configfile)
|
||||
|
||||
|
||||
def main():
|
||||
if hasUWSGI:
|
||||
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
|
||||
else:
|
||||
logger.info("started without uwsgi")
|
||||
|
||||
if options.mqprotocol not in ("pubsub"):
|
||||
logger.error("Can only process pubsub queues, terminating")
|
||||
sys.exit(1)
|
||||
|
||||
# connect to GCP and consume our queue
|
||||
PubSubtaskConsumer(es, options).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# configure ourselves
|
||||
parser = OptionParser()
|
||||
parser.add_option(
|
||||
"-c", dest="configfile", default=sys.argv[0].replace(".py", ".conf"), help="configuration file to use"
|
||||
)
|
||||
(options, args) = parser.parse_args()
|
||||
initConfig()
|
||||
initLogger(options)
|
||||
|
||||
# open ES connection globally so we don't waste time opening it per message
|
||||
es = esConnect()
|
||||
|
||||
main()
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt as e:
|
||||
logger.info("Exiting worker")
|
||||
if options.esbulksize != 0:
|
||||
es.finish_bulk()
|
||||
except Exception as e:
|
||||
if options.esbulksize != 0:
|
||||
es.finish_bulk()
|
||||
raise
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
# Copyright (c) 2017 Mozilla Corporation
|
||||
|
||||
import urllib
|
||||
from mozdef_util.utilities.toUTC import toUTC
|
||||
|
||||
|
||||
class message(object):
|
||||
def __init__(self):
|
||||
"""
|
||||
Plugin used to fix object type discretions with cloudtrail messages
|
||||
"""
|
||||
self.registration = ["pubsub"]
|
||||
self.priority = 5
|
||||
|
||||
def onMessage(self, message, metadata):
|
||||
# trust no one mr mulder
|
||||
if "tags" not in message:
|
||||
return (message, metadata)
|
||||
if "pubsub" not in message["tags"]:
|
||||
return (message, metadata)
|
||||
if "details" not in message:
|
||||
return (message, metadata)
|
||||
|
||||
event = message["details"]
|
||||
|
||||
if "logName" not in event:
|
||||
return (message, metadata)
|
||||
else:
|
||||
# XXX: implement filtering of audit types that we want to see (yaml)
|
||||
newmessage = dict()
|
||||
logtype = "UNKNOWN"
|
||||
if "logName" in event:
|
||||
logtype = urllib.parse.unquote(event["logName"]).split("/")[-1].strip()
|
||||
if "protoPayload" in event:
|
||||
if "@type" in event["protoPayload"]:
|
||||
if event["protoPayload"]["@type"] == "type.googleapis.com/google.cloud.audit.AuditLog":
|
||||
newmessage["category"] = logtype
|
||||
newmessage["source"] = "stackdriver"
|
||||
newmessage["tags"] = message["tags"] + ["stackdriver"]
|
||||
elif "jsonPayload" in event:
|
||||
if "logName" in event:
|
||||
if logtype == "activity_log":
|
||||
newmessage["category"] = "gceactivity"
|
||||
newmessage["source"] = "stackdriver"
|
||||
newmessage["tags"] = message["tags"] + ["stackdriver"]
|
||||
elif "textPayload" in event:
|
||||
if "logName" in event:
|
||||
if logtype == "syslog":
|
||||
newmessage["category"] = logtype
|
||||
newmessage["source"] = "stackdriver"
|
||||
newmessage["tags"] = message["tags"] + ["stackdriver"]
|
||||
|
||||
newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
|
||||
newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat()
|
||||
newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat()
|
||||
newmessage["mozdefhostname"] = message["mozdefhostname"]
|
||||
newmessage["customendpoint"] = ""
|
||||
newmessage["details"] = event
|
||||
|
||||
return (newmessage, metadata)
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
# Copyright (c) 2017 Mozilla Corporation
|
||||
|
||||
from mozdef_util.utilities.toUTC import toUTC
|
||||
import os
|
||||
import yaml
|
||||
import jmespath
|
||||
|
||||
|
||||
class message(object):
|
||||
def __init__(self):
|
||||
"""
|
||||
Plugin used to fix object type discretions with cloudtrail messages
|
||||
"""
|
||||
self.registration = ["stackdriver"]
|
||||
self.priority = 15
|
||||
|
||||
with open(os.path.join(os.path.dirname(__file__), "stackdriver_audit.yml"), "r") as f:
|
||||
mapping_map = f.read()
|
||||
|
||||
yap = yaml.safe_load(mapping_map)
|
||||
self.eventtypes = list(yap.keys())
|
||||
self.yap = yap
|
||||
del mapping_map
|
||||
|
||||
def onMessage(self, message, metadata):
|
||||
if "tags" not in message:
|
||||
return (message, metadata)
|
||||
if "stackdriver" not in message["tags"]:
|
||||
return (message, metadata)
|
||||
if "category" not in message:
|
||||
return (message, metadata)
|
||||
# XXX: move into a config file
|
||||
cats = ["activity", "data_access"]
|
||||
if message["category"] not in cats:
|
||||
return (message, metadata)
|
||||
|
||||
newmessage = dict()
|
||||
|
||||
newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
|
||||
newmessage["timestamp"] = toUTC(message["details"]["timestamp"]).isoformat()
|
||||
newmessage["utctimestamp"] = toUTC(message["details"]["timestamp"]).isoformat()
|
||||
newmessage["category"] = message["category"]
|
||||
newmessage["tags"] = message["tags"]
|
||||
newmessage["source"] = message["source"]
|
||||
newmessage["mozdefhostname"] = message["mozdefhostname"]
|
||||
newmessage["customendpoint"] = ""
|
||||
newmessage["details"] = {}
|
||||
newmessage["details"] = message["details"]
|
||||
newmessage["details"]["gaudit"] = newmessage["details"]["protoPayload"]
|
||||
del newmessage["details"]["protoPayload"]
|
||||
# Stuff fields that will be used as a summary with something, anything. The mapping function will hopefuly find something to overwrite it with.
|
||||
newmessage["details"]["username"] = "UNKNOWN"
|
||||
newmessage["details"]["resourcename"] = "UNKNOWN"
|
||||
if "request" in newmessage["details"]["gaudit"]:
|
||||
if "resource" in newmessage["details"]["gaudit"]["request"]:
|
||||
if type(newmessage["details"]["gaudit"]["request"]["resource"]) is not dict:
|
||||
del newmessage["details"]["gaudit"]["request"]["resource"]
|
||||
|
||||
if message["category"] in self.eventtypes:
|
||||
for key in self.yap[newmessage["category"]]:
|
||||
mappedvalue = jmespath.search(self.yap[newmessage["category"]][key], newmessage)
|
||||
# JMESPath likes to silently return a None object
|
||||
if mappedvalue is not None:
|
||||
newmessage["details"][key] = mappedvalue
|
||||
|
||||
# This is redundant
|
||||
newmessage["summary"] = "{0} executed {1} on {2}".format(
|
||||
newmessage["details"]["username"], newmessage["details"]["action"], newmessage["details"]["resourcename"]
|
||||
)
|
||||
|
||||
return (newmessage, metadata)
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
---
|
||||
data_access:
|
||||
username: details.gaudit.authenticationInfo.principalEmail
|
||||
action: details.gaudit.methodName
|
||||
service: details.gaudit.serviceName
|
||||
serviceData: details.gaudit.serviceData
|
||||
resourcename: details.gaudit.resourceName
|
||||
resourcetype: details.resource.type
|
||||
projectid: details.resource.labels.project_id
|
||||
logname: details.logName
|
||||
sdreceivetimestamp: details.receiveTimestamp
|
||||
sourceipaddress: details.gaudit.requestMetadata.callerIp
|
||||
|
||||
activity:
|
||||
username: details.gaudit.authenticationInfo.principalEmail
|
||||
action: details.gaudit.methodName
|
||||
service: details.gaudit.serviceName
|
||||
serviceData: details.gaudit.serviceData
|
||||
resourcename: details.gaudit.resourceName
|
||||
resourcetype: details.resource.type
|
||||
projectid: details.resource.labels.project_id
|
||||
logname: details.logName
|
||||
sdreceivetimestamp: details.receiveTimestamp
|
||||
sourceipaddress: details.gaudit.requestMetadata.callerIp
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
# Copyright (c) 2017 Mozilla Corporation
|
||||
|
||||
from mozdef_util.utilities.toUTC import toUTC
|
||||
import os
|
||||
import yaml
|
||||
import jmespath
|
||||
|
||||
|
||||
class message(object):
|
||||
def __init__(self):
|
||||
"""
|
||||
Plugin used to fix object type discretions with cloudtrail messages
|
||||
"""
|
||||
self.registration = ["stackdriver"]
|
||||
self.priority = 16
|
||||
|
||||
with open(os.path.join(os.path.dirname(__file__), "stackdriver_gceactivity.yml"), "r") as f:
|
||||
mapping_map = f.read()
|
||||
|
||||
yap = yaml.safe_load(mapping_map)
|
||||
self.eventtypes = list(yap.keys())
|
||||
self.yap = yap
|
||||
del mapping_map
|
||||
|
||||
def onMessage(self, message, metadata):
|
||||
if "tags" not in message:
|
||||
return (message, metadata)
|
||||
if "stackdriver" not in message["tags"]:
|
||||
return (message, metadata)
|
||||
if "category" not in message:
|
||||
return (message, metadata)
|
||||
# XXX: move into a config file
|
||||
cats = ["gceactivity"]
|
||||
if message["category"] not in cats:
|
||||
return (message, metadata)
|
||||
|
||||
event = message["details"]
|
||||
|
||||
newmessage = dict()
|
||||
|
||||
newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
|
||||
newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat()
|
||||
newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat()
|
||||
newmessage["category"] = message["category"]
|
||||
newmessage["tags"] = message["tags"]
|
||||
newmessage["source"] = message["source"]
|
||||
newmessage["mozdefhostname"] = message["mozdefhostname"]
|
||||
newmessage["customendpoint"] = ""
|
||||
newmessage["details"] = {}
|
||||
newmessage["details"] = message["details"]
|
||||
newmessage["details"]["gceactivity"] = newmessage["details"]["jsonPayload"]
|
||||
del newmessage["details"]["jsonPayload"]
|
||||
# This is fake
|
||||
newmessage["details"]["service"] = "compute.googleapis.com"
|
||||
# Stuff fields that will be used as a summary with something, anything. The mapping function will hopefuly find something to overwrite it with.
|
||||
newmessage["details"]["username"] = "UNKNOWN"
|
||||
newmessage["details"]["resourcename"] = "UNKNOWN"
|
||||
|
||||
if message["category"] in self.eventtypes:
|
||||
for key in self.yap[newmessage["category"]]:
|
||||
mappedvalue = jmespath.search(self.yap[newmessage["category"]][key], newmessage)
|
||||
# JMESPath likes to silently return a None object
|
||||
if mappedvalue is not None:
|
||||
newmessage["details"][key] = mappedvalue
|
||||
|
||||
# This is redundant
|
||||
newmessage["summary"] = "{0} executed {1} on {2}".format(
|
||||
newmessage["details"]["username"], newmessage["details"]["action"], newmessage["details"]["resourcename"],
|
||||
)
|
||||
|
||||
return (newmessage, metadata)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
gceactivity:
|
||||
username: details.gceactivity.actor.user
|
||||
action: details.gceactivity.event_subtype
|
||||
actiongroup: details.gceactivity.event_type
|
||||
resourcename: details.gceactivity.resource.name
|
||||
resourcetype: details.gceactivity.resource.type
|
||||
resourceid: details.gceactivity.resource.id
|
||||
projectid: details.resource.labels.project_id
|
||||
logname: details.logName
|
||||
sdreceivetimestamp: details.receiveTimestamp
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
syslog:
|
||||
details.findingid: details.id
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
# Copyright (c) 2017 Mozilla Corporation
|
||||
|
||||
from mozdef_util.utilities.toUTC import toUTC
|
||||
|
||||
|
||||
class message(object):
|
||||
def __init__(self):
|
||||
"""
|
||||
Plugin used to fix object type discretions with cloudtrail messages
|
||||
"""
|
||||
self.registration = ["stackdriver"]
|
||||
self.priority = 15
|
||||
|
||||
def onMessage(self, message, metadata):
|
||||
if "tags" not in message:
|
||||
return (message, metadata)
|
||||
if "stackdriver" not in message["tags"]:
|
||||
return (message, metadata)
|
||||
if "category" not in message:
|
||||
return (message, metadata)
|
||||
if message["category"] != "syslog":
|
||||
return (message, metadata)
|
||||
|
||||
event = message["details"]
|
||||
newmessage = dict()
|
||||
|
||||
newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
|
||||
newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat()
|
||||
newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat()
|
||||
newmessage["category"] = "syslog"
|
||||
newmessage["tags"] = message["tags"]
|
||||
newmessage["source"] = message["source"]
|
||||
newmessage["mozdefhostname"] = message["mozdefhostname"]
|
||||
newmessage["customendpoint"] = ""
|
||||
if "facility" in event:
|
||||
newmessage["facility"] = event["facility"]
|
||||
if "severity" in event:
|
||||
newmessage["severity"] = event["severity"]
|
||||
|
||||
line = event["textPayload"].split()
|
||||
newmessage["hostname"] = line[3]
|
||||
newmessage["processname"] = line[4].strip(":")
|
||||
newmessage["summary"] = " ".join(line[5:])
|
||||
|
||||
return (newmessage, metadata)
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
from mozdef_util.utilities.toUTC import toUTC
|
||||
from mq.plugins.stackdriver import message
|
||||
|
||||
|
||||
class TestStackDriver(object):
|
||||
def setup(self):
|
||||
self.plugin = message()
|
||||
self.metadata = {"index": "events"}
|
||||
|
||||
# Should never match and be modified by the plugin
|
||||
def test_nodetails_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {"tags": "pubsub"}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def verify_metadata(self, metadata):
|
||||
assert metadata["index"] == "events"
|
||||
|
||||
def verify_defaults(self, result):
|
||||
assert result["category"] == "data_access"
|
||||
assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
|
||||
|
||||
def test_defaults(self):
|
||||
event = {
|
||||
"receivedtimestamp": "2019-11-21T22:43:10.041549+00:00",
|
||||
"mozdefhostname": "mozdefqa2.private.mdc1.mozilla.com",
|
||||
"details": {
|
||||
"insertId": "-81ga0vdqblo",
|
||||
"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access",
|
||||
"protoPayload": {
|
||||
"@type": "type.googleapis.com/google.cloud.audit.AuditLog",
|
||||
"authenticationInfo": {"principalEmail": "mpurzynski@gcp.infra.mozilla.com"},
|
||||
"authorizationInfo": [
|
||||
{
|
||||
"granted": True,
|
||||
"permission": "compute.instances.list",
|
||||
"resourceAttributes": {
|
||||
"name": "projects/mcd-001-252615",
|
||||
"service": "resourcemanager",
|
||||
"type": "resourcemanager.projects",
|
||||
},
|
||||
}
|
||||
],
|
||||
"methodName": "beta.compute.instances.aggregatedList",
|
||||
"numResponseItems": "61",
|
||||
"request": {"@type": "type.googleapis.com/compute.instances.aggregatedList"},
|
||||
"requestMetadata": {
|
||||
"callerIp": "2620:101:80fb:224:2864:cebc:a1e:640c",
|
||||
"callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)",
|
||||
"destinationAttributes": {},
|
||||
"requestAttributes": {"auth": {}, "time": "2019-11-21T22:42:26.336Z",},
|
||||
},
|
||||
"resourceLocation": {"currentLocations": ["global"]},
|
||||
"resourceName": "projects/mcd-001-252615/global/instances",
|
||||
"serviceName": "compute.googleapis.com",
|
||||
},
|
||||
"receiveTimestamp": "2019-11-21T22:42:26.904624537Z",
|
||||
"resource": {
|
||||
"labels": {
|
||||
"location": "global",
|
||||
"method": "compute.instances.aggregatedList",
|
||||
"project_id": "mcd-001-252615",
|
||||
"service": "compute.googleapis.com",
|
||||
"version": "beta",
|
||||
},
|
||||
"type": "api",
|
||||
},
|
||||
"severity": "INFO",
|
||||
"timestamp": "2019-11-21T22:42:25.759Z",
|
||||
},
|
||||
"tags": ["projects/mcd-001-252615/subscriptions/mozdefsubscription", "pubsub",],
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
self.verify_defaults(result)
|
||||
self.verify_metadata(metadata)
|
||||
|
||||
def test_nomatch_syslog(self):
|
||||
event = {
|
||||
"category": "syslog",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
|
||||
"severity": "7",
|
||||
"utctimestamp": "2017-09-26T00:22:23+00:00",
|
||||
"timestamp": "2017-09-26T00:22:23+00:00",
|
||||
"hostname": "something1.test.com",
|
||||
"mozdefhostname": "something1.test.com",
|
||||
"summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
|
||||
"eventsource": "systemslogs",
|
||||
"tags": "something",
|
||||
"details": {
|
||||
"processid": "21233",
|
||||
"sourceipv4address": "10.22.74.208",
|
||||
"hostname": "hostname1.subdomain.domain.com",
|
||||
"program": "sshd",
|
||||
"sourceipaddress": "10.22.74.208",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "syslog"
|
||||
assert result["eventsource"] == "systemslogs"
|
||||
assert result == event
|
||||
|
||||
def test_nomatch_auditd(self):
|
||||
event = {
|
||||
"category": "execve",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
|
||||
"severity": "INFO",
|
||||
"utctimestamp": "2017-09-26T00:36:27+00:00",
|
||||
"tags": ["audisp-json", "2.1.1", "audit"],
|
||||
"summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"processname": "audisp-json",
|
||||
"details": {
|
||||
"fsuid": "398",
|
||||
"tty": "(none)",
|
||||
"uid": "398",
|
||||
"process": "/bin/bash",
|
||||
"auditkey": "exec",
|
||||
"pid": "10553",
|
||||
"processname": "sh",
|
||||
"session": "16467",
|
||||
"fsgid": "398",
|
||||
"sgid": "398",
|
||||
"auditserial": "3834716",
|
||||
"inode": "1835094",
|
||||
"ouid": "0",
|
||||
"ogid": "0",
|
||||
"suid": "398",
|
||||
"originaluid": "0",
|
||||
"gid": "398",
|
||||
"originaluser": "pubsub",
|
||||
"ppid": "10552",
|
||||
"cwd": "/",
|
||||
"parentprocess": "stackdriver",
|
||||
"euid": "398",
|
||||
"path": "/bin/sh",
|
||||
"rdev": "00:00",
|
||||
"dev": "08:03",
|
||||
"egid": "398",
|
||||
"command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"mode": "0100755",
|
||||
"user": "squid",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "execve"
|
||||
assert "eventsource" not in result
|
||||
assert result == event
|
||||
|
||||
def test_stackdriver(self):
|
||||
event = {
|
||||
"receivedtimestamp": "2019-11-21T22:43:10.041549+00:00",
|
||||
"mozdefhostname": "mozdefqa2.private.mdc1.mozilla.com",
|
||||
"details": {
|
||||
"insertId": "-81ga0vdqblo",
|
||||
"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access",
|
||||
"protoPayload": {
|
||||
"@type": "type.googleapis.com/google.cloud.audit.AuditLog",
|
||||
"authenticationInfo": {"principalEmail": "mpurzynski@gcp.infra.mozilla.com"},
|
||||
"authorizationInfo": [
|
||||
{
|
||||
"granted": True,
|
||||
"permission": "compute.instances.list",
|
||||
"resourceAttributes": {
|
||||
"name": "projects/mcd-001-252615",
|
||||
"service": "resourcemanager",
|
||||
"type": "resourcemanager.projects",
|
||||
},
|
||||
}
|
||||
],
|
||||
"methodName": "beta.compute.instances.aggregatedList",
|
||||
"numResponseItems": "61",
|
||||
"request": {"@type": "type.googleapis.com/compute.instances.aggregatedList"},
|
||||
"requestMetadata": {
|
||||
"callerIp": "2620:101:80fb:224:2864:cebc:a1e:640c",
|
||||
"callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)",
|
||||
"destinationAttributes": {},
|
||||
"requestAttributes": {"auth": {}, "time": "2019-11-21T22:42:26.336Z",},
|
||||
},
|
||||
"resourceLocation": {"currentLocations": ["global"]},
|
||||
"resourceName": "projects/mcd-001-252615/global/instances",
|
||||
"serviceName": "compute.googleapis.com",
|
||||
},
|
||||
"receiveTimestamp": "2019-11-21T22:42:26.904624537Z",
|
||||
"resource": {
|
||||
"labels": {
|
||||
"location": "global",
|
||||
"method": "compute.instances.aggregatedList",
|
||||
"project_id": "mcd-001-252615",
|
||||
"service": "compute.googleapis.com",
|
||||
"version": "beta",
|
||||
},
|
||||
"type": "api",
|
||||
},
|
||||
"severity": "INFO",
|
||||
"timestamp": "2019-11-21T22:42:25.759Z",
|
||||
},
|
||||
"tags": ["projects/mcd-001-252615/subscriptions/mozdefsubscription", "pubsub",],
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "data_access"
|
||||
assert result["details"]["protoPayload"]["@type"] == "type.googleapis.com/google.cloud.audit.AuditLog"
|
||||
|
|
@ -0,0 +1,326 @@
|
|||
from mozdef_util.utilities.toUTC import toUTC
|
||||
from mq.plugins.stackdriver_audit import message
|
||||
|
||||
|
||||
class TestStackDriverAudit(object):
|
||||
def setup(self):
|
||||
self.plugin = message()
|
||||
self.metadata = {"index": "events"}
|
||||
|
||||
# Should never match and be modified by the plugin
|
||||
def test_notags_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {
|
||||
"source": "stackdriver",
|
||||
"details": {"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access"},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def test_wrongtags_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {
|
||||
"tags": "audit",
|
||||
"source": "stackdriver",
|
||||
"details": {"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access"},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def verify_metadata(self, metadata):
|
||||
assert metadata["index"] == "events"
|
||||
|
||||
def verify_defaults(self, result):
|
||||
assert result["category"] == "data_access"
|
||||
assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
|
||||
|
||||
def test_defaults(self):
|
||||
event = {
|
||||
'category': 'data_access',
|
||||
'source': 'stackdriver',
|
||||
'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
|
||||
'receivedtimestamp': '2019-11-21T23:53:36.695909+00:00',
|
||||
'timestamp': '2019-11-21T23:45:34.930000+00:00',
|
||||
'utctimestamp': '2019-11-21T23:45:34.930000+00:00',
|
||||
'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
|
||||
'customendpoint': '',
|
||||
'details': {
|
||||
'insertId': 'utar0xd1qjq',
|
||||
'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access',
|
||||
'protoPayload': {
|
||||
'@type': 'type.googleapis.com/google.cloud.audit.AuditLog',
|
||||
'authenticationInfo': {},
|
||||
'authorizationInfo': [
|
||||
{
|
||||
'permission': 'storage.buckets.get',
|
||||
'resource': 'projects/_/buckets/mcd-001-252615.appspot.com',
|
||||
'resourceAttributes': {},
|
||||
},
|
||||
{
|
||||
'permission': 'storage.buckets.getIamPolicy',
|
||||
'resource': 'projects/_/buckets/mcd-001-252615.appspot.com',
|
||||
'resourceAttributes': {},
|
||||
},
|
||||
],
|
||||
'methodName': 'storage.buckets.get',
|
||||
'requestMetadata': {
|
||||
'destinationAttributes': {},
|
||||
'requestAttributes': {'auth': {}, 'time': '2019-11-21T23:45:34.949Z'},
|
||||
},
|
||||
'resourceLocation': {'currentLocations': ['asia-northeast2']},
|
||||
'resourceName': 'projects/_/buckets/mcd-001-252615.appspot.com',
|
||||
'serviceName': 'storage.googleapis.com',
|
||||
'status': {'code': 7, 'message': 'PERMISSION_DENIED'},
|
||||
},
|
||||
'receiveTimestamp': '2019-11-21T23:45:35.521115967Z',
|
||||
'resource': {
|
||||
'labels': {
|
||||
'bucket_name': 'mcd-001-252615.appspot.com',
|
||||
'location': 'asia-northeast2',
|
||||
'project_id': 'mcd-001-252615',
|
||||
},
|
||||
'type': 'gcs_bucket',
|
||||
},
|
||||
'severity': 'ERROR',
|
||||
'timestamp': '2019-11-21T23:45:34.93Z',
|
||||
},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
self.verify_defaults(result)
|
||||
self.verify_metadata(metadata)
|
||||
|
||||
def test_nomatch_syslog(self):
|
||||
event = {
|
||||
"category": "syslog",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
|
||||
"severity": "7",
|
||||
"utctimestamp": "2017-09-26T00:22:23+00:00",
|
||||
"timestamp": "2017-09-26T00:22:23+00:00",
|
||||
"hostname": "something1.test.com",
|
||||
"mozdefhostname": "something1.test.com",
|
||||
"summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
|
||||
"eventsource": "systemslogs",
|
||||
"tags": "something",
|
||||
"details": {
|
||||
"processid": "21233",
|
||||
"sourceipv4address": "10.22.74.208",
|
||||
"hostname": "hostname1.subdomain.domain.com",
|
||||
"program": "sshd",
|
||||
"sourceipaddress": "10.22.74.208",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "syslog"
|
||||
assert result["eventsource"] == "systemslogs"
|
||||
assert result == event
|
||||
|
||||
def test_nomatch_auditd(self):
|
||||
event = {
|
||||
"category": "execve",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
|
||||
"severity": "INFO",
|
||||
"utctimestamp": "2017-09-26T00:36:27+00:00",
|
||||
"tags": ["audisp-json", "2.1.1", "audit"],
|
||||
"summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"processname": "audisp-json",
|
||||
"details": {
|
||||
"fsuid": "398",
|
||||
"tty": "(none)",
|
||||
"uid": "398",
|
||||
"process": "/bin/bash",
|
||||
"auditkey": "exec",
|
||||
"pid": "10553",
|
||||
"processname": "sh",
|
||||
"session": "16467",
|
||||
"fsgid": "398",
|
||||
"sgid": "398",
|
||||
"auditserial": "3834716",
|
||||
"inode": "1835094",
|
||||
"ouid": "0",
|
||||
"ogid": "0",
|
||||
"suid": "398",
|
||||
"originaluid": "0",
|
||||
"gid": "398",
|
||||
"originaluser": "pubsub",
|
||||
"ppid": "10552",
|
||||
"cwd": "/",
|
||||
"parentprocess": "stackdriver",
|
||||
"euid": "398",
|
||||
"path": "/bin/sh",
|
||||
"rdev": "00:00",
|
||||
"dev": "08:03",
|
||||
"egid": "398",
|
||||
"command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"mode": "0100755",
|
||||
"user": "squid",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "execve"
|
||||
assert "eventsource" not in result
|
||||
assert result == event
|
||||
|
||||
def test_stackdriver_audit_data_access(self):
|
||||
event = {
|
||||
'category': 'data_access',
|
||||
'source': 'stackdriver',
|
||||
'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
|
||||
'receivedtimestamp': '2019-11-21T22:43:10.041549+00:00',
|
||||
'timestamp': '2019-11-21T22:42:25.759000+00:00',
|
||||
'utctimestamp': '2019-11-21T22:42:25.759000+00:00',
|
||||
'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
|
||||
'customendpoint': '',
|
||||
'details': {
|
||||
'insertId': '-81ga0vdqblo',
|
||||
'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access',
|
||||
'protoPayload': {
|
||||
'@type': 'type.googleapis.com/google.cloud.audit.AuditLog',
|
||||
'authenticationInfo': {'principalEmail': '732492844671-compute@developer.gserviceaccount.com'},
|
||||
'authorizationInfo': [
|
||||
{
|
||||
'granted': True,
|
||||
'permission': 'compute.instances.list',
|
||||
'resourceAttributes': {
|
||||
'name': 'projects/mcd-001-252615',
|
||||
'service': 'resourcemanager',
|
||||
'type': 'resourcemanager.projects',
|
||||
},
|
||||
}
|
||||
],
|
||||
'methodName': 'beta.compute.instances.aggregatedList',
|
||||
'numResponseItems': '61',
|
||||
'request': {'@type': 'type.googleapis.com/compute.instances.aggregatedList'},
|
||||
'requestMetadata': {
|
||||
'callerIp': '2620:101:80fb:224:2864:cebc:a1e:640c',
|
||||
'callerSuppliedUserAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
|
||||
'destinationAttributes': {},
|
||||
'requestAttributes': {'auth': {}, 'time': '2019-11-21T22:42:26.336Z'},
|
||||
},
|
||||
'resourceLocation': {'currentLocations': ['global']},
|
||||
'resourceName': 'projects/mcd-001-252615/global/instances',
|
||||
'serviceName': 'compute.googleapis.com',
|
||||
},
|
||||
'receiveTimestamp': '2019-11-21T22:42:26.904624537Z',
|
||||
'resource': {
|
||||
'labels': {
|
||||
'location': 'global',
|
||||
'method': 'compute.instances.aggregatedList',
|
||||
'project_id': 'mcd-001-252615',
|
||||
'service': 'compute.googleapis.com',
|
||||
'version': 'beta',
|
||||
},
|
||||
'type': 'api',
|
||||
},
|
||||
'severity': 'INFO',
|
||||
'timestamp': '2019-11-21T22:42:25.759Z',
|
||||
},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "data_access"
|
||||
assert result["details"]["action"] == "beta.compute.instances.aggregatedList"
|
||||
assert result["details"]["gaudit"]["authenticationInfo"]["principalEmail"] == "732492844671-compute@developer.gserviceaccount.com"
|
||||
assert result["details"]["gaudit"]["methodName"] == "beta.compute.instances.aggregatedList"
|
||||
assert result["details"]["projectid"] == "mcd-001-252615"
|
||||
assert result["details"]["service"] == "compute.googleapis.com"
|
||||
assert result["details"]["sourceipaddress"] == "2620:101:80fb:224:2864:cebc:a1e:640c"
|
||||
assert result["details"]["username"] == "732492844671-compute@developer.gserviceaccount.com"
|
||||
assert result["source"] == "stackdriver"
|
||||
assert result["utctimestamp"] == "2019-11-21T22:42:25.759000+00:00"
|
||||
assert "protoPayload" not in result["details"]
|
||||
|
||||
def test_stackdriver_audit_activity(self):
|
||||
event = {
|
||||
'category': 'activity',
|
||||
'source': 'stackdriver',
|
||||
'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
|
||||
'receivedtimestamp': '2019-11-22T00:03:20.621831+00:00',
|
||||
'timestamp': '2019-11-22T00:03:18.137000+00:00',
|
||||
'utctimestamp': '2019-11-22T00:03:18.137000+00:00',
|
||||
'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
|
||||
'customendpoint': '',
|
||||
'details': {
|
||||
'insertId': '8w7e9jdcf16',
|
||||
'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Factivity',
|
||||
'operation': {
|
||||
'first': True,
|
||||
'id': 'operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27',
|
||||
'producer': 'type.googleapis.com',
|
||||
},
|
||||
'protoPayload': {
|
||||
'@type': 'type.googleapis.com/google.cloud.audit.AuditLog',
|
||||
'authenticationInfo': {'principalEmail': 'onceuponatime@inagalaxynottoofaraway.com'},
|
||||
'authorizationInfo': [
|
||||
{
|
||||
'granted': True,
|
||||
'permission': 'compute.instances.reset',
|
||||
'resourceAttributes': {
|
||||
'name': 'projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1',
|
||||
'service': 'compute',
|
||||
'type': 'compute.instances',
|
||||
},
|
||||
}
|
||||
],
|
||||
'methodName': 'v1.compute.instances.reset',
|
||||
'request': {'@type': 'type.googleapis.com/compute.instances.reset'},
|
||||
'requestMetadata': {
|
||||
'callerIp': '2620:101:80fb:224:a889:abf2:7b0b:f928',
|
||||
'callerSuppliedUserAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
|
||||
'destinationAttributes': {},
|
||||
'requestAttributes': {'auth': {}, 'time': '2019-11-22T00:03:18.826Z'},
|
||||
},
|
||||
'resourceLocation': {'currentLocations': ['us-west2-a']},
|
||||
'resourceName': 'projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1',
|
||||
'response': {
|
||||
'@type': 'type.googleapis.com/operation',
|
||||
'id': '868140788263590697',
|
||||
'insertTime': '2019-11-21T16:03:18.588-08:00',
|
||||
'name': 'operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27',
|
||||
'operationType': 'reset',
|
||||
'progress': '0',
|
||||
'selfLink': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/operations/operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27',
|
||||
'selfLinkWithId': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/operations/868140788263590697',
|
||||
'startTime': '2019-11-21T16:03:18.597-08:00',
|
||||
'status': 'RUNNING',
|
||||
'targetId': '3401561556013842918',
|
||||
'targetLink': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1',
|
||||
'user': 'onceuponatime@inagalaxynottoofaraway.com',
|
||||
'zone': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a',
|
||||
},
|
||||
'serviceName': 'compute.googleapis.com',
|
||||
},
|
||||
'receiveTimestamp': '2019-11-22T00:03:19.525805615Z',
|
||||
'resource': {
|
||||
'labels': {
|
||||
'instance_id': '3401561556013842918',
|
||||
'project_id': 'mcd-001-252615',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'type': 'gce_instance',
|
||||
},
|
||||
'severity': 'NOTICE',
|
||||
'timestamp': '2019-11-22T00:03:18.137Z',
|
||||
},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "activity"
|
||||
assert result["details"]["action"] == "v1.compute.instances.reset"
|
||||
assert result["details"]["gaudit"]["authenticationInfo"]["principalEmail"] == "onceuponatime@inagalaxynottoofaraway.com"
|
||||
assert result["details"]["gaudit"]["methodName"] == "v1.compute.instances.reset"
|
||||
assert result["details"]["operation"]["producer"] == "type.googleapis.com"
|
||||
assert result["details"]["projectid"] == "mcd-001-252615"
|
||||
assert result["details"]["resourcename"] == "projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1"
|
||||
assert result["details"]["resourcetype"] == "gce_instance"
|
||||
assert result["details"]["service"] == "compute.googleapis.com"
|
||||
assert result["details"]["username"] == "onceuponatime@inagalaxynottoofaraway.com"
|
||||
assert result["summary"] == "onceuponatime@inagalaxynottoofaraway.com executed v1.compute.instances.reset on projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1"
|
||||
|
||||
assert "protoPayload" not in result["details"]
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
from mozdef_util.utilities.toUTC import toUTC
|
||||
from mq.plugins.stackdriver_gceactivity import message
|
||||
|
||||
|
||||
class TestStackDriverGCEActivity(object):
|
||||
def setup(self):
|
||||
self.plugin = message()
|
||||
self.metadata = {"index": "events"}
|
||||
|
||||
def test_notags_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {"category": "gceactivity"}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def test_nocategory_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {"tags": "audit"}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
# Should never match and be modified by the plugin
|
||||
def test_wrongtags_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {"tags": "audit", "category": "gceactivity"}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def verify_metadata(self, metadata):
|
||||
assert metadata["index"] == "events"
|
||||
|
||||
def verify_defaults(self, result):
|
||||
assert result["category"] == "gceactivity"
|
||||
assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
|
||||
|
||||
def test_defaults(self):
|
||||
event = {
|
||||
'category': 'gceactivity',
|
||||
'source': 'stackdriver',
|
||||
'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
|
||||
'receivedtimestamp': '2019-11-22T01:23:49.238723+00:00',
|
||||
'timestamp': '2019-11-22T01:23:47.936931+00:00',
|
||||
'utctimestamp': '2019-11-22T01:23:47.936931+00:00',
|
||||
'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
|
||||
'customendpoint': '',
|
||||
'details': {
|
||||
'insertId': '1y7iw8ag15tmjpz',
|
||||
'jsonPayload': {
|
||||
'actor': {'user': 'luke@or.not'},
|
||||
'event_subtype': 'compute.instances.reset',
|
||||
'event_timestamp_us': '1574385827936931',
|
||||
'event_type': 'GCE_API_CALL',
|
||||
'ip_address': '',
|
||||
'operation': {
|
||||
'id': '2169930274576172620',
|
||||
'name': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
|
||||
'type': 'operation',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'request': {
|
||||
'body': 'null',
|
||||
'url': 'https://compute.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1/reset?key=AIzaSyDWUi9T78xEO-m10evQANR7TMSiB_bjyNc',
|
||||
},
|
||||
'resource': {
|
||||
'id': '3401561556013842918',
|
||||
'name': 'mozdefdevvm1',
|
||||
'type': 'instance',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'trace_id': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
|
||||
'user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
|
||||
'version': '1.2',
|
||||
},
|
||||
'labels': {
|
||||
'compute.googleapis.com/resource_id': '3401561556013842918',
|
||||
'compute.googleapis.com/resource_name': 'mozdefdevvm1',
|
||||
'compute.googleapis.com/resource_type': 'instance',
|
||||
'compute.googleapis.com/resource_zone': 'us-west2-a',
|
||||
},
|
||||
'logName': 'projects/mcd-001-252615/logs/compute.googleapis.com%2Factivity_log',
|
||||
'receiveTimestamp': '2019-11-22T01:23:47.988998161Z',
|
||||
'resource': {
|
||||
'labels': {
|
||||
'instance_id': '3401561556013842918',
|
||||
'project_id': 'mcd-001-252615',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'type': 'gce_instance',
|
||||
},
|
||||
'severity': 'INFO',
|
||||
'timestamp': '2019-11-22T01:23:47.936931Z',
|
||||
},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
self.verify_defaults(result)
|
||||
self.verify_metadata(metadata)
|
||||
|
||||
def test_nomatch_syslog(self):
|
||||
event = {
|
||||
"category": "syslog",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
|
||||
"severity": "7",
|
||||
"utctimestamp": "2017-09-26T00:22:23+00:00",
|
||||
"timestamp": "2017-09-26T00:22:23+00:00",
|
||||
"hostname": "something1.test.com",
|
||||
"mozdefhostname": "something1.test.com",
|
||||
"summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
|
||||
"eventsource": "systemslogs",
|
||||
"tags": "something",
|
||||
"details": {
|
||||
"processid": "21233",
|
||||
"sourceipv4address": "10.22.74.208",
|
||||
"hostname": "hostname1.subdomain.domain.com",
|
||||
"program": "sshd",
|
||||
"sourceipaddress": "10.22.74.208",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "syslog"
|
||||
assert result["eventsource"] == "systemslogs"
|
||||
assert result == event
|
||||
|
||||
def test_nomatch_auditd(self):
|
||||
event = {
|
||||
"category": "execve",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
|
||||
"severity": "INFO",
|
||||
"utctimestamp": "2017-09-26T00:36:27+00:00",
|
||||
"tags": ["audisp-json", "2.1.1", "audit"],
|
||||
"summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"processname": "audisp-json",
|
||||
"details": {
|
||||
"fsuid": "398",
|
||||
"tty": "(none)",
|
||||
"uid": "398",
|
||||
"process": "/bin/bash",
|
||||
"auditkey": "exec",
|
||||
"pid": "10553",
|
||||
"processname": "sh",
|
||||
"session": "16467",
|
||||
"fsgid": "398",
|
||||
"sgid": "398",
|
||||
"auditserial": "3834716",
|
||||
"inode": "1835094",
|
||||
"ouid": "0",
|
||||
"ogid": "0",
|
||||
"suid": "398",
|
||||
"originaluid": "0",
|
||||
"gid": "398",
|
||||
"originaluser": "pubsub",
|
||||
"ppid": "10552",
|
||||
"cwd": "/",
|
||||
"parentprocess": "stackdriver",
|
||||
"euid": "398",
|
||||
"path": "/bin/sh",
|
||||
"rdev": "00:00",
|
||||
"dev": "08:03",
|
||||
"egid": "398",
|
||||
"command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"mode": "0100755",
|
||||
"user": "squid",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "execve"
|
||||
assert "eventsource" not in result
|
||||
assert result == event
|
||||
|
||||
def test_stackdriver(self):
|
||||
event = {
|
||||
'category': 'gceactivity',
|
||||
'source': 'stackdriver',
|
||||
'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
|
||||
'receivedtimestamp': '2019-11-22T01:23:49.238723+00:00',
|
||||
'timestamp': '2019-11-22T01:23:47.936931+00:00',
|
||||
'utctimestamp': '2019-11-22T01:23:47.936931+00:00',
|
||||
'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
|
||||
'customendpoint': '',
|
||||
'details': {
|
||||
'insertId': '1y7iw8ag15tmjpz',
|
||||
'jsonPayload': {
|
||||
'actor': {'user': 'luke@or.not'},
|
||||
'event_subtype': 'compute.instances.reset',
|
||||
'event_timestamp_us': '1574385827936931',
|
||||
'event_type': 'GCE_API_CALL',
|
||||
'ip_address': '',
|
||||
'operation': {
|
||||
'id': '2169930274576172620',
|
||||
'name': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
|
||||
'type': 'operation',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'request': {
|
||||
'body': 'null',
|
||||
'url': 'https://compute.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1/reset?key=AIzaSyDWUi9T78xEO-m10evQANR7TMSiB_bjyNc',
|
||||
},
|
||||
'resource': {
|
||||
'id': '3401561556013842918',
|
||||
'name': 'mozdefdevvm1',
|
||||
'type': 'instance',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'trace_id': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
|
||||
'user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
|
||||
'version': '1.2',
|
||||
},
|
||||
'labels': {
|
||||
'compute.googleapis.com/resource_id': '3401561556013842918',
|
||||
'compute.googleapis.com/resource_name': 'mozdefdevvm1',
|
||||
'compute.googleapis.com/resource_type': 'instance',
|
||||
'compute.googleapis.com/resource_zone': 'us-west2-a',
|
||||
},
|
||||
'logName': 'projects/mcd-001-252615/logs/compute.googleapis.com%2Factivity_log',
|
||||
'receiveTimestamp': '2019-11-22T01:23:47.988998161Z',
|
||||
'resource': {
|
||||
'labels': {
|
||||
'instance_id': '3401561556013842918',
|
||||
'project_id': 'mcd-001-252615',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'type': 'gce_instance',
|
||||
},
|
||||
'severity': 'INFO',
|
||||
'timestamp': '2019-11-22T01:23:47.936931Z',
|
||||
},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
|
||||
assert result["details"]["action"] == "compute.instances.reset"
|
||||
assert result["details"]["gceactivity"]["resource"]["id"] == "3401561556013842918"
|
||||
assert result["utctimestamp"] == "2019-11-22T01:23:47.936931+00:00"
|
||||
assert result["details"]["username"] == "luke@or.not"
|
||||
assert result["details"]["service"] == "compute.googleapis.com"
|
||||
assert result["summary"] == "luke@or.not executed compute.instances.reset on mozdefdevvm1"
|
||||
assert "jsonPayload" not in result["details"]
|
||||
|
|
@ -0,0 +1,189 @@
|
|||
from mozdef_util.utilities.toUTC import toUTC
|
||||
from mq.plugins.stackdriver_syslog import message
|
||||
|
||||
|
||||
class TestStackDriverSyslog(object):
|
||||
def setup(self):
|
||||
self.plugin = message()
|
||||
self.metadata = {"index": "events"}
|
||||
|
||||
# Should never match and be modified by the plugin
|
||||
def test_notags_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {
|
||||
"source": "stackdriver",
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def test_wrongtags_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {
|
||||
"tags": "audit",
|
||||
"source": "stackdriver",
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def test_wrongcategory_log(self):
|
||||
metadata = {"index": "events"}
|
||||
event = {
|
||||
"tags": "audit",
|
||||
"source": "stackdriver",
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, metadata)
|
||||
# in = out - plugin didn't touch it
|
||||
assert result == event
|
||||
|
||||
def verify_metadata(self, metadata):
|
||||
assert metadata["index"] == "events"
|
||||
|
||||
def verify_defaults(self, result):
|
||||
assert result["category"] == "syslog"
|
||||
assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
|
||||
|
||||
def test_defaults(self):
|
||||
event = {
|
||||
'category': 'syslog',
|
||||
'source': 'stackdriver',
|
||||
'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
|
||||
'receivedtimestamp': '2019-11-22T00:32:20.078819+00:00',
|
||||
'timestamp': '2019-11-22T00:32 :13+00:00',
|
||||
'utctimestamp': '2019-11-22T00:32:13+00:00',
|
||||
'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
|
||||
'customendpoint': '',
|
||||
'details': {
|
||||
'insertId': '5s8y8sgro37aodjds',
|
||||
'labels': {'compute.googleapis.com/resource_name': 'mozdefdevvm1'},
|
||||
'logName': 'projects/mcd-001-252615/logs/syslog',
|
||||
'receiveTimestamp': '2019-11-22T00:32:18.754424975Z',
|
||||
'resource': {
|
||||
'labels': {
|
||||
'instance_id': '3401561556013842918',
|
||||
'project_id': 'mcd-001-252615',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'type': 'gce_instance',
|
||||
},
|
||||
'textPayload': 'Nov 22 00:32:13 mozdefdevvm1 systemd: Started Session 1 of user mpurzynski.',
|
||||
'timestamp': '2019-11-22T00:32:13Z',
|
||||
},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
self.verify_defaults(result)
|
||||
self.verify_metadata(metadata)
|
||||
|
||||
def test_nomatch_generic_syslog(self):
|
||||
event = {
|
||||
"category": "syslog",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
|
||||
"severity": "7",
|
||||
"utctimestamp": "2017-09-26T00:22:23+00:00",
|
||||
"timestamp": "2017-09-26T00:22:23+00:00",
|
||||
"hostname": "something1.test.com",
|
||||
"mozdefhostname": "something1.test.com",
|
||||
"summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
|
||||
"eventsource": "systemslogs",
|
||||
"tags": "something",
|
||||
"details": {
|
||||
"processid": "21233",
|
||||
"sourceipv4address": "10.22.74.208",
|
||||
"hostname": "hostname1.subdomain.domain.com",
|
||||
"program": "sshd",
|
||||
"sourceipaddress": "10.22.74.208",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "syslog"
|
||||
assert result["eventsource"] == "systemslogs"
|
||||
assert result == event
|
||||
|
||||
def test_nomatch_auditd(self):
|
||||
event = {
|
||||
"category": "execve",
|
||||
"processid": "0",
|
||||
"receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
|
||||
"severity": "INFO",
|
||||
"utctimestamp": "2017-09-26T00:36:27+00:00",
|
||||
"tags": ["audisp-json", "2.1.1", "audit"],
|
||||
"summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"processname": "audisp-json",
|
||||
"details": {
|
||||
"fsuid": "398",
|
||||
"tty": "(none)",
|
||||
"uid": "398",
|
||||
"process": "/bin/bash",
|
||||
"auditkey": "exec",
|
||||
"pid": "10553",
|
||||
"processname": "sh",
|
||||
"session": "16467",
|
||||
"fsgid": "398",
|
||||
"sgid": "398",
|
||||
"auditserial": "3834716",
|
||||
"inode": "1835094",
|
||||
"ouid": "0",
|
||||
"ogid": "0",
|
||||
"suid": "398",
|
||||
"originaluid": "0",
|
||||
"gid": "398",
|
||||
"originaluser": "pubsub",
|
||||
"ppid": "10552",
|
||||
"cwd": "/",
|
||||
"parentprocess": "stackdriver",
|
||||
"euid": "398",
|
||||
"path": "/bin/sh",
|
||||
"rdev": "00:00",
|
||||
"dev": "08:03",
|
||||
"egid": "398",
|
||||
"command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
|
||||
"mode": "0100755",
|
||||
"user": "squid",
|
||||
},
|
||||
}
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "execve"
|
||||
assert "eventsource" not in result
|
||||
assert result == event
|
||||
|
||||
def test_stackdriver_syslog(self):
|
||||
event = {
|
||||
'category': 'syslog',
|
||||
'source': 'stackdriver',
|
||||
'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
|
||||
'receivedtimestamp': '2019-11-22T00:32:20.078819+00:00',
|
||||
'timestamp': '2019-11-22T00:32 :13+00:00',
|
||||
'utctimestamp': '2019-11-22T00:32:13+00:00',
|
||||
'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
|
||||
'customendpoint': '',
|
||||
'details': {
|
||||
'insertId': '5s8y8sgro37aodjds',
|
||||
'labels': {'compute.googleapis.com/resource_name': 'mozdefdevvm1'},
|
||||
'logName': 'projects/mcd-001-252615/logs/syslog',
|
||||
'receiveTimestamp': '2019-11-22T00:32:18.754424975Z',
|
||||
'resource': {
|
||||
'labels': {
|
||||
'instance_id': '3401561556013842918',
|
||||
'project_id': 'mcd-001-252615',
|
||||
'zone': 'us-west2-a',
|
||||
},
|
||||
'type': 'gce_instance',
|
||||
},
|
||||
'textPayload': 'Nov 22 00:32:13 mozdefdevvm1 systemd: Started Session 1 of user yoda.',
|
||||
'timestamp': '2019-11-22T00:32:13Z',
|
||||
},
|
||||
}
|
||||
|
||||
result, metadata = self.plugin.onMessage(event, self.metadata)
|
||||
assert result["category"] == "syslog"
|
||||
assert result["source"] == "stackdriver"
|
||||
assert result["utctimestamp"] == "2019-11-22T00:32:13+00:00"
|
||||
assert result["hostname"] == "mozdefdevvm1"
|
||||
assert result["processname"] == "systemd"
|
||||
assert result["summary"] == "Started Session 1 of user yoda."
|
||||
Загрузка…
Ссылка в новой задаче