diff --git a/mq/esworker_pubsub.conf b/mq/esworker_pubsub.conf new file mode 100644 index 00000000..f5bbec9b --- /dev/null +++ b/mq/esworker_pubsub.conf @@ -0,0 +1,8 @@ +[options] +prefetch=10 +esbulksize=10 +esservers=http://localhost:9200 +mqprotocol = pubsub +credentials_file = ".json" +project_id = "your project id" +resource_name = "projects//subscriptions/" diff --git a/mq/esworker_pubsub.py b/mq/esworker_pubsub.py new file mode 100644 index 00000000..16a0aec1 --- /dev/null +++ b/mq/esworker_pubsub.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +import json +import sys +import socket +import time +from configlib import getConfig, OptionParser +from datetime import datetime +from mozdef_util.utilities.toUTC import toUTC +from mozdef_util.utilities.logger import logger, initLogger +from mozdef_util.elasticsearch_client import ( + ElasticsearchClient, + ElasticsearchBadServer, + ElasticsearchInvalidIndex, + ElasticsearchException, +) +from google.cloud import pubsub +from lib.plugins import sendEventToPlugins, registerPlugins + +# running under uwsgi? +try: + import uwsgi + + hasUWSGI = True +except ImportError as e: + hasUWSGI = False + + +class PubSubtaskConsumer(object): + def __init__(self, esConnection, options): + self.esConnection = esConnection + self.pluginList = registerPlugins() + self.options = options + self.scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/pubsub "] + self.credentials_file = options.credentials_file + + def run(self): + # XXX: fetch from the config file + subscriber = pubsub.SubscriberClient.from_service_account_file(self.options.credentials_file) + res = subscriber.subscribe(self.options.resource_name, callback=self.onMessage) + try: + res.result() + except Exception as e: + logger.exception(e) + logger.error( + "Received error during subscribing - killing self and my background thread in 5 seconds for uwsgi to bring me back" + ) + time.sleep(5) + res.cancel() + sys.exit(1) + + def onMessage(self, message): + try: + # default elastic search metadata for an event + metadata = {"index": "events", "id": None} + event = {} + + event["receivedtimestamp"] = toUTC(datetime.now()).isoformat() + event["mozdefhostname"] = self.options.mozdefhostname + + event["details"] = json.loads(message.data.decode("UTF-8")) + + if "tags" in event["details"]: + event["tags"] = event["details"]["tags"].extend([self.options.resource_name]) + else: + event["tags"] = [self.options.resource_name] + event["tags"].extend(["pubsub"]) + + (event, metadata) = sendEventToPlugins(event, metadata, self.pluginList) + # Drop message if plugins set to None + if event is None: + message.ack() + return + self.save_event(event, metadata) + message.ack() + except Exception as e: + logger.exception(e) + logger.error("Malformed message: %r" % message) + message.ack() + + def save_event(self, event, metadata): + try: + # drop the message if a plug in set it to None + # signaling a discard + if event is None: + return + + # make a json version for posting to elastic search + jbody = json.JSONEncoder().encode(event) + + try: + bulk = False + if self.options.esbulksize != 0: + bulk = True + + self.esConnection.save_event(index=metadata["index"], doc_id=metadata["id"], body=jbody, bulk=bulk) + + except (ElasticsearchBadServer, ElasticsearchInvalidIndex) as e: + # handle loss of server or race condition with index rotation/creation/aliasing + try: + self.esConnection = esConnect() + return + except (ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException) as e: + logger.exception("ElasticSearchException: {0} reported while indexing event".format(e)) + return + except ElasticsearchException as e: + logger.exception("ElasticSearchException: {0} reported while indexing event".format(e)) + logger.error("Malformed jbody: %r" % jbody) + return + except Exception as e: + logger.exception(e) + logger.error("Malformed message: %r" % event) + + +def esConnect(): + """open or re-open a connection to elastic search""" + return ElasticsearchClient((list("{0}".format(s) for s in options.esservers)), options.esbulksize) + + +def initConfig(): + # capture the hostname + options.mozdefhostname = getConfig("mozdefhostname", socket.gethostname(), options.configfile) + + # elastic search options. set esbulksize to a non-zero value to enable bulk posting, set timeout to post no matter how many events after X seconds. + options.esservers = list(getConfig("esservers", "http://localhost:9200", options.configfile).split(",")) + options.esbulksize = getConfig("esbulksize", 0, options.configfile) + options.esbulktimeout = getConfig("esbulktimeout", 30, options.configfile) + + # GCP PubSub options + options.resource_name = getConfig("resource_name", "", options.configfile) + options.credentials_file = getConfig("credentials_file", "", options.configfile) + + options.mqprotocol = getConfig("mqprotocol", "pubsub", options.configfile) + + +def main(): + if hasUWSGI: + logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id())) + else: + logger.info("started without uwsgi") + + if options.mqprotocol not in ("pubsub"): + logger.error("Can only process pubsub queues, terminating") + sys.exit(1) + + # connect to GCP and consume our queue + PubSubtaskConsumer(es, options).run() + + +if __name__ == "__main__": + # configure ourselves + parser = OptionParser() + parser.add_option( + "-c", dest="configfile", default=sys.argv[0].replace(".py", ".conf"), help="configuration file to use" + ) + (options, args) = parser.parse_args() + initConfig() + initLogger(options) + + # open ES connection globally so we don't waste time opening it per message + es = esConnect() + + main() + try: + main() + except KeyboardInterrupt as e: + logger.info("Exiting worker") + if options.esbulksize != 0: + es.finish_bulk() + except Exception as e: + if options.esbulksize != 0: + es.finish_bulk() + raise diff --git a/mq/plugins/stackdriver.py b/mq/plugins/stackdriver.py new file mode 100644 index 00000000..06fb1053 --- /dev/null +++ b/mq/plugins/stackdriver.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +import urllib +from mozdef_util.utilities.toUTC import toUTC + + +class message(object): + def __init__(self): + """ + Plugin used to fix object type discretions with cloudtrail messages + """ + self.registration = ["pubsub"] + self.priority = 5 + + def onMessage(self, message, metadata): + # trust no one mr mulder + if "tags" not in message: + return (message, metadata) + if "pubsub" not in message["tags"]: + return (message, metadata) + if "details" not in message: + return (message, metadata) + + event = message["details"] + + if "logName" not in event: + return (message, metadata) + else: + # XXX: implement filtering of audit types that we want to see (yaml) + newmessage = dict() + logtype = "UNKNOWN" + if "logName" in event: + logtype = urllib.parse.unquote(event["logName"]).split("/")[-1].strip() + if "protoPayload" in event: + if "@type" in event["protoPayload"]: + if event["protoPayload"]["@type"] == "type.googleapis.com/google.cloud.audit.AuditLog": + newmessage["category"] = logtype + newmessage["source"] = "stackdriver" + newmessage["tags"] = message["tags"] + ["stackdriver"] + elif "jsonPayload" in event: + if "logName" in event: + if logtype == "activity_log": + newmessage["category"] = "gceactivity" + newmessage["source"] = "stackdriver" + newmessage["tags"] = message["tags"] + ["stackdriver"] + elif "textPayload" in event: + if "logName" in event: + if logtype == "syslog": + newmessage["category"] = logtype + newmessage["source"] = "stackdriver" + newmessage["tags"] = message["tags"] + ["stackdriver"] + + newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat() + newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat() + newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat() + newmessage["mozdefhostname"] = message["mozdefhostname"] + newmessage["customendpoint"] = "" + newmessage["details"] = event + + return (newmessage, metadata) diff --git a/mq/plugins/stackdriver_audit.py b/mq/plugins/stackdriver_audit.py new file mode 100644 index 00000000..59dd0e81 --- /dev/null +++ b/mq/plugins/stackdriver_audit.py @@ -0,0 +1,74 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +from mozdef_util.utilities.toUTC import toUTC +import os +import yaml +import jmespath + + +class message(object): + def __init__(self): + """ + Plugin used to fix object type discretions with cloudtrail messages + """ + self.registration = ["stackdriver"] + self.priority = 15 + + with open(os.path.join(os.path.dirname(__file__), "stackdriver_audit.yml"), "r") as f: + mapping_map = f.read() + + yap = yaml.safe_load(mapping_map) + self.eventtypes = list(yap.keys()) + self.yap = yap + del mapping_map + + def onMessage(self, message, metadata): + if "tags" not in message: + return (message, metadata) + if "stackdriver" not in message["tags"]: + return (message, metadata) + if "category" not in message: + return (message, metadata) + # XXX: move into a config file + cats = ["activity", "data_access"] + if message["category"] not in cats: + return (message, metadata) + + newmessage = dict() + + newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat() + newmessage["timestamp"] = toUTC(message["details"]["timestamp"]).isoformat() + newmessage["utctimestamp"] = toUTC(message["details"]["timestamp"]).isoformat() + newmessage["category"] = message["category"] + newmessage["tags"] = message["tags"] + newmessage["source"] = message["source"] + newmessage["mozdefhostname"] = message["mozdefhostname"] + newmessage["customendpoint"] = "" + newmessage["details"] = {} + newmessage["details"] = message["details"] + newmessage["details"]["gaudit"] = newmessage["details"]["protoPayload"] + del newmessage["details"]["protoPayload"] + # Stuff fields that will be used as a summary with something, anything. The mapping function will hopefuly find something to overwrite it with. + newmessage["details"]["username"] = "UNKNOWN" + newmessage["details"]["resourcename"] = "UNKNOWN" + if "request" in newmessage["details"]["gaudit"]: + if "resource" in newmessage["details"]["gaudit"]["request"]: + if type(newmessage["details"]["gaudit"]["request"]["resource"]) is not dict: + del newmessage["details"]["gaudit"]["request"]["resource"] + + if message["category"] in self.eventtypes: + for key in self.yap[newmessage["category"]]: + mappedvalue = jmespath.search(self.yap[newmessage["category"]][key], newmessage) + # JMESPath likes to silently return a None object + if mappedvalue is not None: + newmessage["details"][key] = mappedvalue + + # This is redundant + newmessage["summary"] = "{0} executed {1} on {2}".format( + newmessage["details"]["username"], newmessage["details"]["action"], newmessage["details"]["resourcename"] + ) + + return (newmessage, metadata) diff --git a/mq/plugins/stackdriver_audit.yml b/mq/plugins/stackdriver_audit.yml new file mode 100644 index 00000000..ac54b7e7 --- /dev/null +++ b/mq/plugins/stackdriver_audit.yml @@ -0,0 +1,24 @@ +--- + data_access: + username: details.gaudit.authenticationInfo.principalEmail + action: details.gaudit.methodName + service: details.gaudit.serviceName + serviceData: details.gaudit.serviceData + resourcename: details.gaudit.resourceName + resourcetype: details.resource.type + projectid: details.resource.labels.project_id + logname: details.logName + sdreceivetimestamp: details.receiveTimestamp + sourceipaddress: details.gaudit.requestMetadata.callerIp + + activity: + username: details.gaudit.authenticationInfo.principalEmail + action: details.gaudit.methodName + service: details.gaudit.serviceName + serviceData: details.gaudit.serviceData + resourcename: details.gaudit.resourceName + resourcetype: details.resource.type + projectid: details.resource.labels.project_id + logname: details.logName + sdreceivetimestamp: details.receiveTimestamp + sourceipaddress: details.gaudit.requestMetadata.callerIp diff --git a/mq/plugins/stackdriver_gceactivity.py b/mq/plugins/stackdriver_gceactivity.py new file mode 100644 index 00000000..9aafbfc4 --- /dev/null +++ b/mq/plugins/stackdriver_gceactivity.py @@ -0,0 +1,74 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +from mozdef_util.utilities.toUTC import toUTC +import os +import yaml +import jmespath + + +class message(object): + def __init__(self): + """ + Plugin used to fix object type discretions with cloudtrail messages + """ + self.registration = ["stackdriver"] + self.priority = 16 + + with open(os.path.join(os.path.dirname(__file__), "stackdriver_gceactivity.yml"), "r") as f: + mapping_map = f.read() + + yap = yaml.safe_load(mapping_map) + self.eventtypes = list(yap.keys()) + self.yap = yap + del mapping_map + + def onMessage(self, message, metadata): + if "tags" not in message: + return (message, metadata) + if "stackdriver" not in message["tags"]: + return (message, metadata) + if "category" not in message: + return (message, metadata) + # XXX: move into a config file + cats = ["gceactivity"] + if message["category"] not in cats: + return (message, metadata) + + event = message["details"] + + newmessage = dict() + + newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat() + newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat() + newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat() + newmessage["category"] = message["category"] + newmessage["tags"] = message["tags"] + newmessage["source"] = message["source"] + newmessage["mozdefhostname"] = message["mozdefhostname"] + newmessage["customendpoint"] = "" + newmessage["details"] = {} + newmessage["details"] = message["details"] + newmessage["details"]["gceactivity"] = newmessage["details"]["jsonPayload"] + del newmessage["details"]["jsonPayload"] + # This is fake + newmessage["details"]["service"] = "compute.googleapis.com" + # Stuff fields that will be used as a summary with something, anything. The mapping function will hopefuly find something to overwrite it with. + newmessage["details"]["username"] = "UNKNOWN" + newmessage["details"]["resourcename"] = "UNKNOWN" + + if message["category"] in self.eventtypes: + for key in self.yap[newmessage["category"]]: + mappedvalue = jmespath.search(self.yap[newmessage["category"]][key], newmessage) + # JMESPath likes to silently return a None object + if mappedvalue is not None: + newmessage["details"][key] = mappedvalue + + # This is redundant + newmessage["summary"] = "{0} executed {1} on {2}".format( + newmessage["details"]["username"], newmessage["details"]["action"], newmessage["details"]["resourcename"], + ) + + return (newmessage, metadata) diff --git a/mq/plugins/stackdriver_gceactivity.yml b/mq/plugins/stackdriver_gceactivity.yml new file mode 100644 index 00000000..61025136 --- /dev/null +++ b/mq/plugins/stackdriver_gceactivity.yml @@ -0,0 +1,11 @@ +--- + gceactivity: + username: details.gceactivity.actor.user + action: details.gceactivity.event_subtype + actiongroup: details.gceactivity.event_type + resourcename: details.gceactivity.resource.name + resourcetype: details.gceactivity.resource.type + resourceid: details.gceactivity.resource.id + projectid: details.resource.labels.project_id + logname: details.logName + sdreceivetimestamp: details.receiveTimestamp diff --git a/mq/plugins/stackdriver_mapping.yml b/mq/plugins/stackdriver_mapping.yml new file mode 100644 index 00000000..0e879fb5 --- /dev/null +++ b/mq/plugins/stackdriver_mapping.yml @@ -0,0 +1,5 @@ +--- + syslog: + details.findingid: details.id + + \ No newline at end of file diff --git a/mq/plugins/stackdriver_syslog.py b/mq/plugins/stackdriver_syslog.py new file mode 100644 index 00000000..681ec5ef --- /dev/null +++ b/mq/plugins/stackdriver_syslog.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +from mozdef_util.utilities.toUTC import toUTC + + +class message(object): + def __init__(self): + """ + Plugin used to fix object type discretions with cloudtrail messages + """ + self.registration = ["stackdriver"] + self.priority = 15 + + def onMessage(self, message, metadata): + if "tags" not in message: + return (message, metadata) + if "stackdriver" not in message["tags"]: + return (message, metadata) + if "category" not in message: + return (message, metadata) + if message["category"] != "syslog": + return (message, metadata) + + event = message["details"] + newmessage = dict() + + newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat() + newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat() + newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat() + newmessage["category"] = "syslog" + newmessage["tags"] = message["tags"] + newmessage["source"] = message["source"] + newmessage["mozdefhostname"] = message["mozdefhostname"] + newmessage["customendpoint"] = "" + if "facility" in event: + newmessage["facility"] = event["facility"] + if "severity" in event: + newmessage["severity"] = event["severity"] + + line = event["textPayload"].split() + newmessage["hostname"] = line[3] + newmessage["processname"] = line[4].strip(":") + newmessage["summary"] = " ".join(line[5:]) + + return (newmessage, metadata) diff --git a/tests/mq/plugins/test_stackdriver.py b/tests/mq/plugins/test_stackdriver.py new file mode 100644 index 00000000..ff9317f3 --- /dev/null +++ b/tests/mq/plugins/test_stackdriver.py @@ -0,0 +1,207 @@ +from mozdef_util.utilities.toUTC import toUTC +from mq.plugins.stackdriver import message + + +class TestStackDriver(object): + def setup(self): + self.plugin = message() + self.metadata = {"index": "events"} + + # Should never match and be modified by the plugin + def test_nodetails_log(self): + metadata = {"index": "events"} + event = {"tags": "pubsub"} + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def verify_metadata(self, metadata): + assert metadata["index"] == "events" + + def verify_defaults(self, result): + assert result["category"] == "data_access" + assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"] + + def test_defaults(self): + event = { + "receivedtimestamp": "2019-11-21T22:43:10.041549+00:00", + "mozdefhostname": "mozdefqa2.private.mdc1.mozilla.com", + "details": { + "insertId": "-81ga0vdqblo", + "logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access", + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "authenticationInfo": {"principalEmail": "mpurzynski@gcp.infra.mozilla.com"}, + "authorizationInfo": [ + { + "granted": True, + "permission": "compute.instances.list", + "resourceAttributes": { + "name": "projects/mcd-001-252615", + "service": "resourcemanager", + "type": "resourcemanager.projects", + }, + } + ], + "methodName": "beta.compute.instances.aggregatedList", + "numResponseItems": "61", + "request": {"@type": "type.googleapis.com/compute.instances.aggregatedList"}, + "requestMetadata": { + "callerIp": "2620:101:80fb:224:2864:cebc:a1e:640c", + "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)", + "destinationAttributes": {}, + "requestAttributes": {"auth": {}, "time": "2019-11-21T22:42:26.336Z",}, + }, + "resourceLocation": {"currentLocations": ["global"]}, + "resourceName": "projects/mcd-001-252615/global/instances", + "serviceName": "compute.googleapis.com", + }, + "receiveTimestamp": "2019-11-21T22:42:26.904624537Z", + "resource": { + "labels": { + "location": "global", + "method": "compute.instances.aggregatedList", + "project_id": "mcd-001-252615", + "service": "compute.googleapis.com", + "version": "beta", + }, + "type": "api", + }, + "severity": "INFO", + "timestamp": "2019-11-21T22:42:25.759Z", + }, + "tags": ["projects/mcd-001-252615/subscriptions/mozdefsubscription", "pubsub",], + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + + def test_nomatch_syslog(self): + event = { + "category": "syslog", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00", + "severity": "7", + "utctimestamp": "2017-09-26T00:22:23+00:00", + "timestamp": "2017-09-26T00:22:23+00:00", + "hostname": "something1.test.com", + "mozdefhostname": "something1.test.com", + "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n", + "eventsource": "systemslogs", + "tags": "something", + "details": { + "processid": "21233", + "sourceipv4address": "10.22.74.208", + "hostname": "hostname1.subdomain.domain.com", + "program": "sshd", + "sourceipaddress": "10.22.74.208", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "syslog" + assert result["eventsource"] == "systemslogs" + assert result == event + + def test_nomatch_auditd(self): + event = { + "category": "execve", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00", + "severity": "INFO", + "utctimestamp": "2017-09-26T00:36:27+00:00", + "tags": ["audisp-json", "2.1.1", "audit"], + "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "processname": "audisp-json", + "details": { + "fsuid": "398", + "tty": "(none)", + "uid": "398", + "process": "/bin/bash", + "auditkey": "exec", + "pid": "10553", + "processname": "sh", + "session": "16467", + "fsgid": "398", + "sgid": "398", + "auditserial": "3834716", + "inode": "1835094", + "ouid": "0", + "ogid": "0", + "suid": "398", + "originaluid": "0", + "gid": "398", + "originaluser": "pubsub", + "ppid": "10552", + "cwd": "/", + "parentprocess": "stackdriver", + "euid": "398", + "path": "/bin/sh", + "rdev": "00:00", + "dev": "08:03", + "egid": "398", + "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "mode": "0100755", + "user": "squid", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "execve" + assert "eventsource" not in result + assert result == event + + def test_stackdriver(self): + event = { + "receivedtimestamp": "2019-11-21T22:43:10.041549+00:00", + "mozdefhostname": "mozdefqa2.private.mdc1.mozilla.com", + "details": { + "insertId": "-81ga0vdqblo", + "logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access", + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "authenticationInfo": {"principalEmail": "mpurzynski@gcp.infra.mozilla.com"}, + "authorizationInfo": [ + { + "granted": True, + "permission": "compute.instances.list", + "resourceAttributes": { + "name": "projects/mcd-001-252615", + "service": "resourcemanager", + "type": "resourcemanager.projects", + }, + } + ], + "methodName": "beta.compute.instances.aggregatedList", + "numResponseItems": "61", + "request": {"@type": "type.googleapis.com/compute.instances.aggregatedList"}, + "requestMetadata": { + "callerIp": "2620:101:80fb:224:2864:cebc:a1e:640c", + "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)", + "destinationAttributes": {}, + "requestAttributes": {"auth": {}, "time": "2019-11-21T22:42:26.336Z",}, + }, + "resourceLocation": {"currentLocations": ["global"]}, + "resourceName": "projects/mcd-001-252615/global/instances", + "serviceName": "compute.googleapis.com", + }, + "receiveTimestamp": "2019-11-21T22:42:26.904624537Z", + "resource": { + "labels": { + "location": "global", + "method": "compute.instances.aggregatedList", + "project_id": "mcd-001-252615", + "service": "compute.googleapis.com", + "version": "beta", + }, + "type": "api", + }, + "severity": "INFO", + "timestamp": "2019-11-21T22:42:25.759Z", + }, + "tags": ["projects/mcd-001-252615/subscriptions/mozdefsubscription", "pubsub",], + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "data_access" + assert result["details"]["protoPayload"]["@type"] == "type.googleapis.com/google.cloud.audit.AuditLog" diff --git a/tests/mq/plugins/test_stackdriver_audit.py b/tests/mq/plugins/test_stackdriver_audit.py new file mode 100644 index 00000000..e1169198 --- /dev/null +++ b/tests/mq/plugins/test_stackdriver_audit.py @@ -0,0 +1,326 @@ +from mozdef_util.utilities.toUTC import toUTC +from mq.plugins.stackdriver_audit import message + + +class TestStackDriverAudit(object): + def setup(self): + self.plugin = message() + self.metadata = {"index": "events"} + + # Should never match and be modified by the plugin + def test_notags_log(self): + metadata = {"index": "events"} + event = { + "source": "stackdriver", + "details": {"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access"}, + } + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def test_wrongtags_log(self): + metadata = {"index": "events"} + event = { + "tags": "audit", + "source": "stackdriver", + "details": {"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access"}, + } + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def verify_metadata(self, metadata): + assert metadata["index"] == "events" + + def verify_defaults(self, result): + assert result["category"] == "data_access" + assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"] + + def test_defaults(self): + event = { + 'category': 'data_access', + 'source': 'stackdriver', + 'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'], + 'receivedtimestamp': '2019-11-21T23:53:36.695909+00:00', + 'timestamp': '2019-11-21T23:45:34.930000+00:00', + 'utctimestamp': '2019-11-21T23:45:34.930000+00:00', + 'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com', + 'customendpoint': '', + 'details': { + 'insertId': 'utar0xd1qjq', + 'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access', + 'protoPayload': { + '@type': 'type.googleapis.com/google.cloud.audit.AuditLog', + 'authenticationInfo': {}, + 'authorizationInfo': [ + { + 'permission': 'storage.buckets.get', + 'resource': 'projects/_/buckets/mcd-001-252615.appspot.com', + 'resourceAttributes': {}, + }, + { + 'permission': 'storage.buckets.getIamPolicy', + 'resource': 'projects/_/buckets/mcd-001-252615.appspot.com', + 'resourceAttributes': {}, + }, + ], + 'methodName': 'storage.buckets.get', + 'requestMetadata': { + 'destinationAttributes': {}, + 'requestAttributes': {'auth': {}, 'time': '2019-11-21T23:45:34.949Z'}, + }, + 'resourceLocation': {'currentLocations': ['asia-northeast2']}, + 'resourceName': 'projects/_/buckets/mcd-001-252615.appspot.com', + 'serviceName': 'storage.googleapis.com', + 'status': {'code': 7, 'message': 'PERMISSION_DENIED'}, + }, + 'receiveTimestamp': '2019-11-21T23:45:35.521115967Z', + 'resource': { + 'labels': { + 'bucket_name': 'mcd-001-252615.appspot.com', + 'location': 'asia-northeast2', + 'project_id': 'mcd-001-252615', + }, + 'type': 'gcs_bucket', + }, + 'severity': 'ERROR', + 'timestamp': '2019-11-21T23:45:34.93Z', + }, + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + + def test_nomatch_syslog(self): + event = { + "category": "syslog", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00", + "severity": "7", + "utctimestamp": "2017-09-26T00:22:23+00:00", + "timestamp": "2017-09-26T00:22:23+00:00", + "hostname": "something1.test.com", + "mozdefhostname": "something1.test.com", + "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n", + "eventsource": "systemslogs", + "tags": "something", + "details": { + "processid": "21233", + "sourceipv4address": "10.22.74.208", + "hostname": "hostname1.subdomain.domain.com", + "program": "sshd", + "sourceipaddress": "10.22.74.208", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "syslog" + assert result["eventsource"] == "systemslogs" + assert result == event + + def test_nomatch_auditd(self): + event = { + "category": "execve", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00", + "severity": "INFO", + "utctimestamp": "2017-09-26T00:36:27+00:00", + "tags": ["audisp-json", "2.1.1", "audit"], + "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "processname": "audisp-json", + "details": { + "fsuid": "398", + "tty": "(none)", + "uid": "398", + "process": "/bin/bash", + "auditkey": "exec", + "pid": "10553", + "processname": "sh", + "session": "16467", + "fsgid": "398", + "sgid": "398", + "auditserial": "3834716", + "inode": "1835094", + "ouid": "0", + "ogid": "0", + "suid": "398", + "originaluid": "0", + "gid": "398", + "originaluser": "pubsub", + "ppid": "10552", + "cwd": "/", + "parentprocess": "stackdriver", + "euid": "398", + "path": "/bin/sh", + "rdev": "00:00", + "dev": "08:03", + "egid": "398", + "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "mode": "0100755", + "user": "squid", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "execve" + assert "eventsource" not in result + assert result == event + + def test_stackdriver_audit_data_access(self): + event = { + 'category': 'data_access', + 'source': 'stackdriver', + 'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'], + 'receivedtimestamp': '2019-11-21T22:43:10.041549+00:00', + 'timestamp': '2019-11-21T22:42:25.759000+00:00', + 'utctimestamp': '2019-11-21T22:42:25.759000+00:00', + 'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com', + 'customendpoint': '', + 'details': { + 'insertId': '-81ga0vdqblo', + 'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access', + 'protoPayload': { + '@type': 'type.googleapis.com/google.cloud.audit.AuditLog', + 'authenticationInfo': {'principalEmail': '732492844671-compute@developer.gserviceaccount.com'}, + 'authorizationInfo': [ + { + 'granted': True, + 'permission': 'compute.instances.list', + 'resourceAttributes': { + 'name': 'projects/mcd-001-252615', + 'service': 'resourcemanager', + 'type': 'resourcemanager.projects', + }, + } + ], + 'methodName': 'beta.compute.instances.aggregatedList', + 'numResponseItems': '61', + 'request': {'@type': 'type.googleapis.com/compute.instances.aggregatedList'}, + 'requestMetadata': { + 'callerIp': '2620:101:80fb:224:2864:cebc:a1e:640c', + 'callerSuppliedUserAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)', + 'destinationAttributes': {}, + 'requestAttributes': {'auth': {}, 'time': '2019-11-21T22:42:26.336Z'}, + }, + 'resourceLocation': {'currentLocations': ['global']}, + 'resourceName': 'projects/mcd-001-252615/global/instances', + 'serviceName': 'compute.googleapis.com', + }, + 'receiveTimestamp': '2019-11-21T22:42:26.904624537Z', + 'resource': { + 'labels': { + 'location': 'global', + 'method': 'compute.instances.aggregatedList', + 'project_id': 'mcd-001-252615', + 'service': 'compute.googleapis.com', + 'version': 'beta', + }, + 'type': 'api', + }, + 'severity': 'INFO', + 'timestamp': '2019-11-21T22:42:25.759Z', + }, + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "data_access" + assert result["details"]["action"] == "beta.compute.instances.aggregatedList" + assert result["details"]["gaudit"]["authenticationInfo"]["principalEmail"] == "732492844671-compute@developer.gserviceaccount.com" + assert result["details"]["gaudit"]["methodName"] == "beta.compute.instances.aggregatedList" + assert result["details"]["projectid"] == "mcd-001-252615" + assert result["details"]["service"] == "compute.googleapis.com" + assert result["details"]["sourceipaddress"] == "2620:101:80fb:224:2864:cebc:a1e:640c" + assert result["details"]["username"] == "732492844671-compute@developer.gserviceaccount.com" + assert result["source"] == "stackdriver" + assert result["utctimestamp"] == "2019-11-21T22:42:25.759000+00:00" + assert "protoPayload" not in result["details"] + + def test_stackdriver_audit_activity(self): + event = { + 'category': 'activity', + 'source': 'stackdriver', + 'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'], + 'receivedtimestamp': '2019-11-22T00:03:20.621831+00:00', + 'timestamp': '2019-11-22T00:03:18.137000+00:00', + 'utctimestamp': '2019-11-22T00:03:18.137000+00:00', + 'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com', + 'customendpoint': '', + 'details': { + 'insertId': '8w7e9jdcf16', + 'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Factivity', + 'operation': { + 'first': True, + 'id': 'operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27', + 'producer': 'type.googleapis.com', + }, + 'protoPayload': { + '@type': 'type.googleapis.com/google.cloud.audit.AuditLog', + 'authenticationInfo': {'principalEmail': 'onceuponatime@inagalaxynottoofaraway.com'}, + 'authorizationInfo': [ + { + 'granted': True, + 'permission': 'compute.instances.reset', + 'resourceAttributes': { + 'name': 'projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1', + 'service': 'compute', + 'type': 'compute.instances', + }, + } + ], + 'methodName': 'v1.compute.instances.reset', + 'request': {'@type': 'type.googleapis.com/compute.instances.reset'}, + 'requestMetadata': { + 'callerIp': '2620:101:80fb:224:a889:abf2:7b0b:f928', + 'callerSuppliedUserAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)', + 'destinationAttributes': {}, + 'requestAttributes': {'auth': {}, 'time': '2019-11-22T00:03:18.826Z'}, + }, + 'resourceLocation': {'currentLocations': ['us-west2-a']}, + 'resourceName': 'projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1', + 'response': { + '@type': 'type.googleapis.com/operation', + 'id': '868140788263590697', + 'insertTime': '2019-11-21T16:03:18.588-08:00', + 'name': 'operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27', + 'operationType': 'reset', + 'progress': '0', + 'selfLink': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/operations/operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27', + 'selfLinkWithId': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/operations/868140788263590697', + 'startTime': '2019-11-21T16:03:18.597-08:00', + 'status': 'RUNNING', + 'targetId': '3401561556013842918', + 'targetLink': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1', + 'user': 'onceuponatime@inagalaxynottoofaraway.com', + 'zone': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a', + }, + 'serviceName': 'compute.googleapis.com', + }, + 'receiveTimestamp': '2019-11-22T00:03:19.525805615Z', + 'resource': { + 'labels': { + 'instance_id': '3401561556013842918', + 'project_id': 'mcd-001-252615', + 'zone': 'us-west2-a', + }, + 'type': 'gce_instance', + }, + 'severity': 'NOTICE', + 'timestamp': '2019-11-22T00:03:18.137Z', + }, + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "activity" + assert result["details"]["action"] == "v1.compute.instances.reset" + assert result["details"]["gaudit"]["authenticationInfo"]["principalEmail"] == "onceuponatime@inagalaxynottoofaraway.com" + assert result["details"]["gaudit"]["methodName"] == "v1.compute.instances.reset" + assert result["details"]["operation"]["producer"] == "type.googleapis.com" + assert result["details"]["projectid"] == "mcd-001-252615" + assert result["details"]["resourcename"] == "projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1" + assert result["details"]["resourcetype"] == "gce_instance" + assert result["details"]["service"] == "compute.googleapis.com" + assert result["details"]["username"] == "onceuponatime@inagalaxynottoofaraway.com" + assert result["summary"] == "onceuponatime@inagalaxynottoofaraway.com executed v1.compute.instances.reset on projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1" + + assert "protoPayload" not in result["details"] diff --git a/tests/mq/plugins/test_stackdriver_gceactivity.py b/tests/mq/plugins/test_stackdriver_gceactivity.py new file mode 100644 index 00000000..07ce8fc9 --- /dev/null +++ b/tests/mq/plugins/test_stackdriver_gceactivity.py @@ -0,0 +1,245 @@ +from mozdef_util.utilities.toUTC import toUTC +from mq.plugins.stackdriver_gceactivity import message + + +class TestStackDriverGCEActivity(object): + def setup(self): + self.plugin = message() + self.metadata = {"index": "events"} + + def test_notags_log(self): + metadata = {"index": "events"} + event = {"category": "gceactivity"} + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def test_nocategory_log(self): + metadata = {"index": "events"} + event = {"tags": "audit"} + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + # Should never match and be modified by the plugin + def test_wrongtags_log(self): + metadata = {"index": "events"} + event = {"tags": "audit", "category": "gceactivity"} + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def verify_metadata(self, metadata): + assert metadata["index"] == "events" + + def verify_defaults(self, result): + assert result["category"] == "gceactivity" + assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"] + + def test_defaults(self): + event = { + 'category': 'gceactivity', + 'source': 'stackdriver', + 'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'], + 'receivedtimestamp': '2019-11-22T01:23:49.238723+00:00', + 'timestamp': '2019-11-22T01:23:47.936931+00:00', + 'utctimestamp': '2019-11-22T01:23:47.936931+00:00', + 'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com', + 'customendpoint': '', + 'details': { + 'insertId': '1y7iw8ag15tmjpz', + 'jsonPayload': { + 'actor': {'user': 'luke@or.not'}, + 'event_subtype': 'compute.instances.reset', + 'event_timestamp_us': '1574385827936931', + 'event_type': 'GCE_API_CALL', + 'ip_address': '', + 'operation': { + 'id': '2169930274576172620', + 'name': 'operation-1574385827284-597e543f984be-d1640557-51c07a30', + 'type': 'operation', + 'zone': 'us-west2-a', + }, + 'request': { + 'body': 'null', + 'url': 'https://compute.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1/reset?key=AIzaSyDWUi9T78xEO-m10evQANR7TMSiB_bjyNc', + }, + 'resource': { + 'id': '3401561556013842918', + 'name': 'mozdefdevvm1', + 'type': 'instance', + 'zone': 'us-west2-a', + }, + 'trace_id': 'operation-1574385827284-597e543f984be-d1640557-51c07a30', + 'user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)', + 'version': '1.2', + }, + 'labels': { + 'compute.googleapis.com/resource_id': '3401561556013842918', + 'compute.googleapis.com/resource_name': 'mozdefdevvm1', + 'compute.googleapis.com/resource_type': 'instance', + 'compute.googleapis.com/resource_zone': 'us-west2-a', + }, + 'logName': 'projects/mcd-001-252615/logs/compute.googleapis.com%2Factivity_log', + 'receiveTimestamp': '2019-11-22T01:23:47.988998161Z', + 'resource': { + 'labels': { + 'instance_id': '3401561556013842918', + 'project_id': 'mcd-001-252615', + 'zone': 'us-west2-a', + }, + 'type': 'gce_instance', + }, + 'severity': 'INFO', + 'timestamp': '2019-11-22T01:23:47.936931Z', + }, + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + + def test_nomatch_syslog(self): + event = { + "category": "syslog", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00", + "severity": "7", + "utctimestamp": "2017-09-26T00:22:23+00:00", + "timestamp": "2017-09-26T00:22:23+00:00", + "hostname": "something1.test.com", + "mozdefhostname": "something1.test.com", + "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n", + "eventsource": "systemslogs", + "tags": "something", + "details": { + "processid": "21233", + "sourceipv4address": "10.22.74.208", + "hostname": "hostname1.subdomain.domain.com", + "program": "sshd", + "sourceipaddress": "10.22.74.208", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "syslog" + assert result["eventsource"] == "systemslogs" + assert result == event + + def test_nomatch_auditd(self): + event = { + "category": "execve", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00", + "severity": "INFO", + "utctimestamp": "2017-09-26T00:36:27+00:00", + "tags": ["audisp-json", "2.1.1", "audit"], + "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "processname": "audisp-json", + "details": { + "fsuid": "398", + "tty": "(none)", + "uid": "398", + "process": "/bin/bash", + "auditkey": "exec", + "pid": "10553", + "processname": "sh", + "session": "16467", + "fsgid": "398", + "sgid": "398", + "auditserial": "3834716", + "inode": "1835094", + "ouid": "0", + "ogid": "0", + "suid": "398", + "originaluid": "0", + "gid": "398", + "originaluser": "pubsub", + "ppid": "10552", + "cwd": "/", + "parentprocess": "stackdriver", + "euid": "398", + "path": "/bin/sh", + "rdev": "00:00", + "dev": "08:03", + "egid": "398", + "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "mode": "0100755", + "user": "squid", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "execve" + assert "eventsource" not in result + assert result == event + + def test_stackdriver(self): + event = { + 'category': 'gceactivity', + 'source': 'stackdriver', + 'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'], + 'receivedtimestamp': '2019-11-22T01:23:49.238723+00:00', + 'timestamp': '2019-11-22T01:23:47.936931+00:00', + 'utctimestamp': '2019-11-22T01:23:47.936931+00:00', + 'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com', + 'customendpoint': '', + 'details': { + 'insertId': '1y7iw8ag15tmjpz', + 'jsonPayload': { + 'actor': {'user': 'luke@or.not'}, + 'event_subtype': 'compute.instances.reset', + 'event_timestamp_us': '1574385827936931', + 'event_type': 'GCE_API_CALL', + 'ip_address': '', + 'operation': { + 'id': '2169930274576172620', + 'name': 'operation-1574385827284-597e543f984be-d1640557-51c07a30', + 'type': 'operation', + 'zone': 'us-west2-a', + }, + 'request': { + 'body': 'null', + 'url': 'https://compute.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1/reset?key=AIzaSyDWUi9T78xEO-m10evQANR7TMSiB_bjyNc', + }, + 'resource': { + 'id': '3401561556013842918', + 'name': 'mozdefdevvm1', + 'type': 'instance', + 'zone': 'us-west2-a', + }, + 'trace_id': 'operation-1574385827284-597e543f984be-d1640557-51c07a30', + 'user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)', + 'version': '1.2', + }, + 'labels': { + 'compute.googleapis.com/resource_id': '3401561556013842918', + 'compute.googleapis.com/resource_name': 'mozdefdevvm1', + 'compute.googleapis.com/resource_type': 'instance', + 'compute.googleapis.com/resource_zone': 'us-west2-a', + }, + 'logName': 'projects/mcd-001-252615/logs/compute.googleapis.com%2Factivity_log', + 'receiveTimestamp': '2019-11-22T01:23:47.988998161Z', + 'resource': { + 'labels': { + 'instance_id': '3401561556013842918', + 'project_id': 'mcd-001-252615', + 'zone': 'us-west2-a', + }, + 'type': 'gce_instance', + }, + 'severity': 'INFO', + 'timestamp': '2019-11-22T01:23:47.936931Z', + }, + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + + assert result["details"]["action"] == "compute.instances.reset" + assert result["details"]["gceactivity"]["resource"]["id"] == "3401561556013842918" + assert result["utctimestamp"] == "2019-11-22T01:23:47.936931+00:00" + assert result["details"]["username"] == "luke@or.not" + assert result["details"]["service"] == "compute.googleapis.com" + assert result["summary"] == "luke@or.not executed compute.instances.reset on mozdefdevvm1" + assert "jsonPayload" not in result["details"] diff --git a/tests/mq/plugins/test_stackdriver_syslog.py b/tests/mq/plugins/test_stackdriver_syslog.py new file mode 100644 index 00000000..a107a59c --- /dev/null +++ b/tests/mq/plugins/test_stackdriver_syslog.py @@ -0,0 +1,189 @@ +from mozdef_util.utilities.toUTC import toUTC +from mq.plugins.stackdriver_syslog import message + + +class TestStackDriverSyslog(object): + def setup(self): + self.plugin = message() + self.metadata = {"index": "events"} + + # Should never match and be modified by the plugin + def test_notags_log(self): + metadata = {"index": "events"} + event = { + "source": "stackdriver", + } + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def test_wrongtags_log(self): + metadata = {"index": "events"} + event = { + "tags": "audit", + "source": "stackdriver", + } + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def test_wrongcategory_log(self): + metadata = {"index": "events"} + event = { + "tags": "audit", + "source": "stackdriver", + } + + result, metadata = self.plugin.onMessage(event, metadata) + # in = out - plugin didn't touch it + assert result == event + + def verify_metadata(self, metadata): + assert metadata["index"] == "events" + + def verify_defaults(self, result): + assert result["category"] == "syslog" + assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"] + + def test_defaults(self): + event = { + 'category': 'syslog', + 'source': 'stackdriver', + 'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'], + 'receivedtimestamp': '2019-11-22T00:32:20.078819+00:00', + 'timestamp': '2019-11-22T00:32 :13+00:00', + 'utctimestamp': '2019-11-22T00:32:13+00:00', + 'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com', + 'customendpoint': '', + 'details': { + 'insertId': '5s8y8sgro37aodjds', + 'labels': {'compute.googleapis.com/resource_name': 'mozdefdevvm1'}, + 'logName': 'projects/mcd-001-252615/logs/syslog', + 'receiveTimestamp': '2019-11-22T00:32:18.754424975Z', + 'resource': { + 'labels': { + 'instance_id': '3401561556013842918', + 'project_id': 'mcd-001-252615', + 'zone': 'us-west2-a', + }, + 'type': 'gce_instance', + }, + 'textPayload': 'Nov 22 00:32:13 mozdefdevvm1 systemd: Started Session 1 of user mpurzynski.', + 'timestamp': '2019-11-22T00:32:13Z', + }, + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + + def test_nomatch_generic_syslog(self): + event = { + "category": "syslog", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00", + "severity": "7", + "utctimestamp": "2017-09-26T00:22:23+00:00", + "timestamp": "2017-09-26T00:22:23+00:00", + "hostname": "something1.test.com", + "mozdefhostname": "something1.test.com", + "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n", + "eventsource": "systemslogs", + "tags": "something", + "details": { + "processid": "21233", + "sourceipv4address": "10.22.74.208", + "hostname": "hostname1.subdomain.domain.com", + "program": "sshd", + "sourceipaddress": "10.22.74.208", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "syslog" + assert result["eventsource"] == "systemslogs" + assert result == event + + def test_nomatch_auditd(self): + event = { + "category": "execve", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00", + "severity": "INFO", + "utctimestamp": "2017-09-26T00:36:27+00:00", + "tags": ["audisp-json", "2.1.1", "audit"], + "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "processname": "audisp-json", + "details": { + "fsuid": "398", + "tty": "(none)", + "uid": "398", + "process": "/bin/bash", + "auditkey": "exec", + "pid": "10553", + "processname": "sh", + "session": "16467", + "fsgid": "398", + "sgid": "398", + "auditserial": "3834716", + "inode": "1835094", + "ouid": "0", + "ogid": "0", + "suid": "398", + "originaluid": "0", + "gid": "398", + "originaluser": "pubsub", + "ppid": "10552", + "cwd": "/", + "parentprocess": "stackdriver", + "euid": "398", + "path": "/bin/sh", + "rdev": "00:00", + "dev": "08:03", + "egid": "398", + "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "mode": "0100755", + "user": "squid", + }, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "execve" + assert "eventsource" not in result + assert result == event + + def test_stackdriver_syslog(self): + event = { + 'category': 'syslog', + 'source': 'stackdriver', + 'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'], + 'receivedtimestamp': '2019-11-22T00:32:20.078819+00:00', + 'timestamp': '2019-11-22T00:32 :13+00:00', + 'utctimestamp': '2019-11-22T00:32:13+00:00', + 'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com', + 'customendpoint': '', + 'details': { + 'insertId': '5s8y8sgro37aodjds', + 'labels': {'compute.googleapis.com/resource_name': 'mozdefdevvm1'}, + 'logName': 'projects/mcd-001-252615/logs/syslog', + 'receiveTimestamp': '2019-11-22T00:32:18.754424975Z', + 'resource': { + 'labels': { + 'instance_id': '3401561556013842918', + 'project_id': 'mcd-001-252615', + 'zone': 'us-west2-a', + }, + 'type': 'gce_instance', + }, + 'textPayload': 'Nov 22 00:32:13 mozdefdevvm1 systemd: Started Session 1 of user yoda.', + 'timestamp': '2019-11-22T00:32:13Z', + }, + } + + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result["category"] == "syslog" + assert result["source"] == "stackdriver" + assert result["utctimestamp"] == "2019-11-22T00:32:13+00:00" + assert result["hostname"] == "mozdefdevvm1" + assert result["processname"] == "systemd" + assert result["summary"] == "Started Session 1 of user yoda."