Merge pull request #1481 from mpurzynski/stackdriver_final

Stackdriver final

mq/esworker_pubsub.conf

@@ -0,0 +1,8 @@
+[options]
+prefetch=10
+esbulksize=10
+esservers=http://localhost:9200
+mqprotocol = pubsub
+credentials_file = "<service credentials file>.json"
+project_id = "your project id"
+resource_name = "projects/<your project id>/subscriptions/<sub name>"

mq/esworker_pubsub.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+import json
+import sys
+import socket
+import time
+from configlib import getConfig, OptionParser
+from datetime import datetime
+from mozdef_util.utilities.toUTC import toUTC
+from mozdef_util.utilities.logger import logger, initLogger
+from mozdef_util.elasticsearch_client import (
+    ElasticsearchClient,
+    ElasticsearchBadServer,
+    ElasticsearchInvalidIndex,
+    ElasticsearchException,
+)
+from google.cloud import pubsub
+from lib.plugins import sendEventToPlugins, registerPlugins
+
+# running under uwsgi?
+try:
+    import uwsgi
+
+    hasUWSGI = True
+except ImportError as e:
+    hasUWSGI = False
+
+
+class PubSubtaskConsumer(object):
+    def __init__(self, esConnection, options):
+        self.esConnection = esConnection
+        self.pluginList = registerPlugins()
+        self.options = options
+        self.scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/pubsub "]
+        self.credentials_file = options.credentials_file
+
+    def run(self):
+        # XXX: fetch from the config file
+        subscriber = pubsub.SubscriberClient.from_service_account_file(self.options.credentials_file)
+        res = subscriber.subscribe(self.options.resource_name, callback=self.onMessage)
+        try:
+            res.result()
+        except Exception as e:
+            logger.exception(e)
+            logger.error(
+                "Received error during subscribing - killing self and my background thread in 5 seconds for uwsgi to bring me back"
+            )
+            time.sleep(5)
+            res.cancel()
+            sys.exit(1)
+
+    def onMessage(self, message):
+        try:
+            # default elastic search metadata for an event
+            metadata = {"index": "events", "id": None}
+            event = {}
+
+            event["receivedtimestamp"] = toUTC(datetime.now()).isoformat()
+            event["mozdefhostname"] = self.options.mozdefhostname
+
+            event["details"] = json.loads(message.data.decode("UTF-8"))
+
+            if "tags" in event["details"]:
+                event["tags"] = event["details"]["tags"].extend([self.options.resource_name])
+            else:
+                event["tags"] = [self.options.resource_name]
+            event["tags"].extend(["pubsub"])
+
+            (event, metadata) = sendEventToPlugins(event, metadata, self.pluginList)
+            # Drop message if plugins set to None
+            if event is None:
+                message.ack()
+                return
+            self.save_event(event, metadata)
+            message.ack()
+        except Exception as e:
+            logger.exception(e)
+            logger.error("Malformed message: %r" % message)
+            message.ack()
+
+    def save_event(self, event, metadata):
+        try:
+            # drop the message if a plug in set it to None
+            # signaling a discard
+            if event is None:
+                return
+
+            # make a json version for posting to elastic search
+            jbody = json.JSONEncoder().encode(event)
+
+            try:
+                bulk = False
+                if self.options.esbulksize != 0:
+                    bulk = True
+
+                self.esConnection.save_event(index=metadata["index"], doc_id=metadata["id"], body=jbody, bulk=bulk)
+
+            except (ElasticsearchBadServer, ElasticsearchInvalidIndex) as e:
+                # handle loss of server or race condition with index rotation/creation/aliasing
+                try:
+                    self.esConnection = esConnect()
+                    return
+                except (ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException) as e:
+                    logger.exception("ElasticSearchException: {0} reported while indexing event".format(e))
+                    return
+            except ElasticsearchException as e:
+                logger.exception("ElasticSearchException: {0} reported while indexing event".format(e))
+                logger.error("Malformed jbody: %r" % jbody)
+                return
+        except Exception as e:
+            logger.exception(e)
+            logger.error("Malformed message: %r" % event)
+
+
+def esConnect():
+    """open or re-open a connection to elastic search"""
+    return ElasticsearchClient((list("{0}".format(s) for s in options.esservers)), options.esbulksize)
+
+
+def initConfig():
+    # capture the hostname
+    options.mozdefhostname = getConfig("mozdefhostname", socket.gethostname(), options.configfile)
+
+    # elastic search options. set esbulksize to a non-zero value to enable bulk posting, set timeout to post no matter how many events after X seconds.
+    options.esservers = list(getConfig("esservers", "http://localhost:9200", options.configfile).split(","))
+    options.esbulksize = getConfig("esbulksize", 0, options.configfile)
+    options.esbulktimeout = getConfig("esbulktimeout", 30, options.configfile)
+
+    # GCP PubSub options
+    options.resource_name = getConfig("resource_name", "", options.configfile)
+    options.credentials_file = getConfig("credentials_file", "", options.configfile)
+
+    options.mqprotocol = getConfig("mqprotocol", "pubsub", options.configfile)
+
+
+def main():
+    if hasUWSGI:
+        logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
+    else:
+        logger.info("started without uwsgi")
+
+    if options.mqprotocol not in ("pubsub"):
+        logger.error("Can only process pubsub queues, terminating")
+        sys.exit(1)
+
+    # connect to GCP and consume our queue
+    PubSubtaskConsumer(es, options).run()
+
+
+if __name__ == "__main__":
+    # configure ourselves
+    parser = OptionParser()
+    parser.add_option(
+        "-c", dest="configfile", default=sys.argv[0].replace(".py", ".conf"), help="configuration file to use"
+    )
+    (options, args) = parser.parse_args()
+    initConfig()
+    initLogger(options)
+
+    # open ES connection globally so we don't waste time opening it per message
+    es = esConnect()
+
+    main()
+    try:
+        main()
+    except KeyboardInterrupt as e:
+        logger.info("Exiting worker")
+        if options.esbulksize != 0:
+            es.finish_bulk()
+    except Exception as e:
+        if options.esbulksize != 0:
+            es.finish_bulk()
+        raise

mq/plugins/stackdriver.py

@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+import urllib
+from mozdef_util.utilities.toUTC import toUTC
+
+
+class message(object):
+    def __init__(self):
+        """
+            Plugin used to fix object type discretions with cloudtrail messages
+        """
+        self.registration = ["pubsub"]
+        self.priority = 5
+
+    def onMessage(self, message, metadata):
+        # trust no one mr mulder
+        if "tags" not in message:
+            return (message, metadata)
+        if "pubsub" not in message["tags"]:
+            return (message, metadata)
+        if "details" not in message:
+            return (message, metadata)
+
+        event = message["details"]
+
+        if "logName" not in event:
+            return (message, metadata)
+        else:
+            # XXX: implement filtering of audit types that we want to see (yaml)
+            newmessage = dict()
+            logtype = "UNKNOWN"
+            if "logName" in event:
+                logtype = urllib.parse.unquote(event["logName"]).split("/")[-1].strip()
+            if "protoPayload" in event:
+                if "@type" in event["protoPayload"]:
+                    if event["protoPayload"]["@type"] == "type.googleapis.com/google.cloud.audit.AuditLog":
+                        newmessage["category"] = logtype
+                        newmessage["source"] = "stackdriver"
+                        newmessage["tags"] = message["tags"] + ["stackdriver"]
+            elif "jsonPayload" in event:
+                if "logName" in event:
+                    if logtype == "activity_log":
+                        newmessage["category"] = "gceactivity"
+                        newmessage["source"] = "stackdriver"
+                        newmessage["tags"] = message["tags"] + ["stackdriver"]
+            elif "textPayload" in event:
+                if "logName" in event:
+                    if logtype == "syslog":
+                        newmessage["category"] = logtype
+                        newmessage["source"] = "stackdriver"
+                        newmessage["tags"] = message["tags"] + ["stackdriver"]
+
+            newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
+            newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat()
+            newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat()
+            newmessage["mozdefhostname"] = message["mozdefhostname"]
+            newmessage["customendpoint"] = ""
+            newmessage["details"] = event
+
+        return (newmessage, metadata)

mq/plugins/stackdriver_audit.py

@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+from mozdef_util.utilities.toUTC import toUTC
+import os
+import yaml
+import jmespath
+
+
+class message(object):
+    def __init__(self):
+        """
+            Plugin used to fix object type discretions with cloudtrail messages
+        """
+        self.registration = ["stackdriver"]
+        self.priority = 15
+
+        with open(os.path.join(os.path.dirname(__file__), "stackdriver_audit.yml"), "r") as f:
+            mapping_map = f.read()
+
+        yap = yaml.safe_load(mapping_map)
+        self.eventtypes = list(yap.keys())
+        self.yap = yap
+        del mapping_map
+
+    def onMessage(self, message, metadata):
+        if "tags" not in message:
+            return (message, metadata)
+        if "stackdriver" not in message["tags"]:
+            return (message, metadata)
+        if "category" not in message:
+            return (message, metadata)
+        # XXX: move into a config file
+        cats = ["activity", "data_access"]
+        if message["category"] not in cats:
+            return (message, metadata)
+
+        newmessage = dict()
+
+        newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
+        newmessage["timestamp"] = toUTC(message["details"]["timestamp"]).isoformat()
+        newmessage["utctimestamp"] = toUTC(message["details"]["timestamp"]).isoformat()
+        newmessage["category"] = message["category"]
+        newmessage["tags"] = message["tags"]
+        newmessage["source"] = message["source"]
+        newmessage["mozdefhostname"] = message["mozdefhostname"]
+        newmessage["customendpoint"] = ""
+        newmessage["details"] = {}
+        newmessage["details"] = message["details"]
+        newmessage["details"]["gaudit"] = newmessage["details"]["protoPayload"]
+        del newmessage["details"]["protoPayload"]
+        # Stuff fields that will be used as a summary with something, anything. The mapping function will hopefuly find something to overwrite it with.
+        newmessage["details"]["username"] = "UNKNOWN"
+        newmessage["details"]["resourcename"] = "UNKNOWN"
+        if "request" in newmessage["details"]["gaudit"]:
+            if "resource" in newmessage["details"]["gaudit"]["request"]:
+                if type(newmessage["details"]["gaudit"]["request"]["resource"]) is not dict:
+                    del newmessage["details"]["gaudit"]["request"]["resource"]
+
+        if message["category"] in self.eventtypes:
+            for key in self.yap[newmessage["category"]]:
+                mappedvalue = jmespath.search(self.yap[newmessage["category"]][key], newmessage)
+                # JMESPath likes to silently return a None object
+                if mappedvalue is not None:
+                    newmessage["details"][key] = mappedvalue
+
+        # This is redundant
+        newmessage["summary"] = "{0} executed {1} on {2}".format(
+            newmessage["details"]["username"], newmessage["details"]["action"], newmessage["details"]["resourcename"]
+        )
+
+        return (newmessage, metadata)

mq/plugins/stackdriver_audit.yml

@@ -0,0 +1,24 @@
+---
+  data_access:
+    username: details.gaudit.authenticationInfo.principalEmail
+    action: details.gaudit.methodName
+    service: details.gaudit.serviceName
+    serviceData: details.gaudit.serviceData
+    resourcename: details.gaudit.resourceName
+    resourcetype: details.resource.type
+    projectid: details.resource.labels.project_id
+    logname: details.logName
+    sdreceivetimestamp: details.receiveTimestamp
+    sourceipaddress: details.gaudit.requestMetadata.callerIp
+
+  activity:
+    username: details.gaudit.authenticationInfo.principalEmail
+    action: details.gaudit.methodName
+    service: details.gaudit.serviceName
+    serviceData: details.gaudit.serviceData
+    resourcename: details.gaudit.resourceName
+    resourcetype: details.resource.type
+    projectid: details.resource.labels.project_id
+    logname: details.logName
+    sdreceivetimestamp: details.receiveTimestamp
+    sourceipaddress: details.gaudit.requestMetadata.callerIp

mq/plugins/stackdriver_gceactivity.py

@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+from mozdef_util.utilities.toUTC import toUTC
+import os
+import yaml
+import jmespath
+
+
+class message(object):
+    def __init__(self):
+        """
+            Plugin used to fix object type discretions with cloudtrail messages
+        """
+        self.registration = ["stackdriver"]
+        self.priority = 16
+
+        with open(os.path.join(os.path.dirname(__file__), "stackdriver_gceactivity.yml"), "r") as f:
+            mapping_map = f.read()
+
+        yap = yaml.safe_load(mapping_map)
+        self.eventtypes = list(yap.keys())
+        self.yap = yap
+        del mapping_map
+
+    def onMessage(self, message, metadata):
+        if "tags" not in message:
+            return (message, metadata)
+        if "stackdriver" not in message["tags"]:
+            return (message, metadata)
+        if "category" not in message:
+            return (message, metadata)
+        # XXX: move into a config file
+        cats = ["gceactivity"]
+        if message["category"] not in cats:
+            return (message, metadata)
+
+        event = message["details"]
+
+        newmessage = dict()
+
+        newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
+        newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat()
+        newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat()
+        newmessage["category"] = message["category"]
+        newmessage["tags"] = message["tags"]
+        newmessage["source"] = message["source"]
+        newmessage["mozdefhostname"] = message["mozdefhostname"]
+        newmessage["customendpoint"] = ""
+        newmessage["details"] = {}
+        newmessage["details"] = message["details"]
+        newmessage["details"]["gceactivity"] = newmessage["details"]["jsonPayload"]
+        del newmessage["details"]["jsonPayload"]
+        # This is fake
+        newmessage["details"]["service"] = "compute.googleapis.com"
+        # Stuff fields that will be used as a summary with something, anything. The mapping function will hopefuly find something to overwrite it with.
+        newmessage["details"]["username"] = "UNKNOWN"
+        newmessage["details"]["resourcename"] = "UNKNOWN"
+
+        if message["category"] in self.eventtypes:
+            for key in self.yap[newmessage["category"]]:
+                mappedvalue = jmespath.search(self.yap[newmessage["category"]][key], newmessage)
+                # JMESPath likes to silently return a None object
+                if mappedvalue is not None:
+                    newmessage["details"][key] = mappedvalue
+
+        # This is redundant
+        newmessage["summary"] = "{0} executed {1} on {2}".format(
+            newmessage["details"]["username"], newmessage["details"]["action"], newmessage["details"]["resourcename"],
+        )
+
+        return (newmessage, metadata)

mq/plugins/stackdriver_gceactivity.yml

@@ -0,0 +1,11 @@
+---
+  gceactivity:
+    username: details.gceactivity.actor.user
+    action: details.gceactivity.event_subtype
+    actiongroup: details.gceactivity.event_type
+    resourcename: details.gceactivity.resource.name
+    resourcetype: details.gceactivity.resource.type
+    resourceid: details.gceactivity.resource.id
+    projectid: details.resource.labels.project_id
+    logname: details.logName
+    sdreceivetimestamp: details.receiveTimestamp

mq/plugins/stackdriver_mapping.yml

@@ -0,0 +1,5 @@
+---
+  syslog:
+    details.findingid: details.id
+    
+    

mq/plugins/stackdriver_syslog.py

@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+from mozdef_util.utilities.toUTC import toUTC
+
+
+class message(object):
+    def __init__(self):
+        """
+            Plugin used to fix object type discretions with cloudtrail messages
+        """
+        self.registration = ["stackdriver"]
+        self.priority = 15
+
+    def onMessage(self, message, metadata):
+        if "tags" not in message:
+            return (message, metadata)
+        if "stackdriver" not in message["tags"]:
+            return (message, metadata)
+        if "category" not in message:
+            return (message, metadata)
+        if message["category"] != "syslog":
+            return (message, metadata)
+
+        event = message["details"]
+        newmessage = dict()
+
+        newmessage["receivedtimestamp"] = toUTC(message["receivedtimestamp"]).isoformat()
+        newmessage["timestamp"] = toUTC(event["timestamp"]).isoformat()
+        newmessage["utctimestamp"] = toUTC(event["timestamp"]).isoformat()
+        newmessage["category"] = "syslog"
+        newmessage["tags"] = message["tags"]
+        newmessage["source"] = message["source"]
+        newmessage["mozdefhostname"] = message["mozdefhostname"]
+        newmessage["customendpoint"] = ""
+        if "facility" in event:
+            newmessage["facility"] = event["facility"]
+        if "severity" in event:
+            newmessage["severity"] = event["severity"]
+
+        line = event["textPayload"].split()
+        newmessage["hostname"] = line[3]
+        newmessage["processname"] = line[4].strip(":")
+        newmessage["summary"] = " ".join(line[5:])
+
+        return (newmessage, metadata)

tests/mq/plugins/test_stackdriver.py

@@ -0,0 +1,207 @@
+from mozdef_util.utilities.toUTC import toUTC
+from mq.plugins.stackdriver import message
+
+
+class TestStackDriver(object):
+    def setup(self):
+        self.plugin = message()
+        self.metadata = {"index": "events"}
+
+    # Should never match and be modified by the plugin
+    def test_nodetails_log(self):
+        metadata = {"index": "events"}
+        event = {"tags": "pubsub"}
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def verify_metadata(self, metadata):
+        assert metadata["index"] == "events"
+
+    def verify_defaults(self, result):
+        assert result["category"] == "data_access"
+        assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
+
+    def test_defaults(self):
+        event = {
+            "receivedtimestamp": "2019-11-21T22:43:10.041549+00:00",
+            "mozdefhostname": "mozdefqa2.private.mdc1.mozilla.com",
+            "details": {
+                "insertId": "-81ga0vdqblo",
+                "logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access",
+                "protoPayload": {
+                    "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
+                    "authenticationInfo": {"principalEmail": "mpurzynski@gcp.infra.mozilla.com"},
+                    "authorizationInfo": [
+                        {
+                            "granted": True,
+                            "permission": "compute.instances.list",
+                            "resourceAttributes": {
+                                "name": "projects/mcd-001-252615",
+                                "service": "resourcemanager",
+                                "type": "resourcemanager.projects",
+                            },
+                        }
+                    ],
+                    "methodName": "beta.compute.instances.aggregatedList",
+                    "numResponseItems": "61",
+                    "request": {"@type": "type.googleapis.com/compute.instances.aggregatedList"},
+                    "requestMetadata": {
+                        "callerIp": "2620:101:80fb:224:2864:cebc:a1e:640c",
+                        "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)",
+                        "destinationAttributes": {},
+                        "requestAttributes": {"auth": {}, "time": "2019-11-21T22:42:26.336Z",},
+                    },
+                    "resourceLocation": {"currentLocations": ["global"]},
+                    "resourceName": "projects/mcd-001-252615/global/instances",
+                    "serviceName": "compute.googleapis.com",
+                },
+                "receiveTimestamp": "2019-11-21T22:42:26.904624537Z",
+                "resource": {
+                    "labels": {
+                        "location": "global",
+                        "method": "compute.instances.aggregatedList",
+                        "project_id": "mcd-001-252615",
+                        "service": "compute.googleapis.com",
+                        "version": "beta",
+                    },
+                    "type": "api",
+                },
+                "severity": "INFO",
+                "timestamp": "2019-11-21T22:42:25.759Z",
+            },
+            "tags": ["projects/mcd-001-252615/subscriptions/mozdefsubscription", "pubsub",],
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        self.verify_defaults(result)
+        self.verify_metadata(metadata)
+
+    def test_nomatch_syslog(self):
+        event = {
+            "category": "syslog",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
+            "severity": "7",
+            "utctimestamp": "2017-09-26T00:22:23+00:00",
+            "timestamp": "2017-09-26T00:22:23+00:00",
+            "hostname": "something1.test.com",
+            "mozdefhostname": "something1.test.com",
+            "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
+            "eventsource": "systemslogs",
+            "tags": "something",
+            "details": {
+                "processid": "21233",
+                "sourceipv4address": "10.22.74.208",
+                "hostname": "hostname1.subdomain.domain.com",
+                "program": "sshd",
+                "sourceipaddress": "10.22.74.208",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "syslog"
+        assert result["eventsource"] == "systemslogs"
+        assert result == event
+
+    def test_nomatch_auditd(self):
+        event = {
+            "category": "execve",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
+            "severity": "INFO",
+            "utctimestamp": "2017-09-26T00:36:27+00:00",
+            "tags": ["audisp-json", "2.1.1", "audit"],
+            "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+            "processname": "audisp-json",
+            "details": {
+                "fsuid": "398",
+                "tty": "(none)",
+                "uid": "398",
+                "process": "/bin/bash",
+                "auditkey": "exec",
+                "pid": "10553",
+                "processname": "sh",
+                "session": "16467",
+                "fsgid": "398",
+                "sgid": "398",
+                "auditserial": "3834716",
+                "inode": "1835094",
+                "ouid": "0",
+                "ogid": "0",
+                "suid": "398",
+                "originaluid": "0",
+                "gid": "398",
+                "originaluser": "pubsub",
+                "ppid": "10552",
+                "cwd": "/",
+                "parentprocess": "stackdriver",
+                "euid": "398",
+                "path": "/bin/sh",
+                "rdev": "00:00",
+                "dev": "08:03",
+                "egid": "398",
+                "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+                "mode": "0100755",
+                "user": "squid",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "execve"
+        assert "eventsource" not in result
+        assert result == event
+
+    def test_stackdriver(self):
+        event = {
+            "receivedtimestamp": "2019-11-21T22:43:10.041549+00:00",
+            "mozdefhostname": "mozdefqa2.private.mdc1.mozilla.com",
+            "details": {
+                "insertId": "-81ga0vdqblo",
+                "logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access",
+                "protoPayload": {
+                    "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
+                    "authenticationInfo": {"principalEmail": "mpurzynski@gcp.infra.mozilla.com"},
+                    "authorizationInfo": [
+                        {
+                            "granted": True,
+                            "permission": "compute.instances.list",
+                            "resourceAttributes": {
+                                "name": "projects/mcd-001-252615",
+                                "service": "resourcemanager",
+                                "type": "resourcemanager.projects",
+                            },
+                        }
+                    ],
+                    "methodName": "beta.compute.instances.aggregatedList",
+                    "numResponseItems": "61",
+                    "request": {"@type": "type.googleapis.com/compute.instances.aggregatedList"},
+                    "requestMetadata": {
+                        "callerIp": "2620:101:80fb:224:2864:cebc:a1e:640c",
+                        "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)",
+                        "destinationAttributes": {},
+                        "requestAttributes": {"auth": {}, "time": "2019-11-21T22:42:26.336Z",},
+                    },
+                    "resourceLocation": {"currentLocations": ["global"]},
+                    "resourceName": "projects/mcd-001-252615/global/instances",
+                    "serviceName": "compute.googleapis.com",
+                },
+                "receiveTimestamp": "2019-11-21T22:42:26.904624537Z",
+                "resource": {
+                    "labels": {
+                        "location": "global",
+                        "method": "compute.instances.aggregatedList",
+                        "project_id": "mcd-001-252615",
+                        "service": "compute.googleapis.com",
+                        "version": "beta",
+                    },
+                    "type": "api",
+                },
+                "severity": "INFO",
+                "timestamp": "2019-11-21T22:42:25.759Z",
+            },
+            "tags": ["projects/mcd-001-252615/subscriptions/mozdefsubscription", "pubsub",],
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "data_access"
+        assert result["details"]["protoPayload"]["@type"] == "type.googleapis.com/google.cloud.audit.AuditLog"

tests/mq/plugins/test_stackdriver_audit.py

@@ -0,0 +1,326 @@
+from mozdef_util.utilities.toUTC import toUTC
+from mq.plugins.stackdriver_audit import message
+
+
+class TestStackDriverAudit(object):
+    def setup(self):
+        self.plugin = message()
+        self.metadata = {"index": "events"}
+
+    # Should never match and be modified by the plugin
+    def test_notags_log(self):
+        metadata = {"index": "events"}
+        event = {
+            "source": "stackdriver",
+            "details": {"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access"},
+        }
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def test_wrongtags_log(self):
+        metadata = {"index": "events"}
+        event = {
+            "tags": "audit",
+            "source": "stackdriver",
+            "details": {"logName": "projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access"},
+        }
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def verify_metadata(self, metadata):
+        assert metadata["index"] == "events"
+
+    def verify_defaults(self, result):
+        assert result["category"] == "data_access"
+        assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
+
+    def test_defaults(self):
+        event = {
+            'category': 'data_access',
+            'source': 'stackdriver',
+            'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
+            'receivedtimestamp': '2019-11-21T23:53:36.695909+00:00',
+            'timestamp': '2019-11-21T23:45:34.930000+00:00',
+            'utctimestamp': '2019-11-21T23:45:34.930000+00:00',
+            'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
+            'customendpoint': '',
+            'details': {
+                'insertId': 'utar0xd1qjq',
+                'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access',
+                'protoPayload': {
+                    '@type': 'type.googleapis.com/google.cloud.audit.AuditLog',
+                    'authenticationInfo': {},
+                    'authorizationInfo': [
+                        {
+                            'permission': 'storage.buckets.get',
+                            'resource': 'projects/_/buckets/mcd-001-252615.appspot.com',
+                            'resourceAttributes': {},
+                        },
+                        {
+                            'permission': 'storage.buckets.getIamPolicy',
+                            'resource': 'projects/_/buckets/mcd-001-252615.appspot.com',
+                            'resourceAttributes': {},
+                        },
+                    ],
+                    'methodName': 'storage.buckets.get',
+                    'requestMetadata': {
+                        'destinationAttributes': {},
+                        'requestAttributes': {'auth': {}, 'time': '2019-11-21T23:45:34.949Z'},
+                    },
+                    'resourceLocation': {'currentLocations': ['asia-northeast2']},
+                    'resourceName': 'projects/_/buckets/mcd-001-252615.appspot.com',
+                    'serviceName': 'storage.googleapis.com',
+                    'status': {'code': 7, 'message': 'PERMISSION_DENIED'},
+                },
+                'receiveTimestamp': '2019-11-21T23:45:35.521115967Z',
+                'resource': {
+                    'labels': {
+                        'bucket_name': 'mcd-001-252615.appspot.com',
+                        'location': 'asia-northeast2',
+                        'project_id': 'mcd-001-252615',
+                    },
+                    'type': 'gcs_bucket',
+                },
+                'severity': 'ERROR',
+                'timestamp': '2019-11-21T23:45:34.93Z',
+            },
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        self.verify_defaults(result)
+        self.verify_metadata(metadata)
+
+    def test_nomatch_syslog(self):
+        event = {
+            "category": "syslog",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
+            "severity": "7",
+            "utctimestamp": "2017-09-26T00:22:23+00:00",
+            "timestamp": "2017-09-26T00:22:23+00:00",
+            "hostname": "something1.test.com",
+            "mozdefhostname": "something1.test.com",
+            "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
+            "eventsource": "systemslogs",
+            "tags": "something",
+            "details": {
+                "processid": "21233",
+                "sourceipv4address": "10.22.74.208",
+                "hostname": "hostname1.subdomain.domain.com",
+                "program": "sshd",
+                "sourceipaddress": "10.22.74.208",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "syslog"
+        assert result["eventsource"] == "systemslogs"
+        assert result == event
+
+    def test_nomatch_auditd(self):
+        event = {
+            "category": "execve",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
+            "severity": "INFO",
+            "utctimestamp": "2017-09-26T00:36:27+00:00",
+            "tags": ["audisp-json", "2.1.1", "audit"],
+            "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+            "processname": "audisp-json",
+            "details": {
+                "fsuid": "398",
+                "tty": "(none)",
+                "uid": "398",
+                "process": "/bin/bash",
+                "auditkey": "exec",
+                "pid": "10553",
+                "processname": "sh",
+                "session": "16467",
+                "fsgid": "398",
+                "sgid": "398",
+                "auditserial": "3834716",
+                "inode": "1835094",
+                "ouid": "0",
+                "ogid": "0",
+                "suid": "398",
+                "originaluid": "0",
+                "gid": "398",
+                "originaluser": "pubsub",
+                "ppid": "10552",
+                "cwd": "/",
+                "parentprocess": "stackdriver",
+                "euid": "398",
+                "path": "/bin/sh",
+                "rdev": "00:00",
+                "dev": "08:03",
+                "egid": "398",
+                "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+                "mode": "0100755",
+                "user": "squid",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "execve"
+        assert "eventsource" not in result
+        assert result == event
+
+    def test_stackdriver_audit_data_access(self):
+        event = {
+            'category': 'data_access',
+            'source': 'stackdriver',
+            'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
+            'receivedtimestamp': '2019-11-21T22:43:10.041549+00:00',
+            'timestamp': '2019-11-21T22:42:25.759000+00:00',
+            'utctimestamp': '2019-11-21T22:42:25.759000+00:00',
+            'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
+            'customendpoint': '',
+            'details': {
+                'insertId': '-81ga0vdqblo',
+                'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Fdata_access',
+                'protoPayload': {
+                    '@type': 'type.googleapis.com/google.cloud.audit.AuditLog',
+                    'authenticationInfo': {'principalEmail': '732492844671-compute@developer.gserviceaccount.com'},
+                    'authorizationInfo': [
+                        {
+                            'granted': True,
+                            'permission': 'compute.instances.list',
+                            'resourceAttributes': {
+                                'name': 'projects/mcd-001-252615',
+                                'service': 'resourcemanager',
+                                'type': 'resourcemanager.projects',
+                            },
+                        }
+                    ],
+                    'methodName': 'beta.compute.instances.aggregatedList',
+                    'numResponseItems': '61',
+                    'request': {'@type': 'type.googleapis.com/compute.instances.aggregatedList'},
+                    'requestMetadata': {
+                        'callerIp': '2620:101:80fb:224:2864:cebc:a1e:640c',
+                        'callerSuppliedUserAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
+                        'destinationAttributes': {},
+                        'requestAttributes': {'auth': {}, 'time': '2019-11-21T22:42:26.336Z'},
+                    },
+                    'resourceLocation': {'currentLocations': ['global']},
+                    'resourceName': 'projects/mcd-001-252615/global/instances',
+                    'serviceName': 'compute.googleapis.com',
+                },
+                'receiveTimestamp': '2019-11-21T22:42:26.904624537Z',
+                'resource': {
+                    'labels': {
+                        'location': 'global',
+                        'method': 'compute.instances.aggregatedList',
+                        'project_id': 'mcd-001-252615',
+                        'service': 'compute.googleapis.com',
+                        'version': 'beta',
+                    },
+                    'type': 'api',
+                },
+                'severity': 'INFO',
+                'timestamp': '2019-11-21T22:42:25.759Z',
+            },
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "data_access"
+        assert result["details"]["action"] == "beta.compute.instances.aggregatedList"
+        assert result["details"]["gaudit"]["authenticationInfo"]["principalEmail"] == "732492844671-compute@developer.gserviceaccount.com"
+        assert result["details"]["gaudit"]["methodName"] == "beta.compute.instances.aggregatedList"
+        assert result["details"]["projectid"] == "mcd-001-252615"
+        assert result["details"]["service"] == "compute.googleapis.com"
+        assert result["details"]["sourceipaddress"] == "2620:101:80fb:224:2864:cebc:a1e:640c"
+        assert result["details"]["username"] == "732492844671-compute@developer.gserviceaccount.com"
+        assert result["source"] == "stackdriver"
+        assert result["utctimestamp"] == "2019-11-21T22:42:25.759000+00:00"
+        assert "protoPayload" not in result["details"]
+
+    def test_stackdriver_audit_activity(self):
+        event = {
+            'category': 'activity',
+            'source': 'stackdriver',
+            'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
+            'receivedtimestamp': '2019-11-22T00:03:20.621831+00:00',
+            'timestamp': '2019-11-22T00:03:18.137000+00:00',
+            'utctimestamp': '2019-11-22T00:03:18.137000+00:00',
+            'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
+            'customendpoint': '',
+            'details': {
+                'insertId': '8w7e9jdcf16',
+                'logName': 'projects/mcd-001-252615/logs/cloudaudit.googleapis.com%2Factivity',
+                'operation': {
+                    'first': True,
+                    'id': 'operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27',
+                    'producer': 'type.googleapis.com',
+                },
+                'protoPayload': {
+                    '@type': 'type.googleapis.com/google.cloud.audit.AuditLog',
+                    'authenticationInfo': {'principalEmail': 'onceuponatime@inagalaxynottoofaraway.com'},
+                    'authorizationInfo': [
+                        {
+                            'granted': True,
+                            'permission': 'compute.instances.reset',
+                            'resourceAttributes': {
+                                'name': 'projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1',
+                                'service': 'compute',
+                                'type': 'compute.instances',
+                            },
+                        }
+                    ],
+                    'methodName': 'v1.compute.instances.reset',
+                    'request': {'@type': 'type.googleapis.com/compute.instances.reset'},
+                    'requestMetadata': {
+                        'callerIp': '2620:101:80fb:224:a889:abf2:7b0b:f928',
+                        'callerSuppliedUserAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
+                        'destinationAttributes': {},
+                        'requestAttributes': {'auth': {}, 'time': '2019-11-22T00:03:18.826Z'},
+                    },
+                    'resourceLocation': {'currentLocations': ['us-west2-a']},
+                    'resourceName': 'projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1',
+                    'response': {
+                        '@type': 'type.googleapis.com/operation',
+                        'id': '868140788263590697',
+                        'insertTime': '2019-11-21T16:03:18.588-08:00',
+                        'name': 'operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27',
+                        'operationType': 'reset',
+                        'progress': '0',
+                        'selfLink': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/operations/operation-1574380998061-597e424216be9-afa9fe5d-5f5c5c27',
+                        'selfLinkWithId': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/operations/868140788263590697',
+                        'startTime': '2019-11-21T16:03:18.597-08:00',
+                        'status': 'RUNNING',
+                        'targetId': '3401561556013842918',
+                        'targetLink': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1',
+                        'user': 'onceuponatime@inagalaxynottoofaraway.com',
+                        'zone': 'https://www.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a',
+                    },
+                    'serviceName': 'compute.googleapis.com',
+                },
+                'receiveTimestamp': '2019-11-22T00:03:19.525805615Z',
+                'resource': {
+                    'labels': {
+                        'instance_id': '3401561556013842918',
+                        'project_id': 'mcd-001-252615',
+                        'zone': 'us-west2-a',
+                    },
+                    'type': 'gce_instance',
+                },
+                'severity': 'NOTICE',
+                'timestamp': '2019-11-22T00:03:18.137Z',
+            },
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "activity"
+        assert result["details"]["action"] == "v1.compute.instances.reset"
+        assert result["details"]["gaudit"]["authenticationInfo"]["principalEmail"] == "onceuponatime@inagalaxynottoofaraway.com"
+        assert result["details"]["gaudit"]["methodName"] == "v1.compute.instances.reset"
+        assert result["details"]["operation"]["producer"] == "type.googleapis.com"
+        assert result["details"]["projectid"] == "mcd-001-252615"
+        assert result["details"]["resourcename"] == "projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1"
+        assert result["details"]["resourcetype"] == "gce_instance"
+        assert result["details"]["service"] == "compute.googleapis.com"
+        assert result["details"]["username"] == "onceuponatime@inagalaxynottoofaraway.com"
+        assert result["summary"] == "onceuponatime@inagalaxynottoofaraway.com executed v1.compute.instances.reset on projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1"
+
+        assert "protoPayload" not in result["details"]

tests/mq/plugins/test_stackdriver_gceactivity.py

@@ -0,0 +1,245 @@
+from mozdef_util.utilities.toUTC import toUTC
+from mq.plugins.stackdriver_gceactivity import message
+
+
+class TestStackDriverGCEActivity(object):
+    def setup(self):
+        self.plugin = message()
+        self.metadata = {"index": "events"}
+
+    def test_notags_log(self):
+        metadata = {"index": "events"}
+        event = {"category": "gceactivity"}
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def test_nocategory_log(self):
+        metadata = {"index": "events"}
+        event = {"tags": "audit"}
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    # Should never match and be modified by the plugin
+    def test_wrongtags_log(self):
+        metadata = {"index": "events"}
+        event = {"tags": "audit", "category": "gceactivity"}
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def verify_metadata(self, metadata):
+        assert metadata["index"] == "events"
+
+    def verify_defaults(self, result):
+        assert result["category"] == "gceactivity"
+        assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
+
+    def test_defaults(self):
+        event = {
+            'category': 'gceactivity',
+            'source': 'stackdriver',
+            'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
+            'receivedtimestamp': '2019-11-22T01:23:49.238723+00:00',
+            'timestamp': '2019-11-22T01:23:47.936931+00:00',
+            'utctimestamp': '2019-11-22T01:23:47.936931+00:00',
+            'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
+            'customendpoint': '',
+            'details': {
+                'insertId': '1y7iw8ag15tmjpz',
+                'jsonPayload': {
+                    'actor': {'user': 'luke@or.not'},
+                    'event_subtype': 'compute.instances.reset',
+                    'event_timestamp_us': '1574385827936931',
+                    'event_type': 'GCE_API_CALL',
+                    'ip_address': '',
+                    'operation': {
+                        'id': '2169930274576172620',
+                        'name': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
+                        'type': 'operation',
+                        'zone': 'us-west2-a',
+                    },
+                    'request': {
+                        'body': 'null',
+                        'url': 'https://compute.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1/reset?key=AIzaSyDWUi9T78xEO-m10evQANR7TMSiB_bjyNc',
+                    },
+                    'resource': {
+                        'id': '3401561556013842918',
+                        'name': 'mozdefdevvm1',
+                        'type': 'instance',
+                        'zone': 'us-west2-a',
+                    },
+                    'trace_id': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
+                    'user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
+                    'version': '1.2',
+                },
+                'labels': {
+                    'compute.googleapis.com/resource_id': '3401561556013842918',
+                    'compute.googleapis.com/resource_name': 'mozdefdevvm1',
+                    'compute.googleapis.com/resource_type': 'instance',
+                    'compute.googleapis.com/resource_zone': 'us-west2-a',
+                },
+                'logName': 'projects/mcd-001-252615/logs/compute.googleapis.com%2Factivity_log',
+                'receiveTimestamp': '2019-11-22T01:23:47.988998161Z',
+                'resource': {
+                    'labels': {
+                        'instance_id': '3401561556013842918',
+                        'project_id': 'mcd-001-252615',
+                        'zone': 'us-west2-a',
+                    },
+                    'type': 'gce_instance',
+                },
+                'severity': 'INFO',
+                'timestamp': '2019-11-22T01:23:47.936931Z',
+            },
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        self.verify_defaults(result)
+        self.verify_metadata(metadata)
+
+    def test_nomatch_syslog(self):
+        event = {
+            "category": "syslog",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
+            "severity": "7",
+            "utctimestamp": "2017-09-26T00:22:23+00:00",
+            "timestamp": "2017-09-26T00:22:23+00:00",
+            "hostname": "something1.test.com",
+            "mozdefhostname": "something1.test.com",
+            "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
+            "eventsource": "systemslogs",
+            "tags": "something",
+            "details": {
+                "processid": "21233",
+                "sourceipv4address": "10.22.74.208",
+                "hostname": "hostname1.subdomain.domain.com",
+                "program": "sshd",
+                "sourceipaddress": "10.22.74.208",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "syslog"
+        assert result["eventsource"] == "systemslogs"
+        assert result == event
+
+    def test_nomatch_auditd(self):
+        event = {
+            "category": "execve",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
+            "severity": "INFO",
+            "utctimestamp": "2017-09-26T00:36:27+00:00",
+            "tags": ["audisp-json", "2.1.1", "audit"],
+            "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+            "processname": "audisp-json",
+            "details": {
+                "fsuid": "398",
+                "tty": "(none)",
+                "uid": "398",
+                "process": "/bin/bash",
+                "auditkey": "exec",
+                "pid": "10553",
+                "processname": "sh",
+                "session": "16467",
+                "fsgid": "398",
+                "sgid": "398",
+                "auditserial": "3834716",
+                "inode": "1835094",
+                "ouid": "0",
+                "ogid": "0",
+                "suid": "398",
+                "originaluid": "0",
+                "gid": "398",
+                "originaluser": "pubsub",
+                "ppid": "10552",
+                "cwd": "/",
+                "parentprocess": "stackdriver",
+                "euid": "398",
+                "path": "/bin/sh",
+                "rdev": "00:00",
+                "dev": "08:03",
+                "egid": "398",
+                "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+                "mode": "0100755",
+                "user": "squid",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "execve"
+        assert "eventsource" not in result
+        assert result == event
+
+    def test_stackdriver(self):
+        event = {
+            'category': 'gceactivity',
+            'source': 'stackdriver',
+            'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
+            'receivedtimestamp': '2019-11-22T01:23:49.238723+00:00',
+            'timestamp': '2019-11-22T01:23:47.936931+00:00',
+            'utctimestamp': '2019-11-22T01:23:47.936931+00:00',
+            'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
+            'customendpoint': '',
+            'details': {
+                'insertId': '1y7iw8ag15tmjpz',
+                'jsonPayload': {
+                    'actor': {'user': 'luke@or.not'},
+                    'event_subtype': 'compute.instances.reset',
+                    'event_timestamp_us': '1574385827936931',
+                    'event_type': 'GCE_API_CALL',
+                    'ip_address': '',
+                    'operation': {
+                        'id': '2169930274576172620',
+                        'name': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
+                        'type': 'operation',
+                        'zone': 'us-west2-a',
+                    },
+                    'request': {
+                        'body': 'null',
+                        'url': 'https://compute.googleapis.com/compute/v1/projects/mcd-001-252615/zones/us-west2-a/instances/mozdefdevvm1/reset?key=AIzaSyDWUi9T78xEO-m10evQANR7TMSiB_bjyNc',
+                    },
+                    'resource': {
+                        'id': '3401561556013842918',
+                        'name': 'mozdefdevvm1',
+                        'type': 'instance',
+                        'zone': 'us-west2-a',
+                    },
+                    'trace_id': 'operation-1574385827284-597e543f984be-d1640557-51c07a30',
+                    'user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0,gzip(gfe),gzip(gfe)',
+                    'version': '1.2',
+                },
+                'labels': {
+                    'compute.googleapis.com/resource_id': '3401561556013842918',
+                    'compute.googleapis.com/resource_name': 'mozdefdevvm1',
+                    'compute.googleapis.com/resource_type': 'instance',
+                    'compute.googleapis.com/resource_zone': 'us-west2-a',
+                },
+                'logName': 'projects/mcd-001-252615/logs/compute.googleapis.com%2Factivity_log',
+                'receiveTimestamp': '2019-11-22T01:23:47.988998161Z',
+                'resource': {
+                    'labels': {
+                        'instance_id': '3401561556013842918',
+                        'project_id': 'mcd-001-252615',
+                        'zone': 'us-west2-a',
+                    },
+                    'type': 'gce_instance',
+                },
+                'severity': 'INFO',
+                'timestamp': '2019-11-22T01:23:47.936931Z',
+            },
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+
+        assert result["details"]["action"] == "compute.instances.reset"
+        assert result["details"]["gceactivity"]["resource"]["id"] == "3401561556013842918"
+        assert result["utctimestamp"] == "2019-11-22T01:23:47.936931+00:00"
+        assert result["details"]["username"] == "luke@or.not"
+        assert result["details"]["service"] == "compute.googleapis.com"
+        assert result["summary"] == "luke@or.not executed compute.instances.reset on mozdefdevvm1"
+        assert "jsonPayload" not in result["details"]

tests/mq/plugins/test_stackdriver_syslog.py

@@ -0,0 +1,189 @@
+from mozdef_util.utilities.toUTC import toUTC
+from mq.plugins.stackdriver_syslog import message
+
+
+class TestStackDriverSyslog(object):
+    def setup(self):
+        self.plugin = message()
+        self.metadata = {"index": "events"}
+
+    # Should never match and be modified by the plugin
+    def test_notags_log(self):
+        metadata = {"index": "events"}
+        event = {
+            "source": "stackdriver",
+        }
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def test_wrongtags_log(self):
+        metadata = {"index": "events"}
+        event = {
+            "tags": "audit",
+            "source": "stackdriver",
+        }
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def test_wrongcategory_log(self):
+        metadata = {"index": "events"}
+        event = {
+            "tags": "audit",
+            "source": "stackdriver",
+        }
+
+        result, metadata = self.plugin.onMessage(event, metadata)
+        # in = out - plugin didn't touch it
+        assert result == event
+
+    def verify_metadata(self, metadata):
+        assert metadata["index"] == "events"
+
+    def verify_defaults(self, result):
+        assert result["category"] == "syslog"
+        assert toUTC(result["receivedtimestamp"]).isoformat() == result["receivedtimestamp"]
+
+    def test_defaults(self):
+        event = {
+            'category': 'syslog',
+            'source': 'stackdriver',
+            'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
+            'receivedtimestamp': '2019-11-22T00:32:20.078819+00:00',
+            'timestamp': '2019-11-22T00:32  :13+00:00',
+            'utctimestamp': '2019-11-22T00:32:13+00:00',
+            'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
+            'customendpoint': '',
+            'details': {
+                'insertId': '5s8y8sgro37aodjds',
+                'labels': {'compute.googleapis.com/resource_name': 'mozdefdevvm1'},
+                'logName': 'projects/mcd-001-252615/logs/syslog',
+                'receiveTimestamp': '2019-11-22T00:32:18.754424975Z',
+                'resource': {
+                    'labels': {
+                        'instance_id': '3401561556013842918',
+                        'project_id': 'mcd-001-252615',
+                        'zone': 'us-west2-a',
+                    },
+                    'type': 'gce_instance',
+                },
+                'textPayload': 'Nov 22 00:32:13 mozdefdevvm1 systemd: Started Session 1 of user mpurzynski.',
+                'timestamp': '2019-11-22T00:32:13Z',
+            },
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        self.verify_defaults(result)
+        self.verify_metadata(metadata)
+
+    def test_nomatch_generic_syslog(self):
+        event = {
+            "category": "syslog",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00",
+            "severity": "7",
+            "utctimestamp": "2017-09-26T00:22:23+00:00",
+            "timestamp": "2017-09-26T00:22:23+00:00",
+            "hostname": "something1.test.com",
+            "mozdefhostname": "something1.test.com",
+            "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 pubsub stackdriver port 22\n",
+            "eventsource": "systemslogs",
+            "tags": "something",
+            "details": {
+                "processid": "21233",
+                "sourceipv4address": "10.22.74.208",
+                "hostname": "hostname1.subdomain.domain.com",
+                "program": "sshd",
+                "sourceipaddress": "10.22.74.208",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "syslog"
+        assert result["eventsource"] == "systemslogs"
+        assert result == event
+
+    def test_nomatch_auditd(self):
+        event = {
+            "category": "execve",
+            "processid": "0",
+            "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00",
+            "severity": "INFO",
+            "utctimestamp": "2017-09-26T00:36:27+00:00",
+            "tags": ["audisp-json", "2.1.1", "audit"],
+            "summary": "Execve: sh -c sudo squid proxy /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+            "processname": "audisp-json",
+            "details": {
+                "fsuid": "398",
+                "tty": "(none)",
+                "uid": "398",
+                "process": "/bin/bash",
+                "auditkey": "exec",
+                "pid": "10553",
+                "processname": "sh",
+                "session": "16467",
+                "fsgid": "398",
+                "sgid": "398",
+                "auditserial": "3834716",
+                "inode": "1835094",
+                "ouid": "0",
+                "ogid": "0",
+                "suid": "398",
+                "originaluid": "0",
+                "gid": "398",
+                "originaluser": "pubsub",
+                "ppid": "10552",
+                "cwd": "/",
+                "parentprocess": "stackdriver",
+                "euid": "398",
+                "path": "/bin/sh",
+                "rdev": "00:00",
+                "dev": "08:03",
+                "egid": "398",
+                "command": "sh -c sudo /usr/lib64/nagios/plugins/custom/check_auditd.sh",
+                "mode": "0100755",
+                "user": "squid",
+            },
+        }
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "execve"
+        assert "eventsource" not in result
+        assert result == event
+
+    def test_stackdriver_syslog(self):
+        event = {
+            'category': 'syslog',
+            'source': 'stackdriver',
+            'tags': ['projects/mcd-001-252615/subscriptions/mozdefsubscription', 'pubsub', 'stackdriver'],
+            'receivedtimestamp': '2019-11-22T00:32:20.078819+00:00',
+            'timestamp': '2019-11-22T00:32  :13+00:00',
+            'utctimestamp': '2019-11-22T00:32:13+00:00',
+            'mozdefhostname': 'mozdefqa2.private.mdc1.mozilla.com',
+            'customendpoint': '',
+            'details': {
+                'insertId': '5s8y8sgro37aodjds',
+                'labels': {'compute.googleapis.com/resource_name': 'mozdefdevvm1'},
+                'logName': 'projects/mcd-001-252615/logs/syslog',
+                'receiveTimestamp': '2019-11-22T00:32:18.754424975Z',
+                'resource': {
+                    'labels': {
+                        'instance_id': '3401561556013842918',
+                        'project_id': 'mcd-001-252615',
+                        'zone': 'us-west2-a',
+                    },
+                    'type': 'gce_instance',
+                },
+                'textPayload': 'Nov 22 00:32:13 mozdefdevvm1 systemd: Started Session 1 of user yoda.',
+                'timestamp': '2019-11-22T00:32:13Z',
+            },
+        }
+
+        result, metadata = self.plugin.onMessage(event, self.metadata)
+        assert result["category"] == "syslog"
+        assert result["source"] == "stackdriver"
+        assert result["utctimestamp"] == "2019-11-22T00:32:13+00:00"
+        assert result["hostname"] == "mozdefdevvm1"
+        assert result["processname"] == "systemd"
+        assert result["summary"] == "Started Session 1 of user yoda."