Merge pull request #2869 from vmray/vmray-etd-connector
Data connector for VMRay Email Threat Defender
This commit is contained in:
v-rucdu
GitHub
Коммит
77c03bb010
|
|
@ -0,0 +1,93 @@
|
|||
{
|
||||
"Name": "vmray_emails_CL",
|
||||
"Properties": [
|
||||
{
|
||||
"Name": "TenantId",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "SourceSystem",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "MG",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "ManagementGroupName",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "TimeGenerated",
|
||||
"Type": "DateTime"
|
||||
},
|
||||
{
|
||||
"Name": "Computer",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "RawData",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_attachments_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_headers_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_message_id_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_received_t",
|
||||
"Type": "DateTime"
|
||||
},
|
||||
{
|
||||
"Name": "email_recipients_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_sender_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_sensor_id_d",
|
||||
"Type": "Double"
|
||||
},
|
||||
{
|
||||
"Name": "email_sent_t",
|
||||
"Type": "DateTime"
|
||||
},
|
||||
{
|
||||
"Name": "email_subject_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_urls_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_verdict_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "email_vmray_uuid_g",
|
||||
"Type": "Guid"
|
||||
},
|
||||
{
|
||||
"Name": "email_webif_url_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "Type",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "_ResourceId",
|
||||
"Type": "String"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg version="1.1" viewBox="0 0 400 400" xml:space="preserve"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<metadata>
|
||||
<rdf:RDF>
|
||||
<cc:Work rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g transform="matrix(1.3333 0 0 -1.3333 0 400)">
|
||||
<g transform="scale(.1)">
|
||||
<path d="m1717.7 430.58h-67.12l-165.49 198.38 28.43 1.418c76.46 2.34 81.59 52.801 83.56 67.461 7.58 54.34-28.3 78.98-72.83 79.691l-75.99 0.321-52.21 44.808 126.1-0.449h1.71c97.34 0 135.53-59.75 127.81-116.42-1.87-40.168-26.57-85.641-79.73-104.33l145.76-170.88" fill="#1e516e"/>
|
||||
<path d="m799.2 433.6h-69.523l-142.71 392.07h55.816l122.97-344.94 122.97 344.94h53.191l-142.72-392.07" fill="#1e516e"/>
|
||||
<path d="m1347.3 433.6h-52.13v337.84l-109.02-229.88h-31.07l-108.23 229.88v-337.84h-48.708v392.07h71.098l104.54-218.28 101.11 218.28h72.41v-392.07" fill="#1e516e"/>
|
||||
<path d="m2114.8 430.58h-55.56l-38.45 109.27h-169.57l-38.45-109.27h-52.92l142.72 392.08h69.51l142.72-392.08m-178.79 346.52-68.99-192.48h137.72l-68.73 192.48" fill="#1e516e"/>
|
||||
<path d="m2275.1 430.58h-52.14v166.41l-138.51 225.67h57.67l107.18-175.37 108.22 175.37h55.56l-137.98-220.14v-171.94" fill="#1e516e"/>
|
||||
<path d="m1500 1982.8-308 586.68h616l-308-586.68" fill="#1e516e"/>
|
||||
<path d="m1023.3 1066.1h-623.33l550 1041.3 315.33-579.33-242-462" fill="#1e516e"/>
|
||||
<path d="m2600 1066.1h-623.25l-121 231-120.92 231-0.03-0.08-0.04 0.08 315.33 579.33 549.91-1041.3" fill="#1e516e"/>
|
||||
<path d="m1500 1066.1h-476.67l242 462 234.67-462" fill="#82a1af"/>
|
||||
<path d="m1976.7 1066.1h-476.67l234.8 461.92 120.95-230.92 120.92-231" fill="#82a1af"/>
|
||||
<path d="m1855.8 1297.1-120.95 230.92 0.03 0.08 120.92-231" fill="#0f4864"/>
|
||||
<path d="m1265.3 1528.1-315.33 579.33 242 462.01 308-586.68-234.67-454.66" fill="#82a1af"/>
|
||||
<path d="m1734.7 1528.1-234.67 454.66 308 586.68 242-462.01-315.33-579.33" fill="#82a1af"/>
|
||||
<path d="m1500 1066.1-234.67 462 234.67 454.66 234.67-454.66-234.67-462" fill="#ccd9de"/>
|
||||
<path d="m950 2107.4-242 462.01h484l-242-462.01" fill="#1e516e"/>
|
||||
<path d="m2050 2107.4-242 462.01h484l-242-462.01" fill="#1e516e"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
После Ширина: | Высота: | Размер: 2.6 KiB |
|
|
@ -0,0 +1,98 @@
|
|||
[
|
||||
{
|
||||
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"TimeGenerated": "2021-08-04T11:14:45Z",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"email_attachments_s": "[]",
|
||||
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id 11E59160813;\\tWed, 4 Aug 2021 11:14:45 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============1502096034021184596==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807568462.9.9036724522137395510_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:14:44 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"4 questions answered about computer\"}]",
|
||||
"email_message_id_s": "<162807568462.9.9036724522137395510_sanitized.com>",
|
||||
"email_received_t": "2021-08-04T11:14:45Z",
|
||||
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
|
||||
"email_sender_s": "sanitized@sanitized.com",
|
||||
"email_sensor_id_d": 1,
|
||||
"email_sent_t": "2021-08-04T11:14:44Z",
|
||||
"email_subject_s": "4 questions answered about computer",
|
||||
"email_urls_s": "[]",
|
||||
"email_verdict_s": "clean",
|
||||
"email_vmray_uuid_g": "337317e3-cddb-4e14-a310-9c4aa4badd01",
|
||||
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=337317e3-cddb-4e14-a310-9c4aa4badd01",
|
||||
"Type": "vmray_emails_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"TimeGenerated": "2021-08-04T11:19:47Z",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"email_attachments_s": "[{\"attachment_filename\": \"cover1.jpg\", \"attachment_sha256hash\": \"82899dee46a0e34f9a7f2ecb664522e6cf556527702a7e482928ec6039e6c4b1\", \"attachment_verdict\": \"clean\"}]",
|
||||
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id 6DCA2160813;\\tWed, 4 Aug 2021 11:19:45 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============0502589840769161396==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807598491.9.15160501248599303023_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:19:44 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"The fastest way to get better at anything\"}]",
|
||||
"email_message_id_s": "<162807598491.9.15160501248599303023_sanitized.com>",
|
||||
"email_received_t": "2021-08-04T11:19:45Z",
|
||||
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
|
||||
"email_sender_s": "sanitized@sanitized.com",
|
||||
"email_sensor_id_d": 1,
|
||||
"email_sent_t": "2021-08-04T11:19:44Z",
|
||||
"email_subject_s": "The fastest way to get better at anything",
|
||||
"email_urls_s": "[]",
|
||||
"email_verdict_s": "clean",
|
||||
"email_vmray_uuid_g": "23053748-da0e-4cce-81ca-9dd6b533b36e",
|
||||
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=23053748-da0e-4cce-81ca-9dd6b533b36e",
|
||||
"Type": "vmray_emails_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"TimeGenerated": "2021-08-04T11:22:26Z",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"email_attachments_s": "[{\"attachment_filename\": \"vs_BuildTools.exe\", \"attachment_sha256hash\": \"1778d923fd40c62c29f85cf4ef63a0c855a61c13b1dd68419bcda2cff38c984f\", \"attachment_verdict\": \"clean\"}, {\"attachment_filename\": \"Firefox Installer.exe\", \"attachment_sha256hash\": \"40cd4bef9a5dcb8a33840367a03932ecf06e82729c10b73d814853818ddad298\", \"attachment_verdict\": \"malicious\"}]",
|
||||
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id E8AA8160813;\\tWed, 4 Aug 2021 11:22:17 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============7106368624508490837==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807613748.9.17138733245689342939_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:22:17 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"The fastest way to get better at anything\"}]",
|
||||
"email_message_id_s": "<162807613748.9.17138733245689342939_sanitized.com>",
|
||||
"email_received_t": "2021-08-04T11:22:18Z",
|
||||
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
|
||||
"email_sender_s": "sanitized@sanitized.com",
|
||||
"email_sensor_id_d": 1,
|
||||
"email_sent_t": "2021-08-04T11:22:17Z",
|
||||
"email_subject_s": "The fastest way to get better at anything",
|
||||
"email_urls_s": "[]",
|
||||
"email_verdict_s": "malicious",
|
||||
"email_vmray_uuid_g": "51948389-b7c5-4634-aa21-cf46e33436d2",
|
||||
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=51948389-b7c5-4634-aa21-cf46e33436d2",
|
||||
"Type": "vmray_emails_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"TimeGenerated": "2021-08-04T11:24:01Z",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"email_attachments_s": "[]",
|
||||
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id DB521160813;\\tWed, 4 Aug 2021 11:24:00 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============2951710851682953810==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807624057.9.8491175543255676034_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:24:00 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"The uncomfortable truth about Python\"}]",
|
||||
"email_message_id_s": "<162807624057.9.8491175543255676034_sanitized.com>",
|
||||
"email_received_t": "2021-08-04T11:24:01Z",
|
||||
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
|
||||
"email_sender_s": "sanitized@sanitized.com",
|
||||
"email_sensor_id_d": 2,
|
||||
"email_sent_t": "2021-08-04T11:24:00Z",
|
||||
"email_subject_s": "The uncomfortable truth about Python",
|
||||
"email_urls_s": "[{\"url_original_url\": \"https://www.microsofft.com/de-de/\", \"url_verdict\": \"suspicious\"}, {\"url_original_url\": \"https://en.wikipedia.org/wiki/Microsoft_Azure\", \"url_verdict\": \"clean\"}]",
|
||||
"email_verdict_s": "suspicious",
|
||||
"email_vmray_uuid_g": "a7c2f692-7611-49fc-abdb-174f9da60c17",
|
||||
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=a7c2f692-7611-49fc-abdb-174f9da60c17",
|
||||
"Type": "vmray_emails_CL",
|
||||
"_ResourceId": ""
|
||||
}
|
||||
]
|
||||
Двоичный файл не отображается.
|
|
@ -0,0 +1,299 @@
|
|||
import base64
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import formatdate
|
||||
from enum import Enum
|
||||
from typing import Dict, Iterator, List, NamedTuple, Optional
|
||||
|
||||
import azure.functions as func
|
||||
import requests
|
||||
|
||||
|
||||
LOG_TYPE = "vmray_emails"
|
||||
|
||||
|
||||
class Verdict(Enum):
|
||||
clean = "clean"
|
||||
suspicious = "suspicious"
|
||||
malicious = "malicious"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
workspace_id: str
|
||||
workspace_key: str
|
||||
log_analytics_url: str
|
||||
vmray_platform_url: str
|
||||
vmray_api_key: str
|
||||
poll_additional_fields: List[str]
|
||||
poll_minimum_verdict: Verdict
|
||||
verify_tls: bool
|
||||
|
||||
@classmethod
|
||||
def from_environ(cls) -> "Config":
|
||||
poll_additional_fields = []
|
||||
for field in os.environ["POLL_ADDITIONAL_FIELDS"].split(","):
|
||||
field = field.strip()
|
||||
if field.startswith("email_"):
|
||||
poll_additional_fields.append(field)
|
||||
else:
|
||||
logging.error(
|
||||
"Ignoring invalid entry in POLL_ADDITIONAL_FIELDS: '%s'", field
|
||||
)
|
||||
|
||||
try:
|
||||
poll_minimum_verdict = Verdict(os.environ["POLL_MINIMUM_VERDICT"])
|
||||
except ValueError:
|
||||
logging.error("POLL_MINIMUM_VERDICT value is invalid; falling back to 'clean'")
|
||||
poll_minimum_verdict = Verdict.clean
|
||||
|
||||
verify_tls_env = os.environ["VERIFY_VMRAY_PLATFORM_TLS"].lower()
|
||||
try:
|
||||
verify_tls = {"true": True, "false": False}[verify_tls_env]
|
||||
except KeyError:
|
||||
logging.error("VERIFY_VMRAY_PLATFORM_TLS value is invalid; defaulting to 'true'")
|
||||
verify_tls = True
|
||||
|
||||
return cls(
|
||||
workspace_id=os.environ["WORKSPACE_ID"],
|
||||
workspace_key=os.environ["WORKSPACE_KEY"],
|
||||
log_analytics_url=os.environ["LOG_ANALYTICS_URL"],
|
||||
vmray_platform_url=os.environ["VMRAY_PLATFORM_URL"],
|
||||
vmray_api_key=os.environ["VMRAY_API_KEY"],
|
||||
poll_additional_fields=poll_additional_fields,
|
||||
poll_minimum_verdict=poll_minimum_verdict,
|
||||
verify_tls=verify_tls,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class State:
|
||||
last_poll: datetime
|
||||
|
||||
@classmethod
|
||||
def load(cls, raw: str) -> "State":
|
||||
dict_ = json.loads(raw)
|
||||
return cls(
|
||||
last_poll=datetime.fromisoformat(dict_["last_poll"]),
|
||||
)
|
||||
|
||||
def dump(self) -> str:
|
||||
dict_ = {
|
||||
"last_poll": self.last_poll.isoformat(),
|
||||
}
|
||||
return json.dumps(dict_)
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
@classmethod
|
||||
def expected(cls, code: int, message: str) -> "APIError":
|
||||
return cls(f"({code}) {message}")
|
||||
|
||||
@classmethod
|
||||
def unexpected(cls, resp: requests.Response) -> "APIError":
|
||||
return cls(f"Unexpected error: ({resp.status_code}) {resp.text}")
|
||||
|
||||
|
||||
class FeederError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
EmailInfo = Dict[str, object]
|
||||
|
||||
|
||||
REQUIRED_API_FIELDS = [
|
||||
"email_vmray_uuid",
|
||||
"email_message_id",
|
||||
"email_sent",
|
||||
"email_received",
|
||||
"email_sensor_id",
|
||||
"email_verdict",
|
||||
"email_verdict_reached",
|
||||
"email_webif_url",
|
||||
]
|
||||
|
||||
|
||||
class Feeder:
|
||||
def __init__(self, config: Config, state: State):
|
||||
self.config = config
|
||||
self.state = state
|
||||
|
||||
@property
|
||||
def api_url(self) -> str:
|
||||
return f"{self.config.vmray_platform_url}/rest"
|
||||
|
||||
@property
|
||||
def auth_headers(self) -> Dict[str, str]:
|
||||
return {"Authorization": f"api_key {self.config.vmray_api_key}"}
|
||||
|
||||
def feed(self) -> None:
|
||||
"""
|
||||
Feed email data fetched from the ETD API into Log Analytics.
|
||||
|
||||
We feed the data in batches to limit our memory usage and to
|
||||
ensure the requests to the Log Analytics API don't become too
|
||||
large.
|
||||
"""
|
||||
|
||||
poll_from = self.state.last_poll
|
||||
poll_until = datetime.now(tz=timezone.utc)
|
||||
logging.info("Polling from %s until %s", poll_from, poll_until)
|
||||
|
||||
batches_iter = self.poll_etd(poll_from, poll_until)
|
||||
count = 0
|
||||
while True:
|
||||
try:
|
||||
batch = next(batches_iter)
|
||||
except (requests.RequestException, APIError) as exc:
|
||||
raise FeederError(f"Error polling from VMRay ETD: {exc}")
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
try:
|
||||
self.push_to_log_analytics(batch)
|
||||
except (requests.RequestException, APIError) as exc:
|
||||
raise FeederError(f"Error pushing to Log Analytics: {exc}")
|
||||
|
||||
count += len(batch)
|
||||
|
||||
logging.info("Pushed %d emails", count)
|
||||
self.state.last_poll = poll_until
|
||||
|
||||
def poll_etd(self, start: datetime, end: datetime) -> Iterator[List[EmailInfo]]:
|
||||
time_fmt = "%Y-%m-%dT%H:%M:%S.%f"
|
||||
poll_interval = f"{start.strftime(time_fmt)}~{end.strftime(time_fmt)}"
|
||||
|
||||
fields = REQUIRED_API_FIELDS + self.config.poll_additional_fields
|
||||
fields_spec = "(" + ",".join(fields) + ")"
|
||||
|
||||
fixed_params = {
|
||||
"email_verdict_reached": poll_interval,
|
||||
"_fields": fields_spec,
|
||||
}
|
||||
|
||||
verdict_filters_map: Dict[Verdict, List[Optional[str]]] = {
|
||||
Verdict.clean: [None],
|
||||
Verdict.suspicious: ["suspicious", "malicious"],
|
||||
Verdict.malicious: ["malicious"],
|
||||
}
|
||||
verdict_filters = verdict_filters_map[self.config.poll_minimum_verdict]
|
||||
|
||||
for verdict in verdict_filters:
|
||||
if verdict:
|
||||
params = {"email_verdict": verdict, **fixed_params}
|
||||
else:
|
||||
params = fixed_params
|
||||
|
||||
yield from (b for b in self.poll_etd_with_params(params) if b)
|
||||
|
||||
def poll_etd_with_params(self, params: Dict[str, str]) -> Iterator[List[EmailInfo]]:
|
||||
resp = requests.get(
|
||||
url=f"{self.api_url}/email",
|
||||
headers=self.auth_headers,
|
||||
params=params,
|
||||
verify=self.config.verify_tls,
|
||||
)
|
||||
etd_resp = extract_etd_response(resp)
|
||||
yield etd_resp.emails
|
||||
|
||||
while etd_resp.continuation_id:
|
||||
resp = requests.get(
|
||||
url=f"{self.api_url}/continuation/{etd_resp.continuation_id}",
|
||||
headers=self.auth_headers,
|
||||
verify=self.config.verify_tls,
|
||||
)
|
||||
etd_resp = extract_etd_response(resp)
|
||||
yield etd_resp.emails
|
||||
|
||||
def push_to_log_analytics(self, email_data: List[EmailInfo]) -> None:
|
||||
method = "POST"
|
||||
resource = "/api/logs"
|
||||
content_type = "application/json"
|
||||
data = json.dumps(email_data)
|
||||
content_length = len(data)
|
||||
x_ms_date = formatdate(usegmt=True)
|
||||
|
||||
def build_signature() -> str:
|
||||
string_to_sign = "\n".join(
|
||||
[
|
||||
method,
|
||||
str(content_length),
|
||||
content_type,
|
||||
f"x-ms-date:{x_ms_date}",
|
||||
resource,
|
||||
]
|
||||
)
|
||||
bytes_to_sign = string_to_sign.encode("utf-8")
|
||||
key = base64.b64decode(self.config.workspace_key)
|
||||
mac = hmac.digest(key, bytes_to_sign, "sha256")
|
||||
signature = base64.b64encode(mac).decode()
|
||||
return signature
|
||||
|
||||
signature = build_signature()
|
||||
shared_key = f"{self.config.workspace_id}:{signature}"
|
||||
|
||||
resp = requests.request(
|
||||
method=method,
|
||||
url=f"{self.config.log_analytics_url}{resource}",
|
||||
params={"api-version": "2016-04-01"},
|
||||
headers={
|
||||
"Authorization": f"SharedKey {shared_key}",
|
||||
"Content-Type": content_type,
|
||||
"Log-Type": LOG_TYPE,
|
||||
"x-ms-date": x_ms_date,
|
||||
"time-generated-field": "email_verdict_reached",
|
||||
},
|
||||
data=data,
|
||||
)
|
||||
|
||||
success = 200 <= resp.status_code < 300
|
||||
if not success:
|
||||
raise APIError.unexpected(resp)
|
||||
|
||||
|
||||
class ETDResponse(NamedTuple):
|
||||
emails: List[EmailInfo]
|
||||
continuation_id: Optional[int]
|
||||
|
||||
|
||||
def extract_etd_response(resp: requests.Response) -> ETDResponse:
|
||||
try:
|
||||
resp_json = resp.json()
|
||||
result = resp_json["result"]
|
||||
if result == "ok":
|
||||
emails = resp_json["data"]
|
||||
continuation_id = resp_json.get("continuation_id")
|
||||
else:
|
||||
error_msg = resp_json["error_msg"]
|
||||
raise APIError.expected(resp.status_code, error_msg)
|
||||
|
||||
except (ValueError, KeyError):
|
||||
raise APIError.unexpected(resp)
|
||||
|
||||
return ETDResponse(emails, continuation_id)
|
||||
|
||||
|
||||
def main(timer: func.TimerRequest, state: func.InputStream) -> str:
|
||||
if state:
|
||||
decoded = state.read().decode()
|
||||
state_ = State.load(decoded)
|
||||
else:
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
state_ = State(
|
||||
last_poll=now - timedelta(minutes=10),
|
||||
)
|
||||
|
||||
config = Config.from_environ()
|
||||
|
||||
try:
|
||||
feeder = Feeder(config, state_)
|
||||
feeder.feed()
|
||||
except FeederError as exc:
|
||||
logging.error("%s", exc)
|
||||
raise
|
||||
|
||||
return feeder.state.dump()
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"scriptFile": "__init__.py",
|
||||
"bindings": [
|
||||
{
|
||||
"name": "timer",
|
||||
"type": "timerTrigger",
|
||||
"direction": "in",
|
||||
"schedule": "%POLLING_SCHEDULE%"
|
||||
},
|
||||
{
|
||||
"name": "state",
|
||||
"type": "blob",
|
||||
"path": "vmray-etd-feeder/state.json",
|
||||
"direction": "in"
|
||||
},
|
||||
{
|
||||
"name": "$return",
|
||||
"type": "blob",
|
||||
"path": "vmray-etd-feeder/state.json",
|
||||
"direction": "out"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
{
|
||||
"id": "VMRayETD",
|
||||
"title": "VMRay Email Threat Defender (ETD)",
|
||||
"publisher": "VMRay",
|
||||
"descriptionMarkdown": "This connector ingests email data collected by [VMRay Email Threat Defender (ETD)](https://www.vmray.com/products/email-threat-defender/). It requires a connection to a VMRay Platform 4.3.0 (or later) installation that has ETD enabled.",
|
||||
"graphQueries": [
|
||||
{
|
||||
"metricName": "Total data received",
|
||||
"legend": "Emails",
|
||||
"baseQuery": "vmray_emails_CL"
|
||||
}
|
||||
],
|
||||
"sampleQueries": [
|
||||
{
|
||||
"description": "All ingested emails",
|
||||
"query": "vmray_emails_CL\n| order by TimeGenerated desc"
|
||||
},
|
||||
{
|
||||
"description" : "Malicious emails",
|
||||
"query": "vmray_emails_CL\n| where email_verdict_s == \"malicious\""
|
||||
},
|
||||
{
|
||||
"description": "Recipients of malicious emails",
|
||||
"query": "vmray_emails_CL\n| where email_verdict_s == \"malicious\"\n| extend recipient=parse_json(email_recipients_s)\n| mv-expand recipient to typeof(string)\n| distinct recipient"
|
||||
},
|
||||
{
|
||||
"description": "Malicious attachments",
|
||||
"query": "vmray_emails_CL\n| where email_attachments_s has '\"malicious\"'\n| extend attachment=parse_json(email_attachments_s)\n| mv-expand attachment\n| where attachment.attachment_verdict == \"malicious\"\n| extend filename=tostring(attachment.attachment_filename)\n| extend hash=tostring(attachment.attachment_sha256hash)\n| summarize filenames=make_set(filename, 5) by hash"
|
||||
}
|
||||
],
|
||||
"dataTypes": [
|
||||
{
|
||||
"name": "vmray_emails_CL",
|
||||
"lastDataReceivedQuery": "vmray_emails_CL\n| summarize Time = max(TimeGenerated)\n| where isnotempty(Time)"
|
||||
}
|
||||
],
|
||||
"connectivityCriterias": [
|
||||
{
|
||||
"type": "IsConnectedQuery",
|
||||
"value": [
|
||||
"vmray_emails_CL\n| summarize LastLogReceived = max(TimeGenerated)\n| project IsConnected = LastLogReceived > ago(30d)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"availability": {
|
||||
"status": 1,
|
||||
"isPreview": true
|
||||
},
|
||||
"permissions": {
|
||||
"resourceProvider": [
|
||||
{
|
||||
"provider": "Microsoft.OperationalInsights/workspaces",
|
||||
"permissionsDisplayText": "read and write permissions on the workspace are required.",
|
||||
"providerDisplayName": "Workspace",
|
||||
"scope": "Workspace",
|
||||
"requiredPermissions": {
|
||||
"write": true,
|
||||
"read": true,
|
||||
"delete": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"provider": "Microsoft.OperationalInsights/workspaces/sharedKeys",
|
||||
"permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).",
|
||||
"providerDisplayName": "Keys",
|
||||
"scope": "Workspace",
|
||||
"requiredPermissions": {
|
||||
"action": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"customs": [
|
||||
{
|
||||
"name": "Microsoft.Web/sites permissions",
|
||||
"description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)."
|
||||
},
|
||||
{
|
||||
"name": "VMRay Platform Access",
|
||||
"description": "Access to a VMRay Platform 4.3.0 (or later) installation that has ETD enabled. Ensure that the VMRay Platform's REST API is accessible to external clients so that the connector can fetch data through it."
|
||||
}
|
||||
]
|
||||
},
|
||||
"instructionSteps": [
|
||||
{
|
||||
"title": "",
|
||||
"description": ">**NOTE:** This connector uses Azure Functions to connect to a VMRay Platform API to pull ETD email data into Azure Sentinel. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": ">**(Optional Step)** Securely store Workspace and API keys in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Function App."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": "**STEP 1 - Configuration Steps for the VMRay Platform**\n\nIn the VMRay Platform, create a user that has permissions to access ETD and the email data you want to ingest. If you are using ETD Cloud, see the *Setting up and Managing Users* chapter of the *Cloud Account Manager Guide* for more information. If you are using ETD On Premises, see the *Setting up Users and Assigning Roles* section of the *On Premises Installation Guide* for more information.\n\nNext, create an API key for that user, which will be used by the Azure Sentinel connector. For detailed instructions, refer to the *API Programmer Guide*.\n\nVerify that your API key has the required permissions by running the following command:\n\n\t$ curl -H \"Authorization: api_key <vmray_api_key>\" <vmray_platform_url>/rest/email\nThis should return a list of emails that ETD has analyzed."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": "**STEP 2 - Deploy the Azure Resource Manager (ARM) Template**\n\n>**IMPORTANT:** Before deploying the VMRay ETD connector, have the Azure Sentinel Workspace ID and Primary Key (can be copied from below), as well as the VMRay Platform API key, readily available.",
|
||||
"instructions": [
|
||||
{
|
||||
"parameters": {
|
||||
"fillWith": [
|
||||
"WorkspaceId"
|
||||
],
|
||||
"label": "Workspace ID"
|
||||
},
|
||||
"type": "CopyableLabel"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fillWith": [
|
||||
"PrimaryKey"
|
||||
],
|
||||
"label": "Workspace Primary Key"
|
||||
},
|
||||
"type": "CopyableLabel"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": "1. Click the **Deploy to Azure** button below.\n\n\t[](https://aka.ms/sentinel-VMRayETD-azuredeploy)\n2. Select the preferred **Subscription**, **Resource group** and **Region**.\n3. Enter the **Workspace ID**, **Workspace Key**, **VMRay Platform URL** and **VMRay API Key**.\n>Note: If you are using Azure Key Vault secrets for any of the values above, use the`@Microsoft.KeyVault(SecretUri={Security Identifier})`schema in place of the string values. Refer to the [Key Vault references documentation](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) for further details.\nAdjust the other connector settings as required.\n\n4. Finish the wizard and wait for the deployment to complete."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
{
|
||||
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"ResourcePrefix": {
|
||||
"type": "string",
|
||||
"defaultValue": "vmrayetd",
|
||||
"minLength": 1,
|
||||
"maxLength": 11,
|
||||
"metadata": {
|
||||
"description": "Prefix for all resources created by this Azure Resource Manager (ARM) template."
|
||||
}
|
||||
},
|
||||
"WorkspaceID": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"description": "Azure Sentinel Workspace ID."
|
||||
}
|
||||
},
|
||||
"WorkspaceKey": {
|
||||
"type": "securestring",
|
||||
"metadata": {
|
||||
"description": "Azure Sentinel Workspace Primary Key."
|
||||
}
|
||||
},
|
||||
"VmrayPlatformURL": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"description": "URL of the VMRay Platform from which email data should be ingested."
|
||||
}
|
||||
},
|
||||
"VmrayAPIKey": {
|
||||
"type": "securestring",
|
||||
"metadata": {
|
||||
"description": "VMRay Platform API key with sufficient permissions to access email data. For example: 'https://eu.cloud.vmray.com'"
|
||||
}
|
||||
},
|
||||
"PollingSchedule": {
|
||||
"type": "string",
|
||||
"defaultValue": "0 * * * * *",
|
||||
"metadata": {
|
||||
"description": "Cron expression specifying the schedule for fetching data from the VMRay Platform."
|
||||
}
|
||||
},
|
||||
"PollAdditionalFields": {
|
||||
"type": "string",
|
||||
"defaultValue": "email_sender, email_recipients, email_subject, email_headers, email_attachments, email_urls",
|
||||
"metadata": {
|
||||
"description": "Comma-separated list of email fields that should be fetched in addition to the required fields. See the ETD API Reference, which is included with the VMRay Platform documentation, for complete information about the /rest/email endpoint, including a list of available email fields."
|
||||
}
|
||||
},
|
||||
"PollMinimumVerdict": {
|
||||
"type": "string",
|
||||
"defaultValue": "clean",
|
||||
"allowedValues": [
|
||||
"clean",
|
||||
"suspicious",
|
||||
"malicious"
|
||||
],
|
||||
"metadata": {
|
||||
"description": "Verdict filter for fetched emails. Specify the least severe verdict you want to fetch. For example, to fetch only suspicious and malicious emails, set this to 'suspicious'. To select all verdicts, specify 'clean'."
|
||||
}
|
||||
},
|
||||
"VerifyVmrayPlatformTLS": {
|
||||
"type": "bool",
|
||||
"defaultValue": "true",
|
||||
"metadata": {
|
||||
"description": "Set this to 'false' if you want to ignore TLS certificate errors when connecting to the VMRay Platform. This is not recommended for production but can be useful during testing."
|
||||
}
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"resourceName": "[concat(toLower(parameters('ResourcePrefix')), uniqueString(resourceGroup().id))]",
|
||||
"storageSuffix": "[tolower(environment().suffixes.storage)]",
|
||||
"logAnalyticsURL": "[replace(environment().portal, 'https://portal', concat('https://', toLower(parameters('WorkspaceID')), '.ods.opinsights'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts",
|
||||
"apiVersion": "2021-02-01",
|
||||
"name": "[variables('resourceName')]",
|
||||
"location": "[resourceGroup().location]",
|
||||
"kind": "StorageV2",
|
||||
"sku": {
|
||||
"name": "Standard_LRS",
|
||||
"tier": "Standard"
|
||||
},
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Insights/components",
|
||||
"apiVersion": "2015-05-01",
|
||||
"name": "[variables('resourceName')]",
|
||||
"location": "[resourceGroup().location]",
|
||||
"kind": "web",
|
||||
"properties": {
|
||||
"Application_Type": "web",
|
||||
"ApplicationId": "[variables('resourceName')]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Web/sites",
|
||||
"apiVersion": "2020-12-01",
|
||||
"name": "[variables('resourceName')]",
|
||||
"location": "[resourceGroup().location]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Storage/storageAccounts', variables('resourceName'))]",
|
||||
"[resourceId('Microsoft.Insights/components', variables('resourceName'))]"
|
||||
],
|
||||
"kind": "functionapp,linux",
|
||||
"properties": {
|
||||
"reserved": true,
|
||||
"siteConfig": {
|
||||
"linuxFxVersion": "python|3.8",
|
||||
"appSettings": [
|
||||
{
|
||||
"name": "AzureWebJobsStorage",
|
||||
"value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('resourceName'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('resourceName')), '2021-02-01').keys[0].value, ';EndpointSuffix=', variables('storageSuffix'))]"
|
||||
},
|
||||
{
|
||||
"name": "APPINSIGHTS_INSTRUMENTATIONKEY",
|
||||
"value": "[reference(resourceId('Microsoft.insights/components', variables('resourceName'))).InstrumentationKey]"
|
||||
},
|
||||
{
|
||||
"name": "FUNCTIONS_EXTENSION_VERSION",
|
||||
"value": "~3"
|
||||
},
|
||||
{
|
||||
"name": "FUNCTIONS_WORKER_RUNTIME",
|
||||
"value": "python"
|
||||
},
|
||||
{
|
||||
"name": "WEBSITE_RUN_FROM_PACKAGE",
|
||||
"value": "https://aka.ms/sentinel-VMRayETD-functionapp"
|
||||
},
|
||||
{
|
||||
"name": "WORKSPACE_ID",
|
||||
"value": "[parameters('WorkspaceID')]"
|
||||
},
|
||||
{
|
||||
"name": "WORKSPACE_KEY",
|
||||
"value": "[parameters('WorkspaceKey')]"
|
||||
},
|
||||
{
|
||||
"name": "LOG_ANALYTICS_URL",
|
||||
"value": "[variables('logAnalyticsURL')]"
|
||||
},
|
||||
{
|
||||
"name": "VMRAY_PLATFORM_URL",
|
||||
"value": "[parameters('VmrayPlatformURL')]"
|
||||
},
|
||||
{
|
||||
"name": "VMRAY_API_KEY",
|
||||
"value": "[parameters('VmrayAPIKey')]"
|
||||
},
|
||||
{
|
||||
"name": "POLLING_SCHEDULE",
|
||||
"value": "[parameters('PollingSchedule')]"
|
||||
},
|
||||
{
|
||||
"name": "POLL_ADDITIONAL_FIELDS",
|
||||
"value": "[parameters('PollAdditionalFields')]"
|
||||
},
|
||||
{
|
||||
"name": "POLL_MINIMUM_VERDICT",
|
||||
"value": "[parameters('PollMinimumVerdict')]"
|
||||
},
|
||||
{
|
||||
"name": "VERIFY_VMRAY_PLATFORM_TLS",
|
||||
"value": "[parameters('VerifyVmrayPlatformTLS')]"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"version": "2.0",
|
||||
"extensionBundle": {
|
||||
"id": "Microsoft.Azure.Functions.ExtensionBundle",
|
||||
"version": "[2.*, 3.0.0)"
|
||||
},
|
||||
"logging": {
|
||||
"applicationInsights": {
|
||||
"samplingSettings": {
|
||||
"isEnabled": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
azure-functions
|
||||
requests==2.25.1
|
||||
Загрузка…
Ссылка в новой задаче