Merge pull request #2869 from vmray/vmray-etd-connector

Data connector for VMRay Email Threat Defender
This commit is contained in:
v-rucdu 2021-09-06 16:45:04 +05:30 коммит произвёл GitHub
Родитель 93c6ab2597 0a4fccab8c
Коммит 77c03bb010
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
10 изменённых файлов: 866 добавлений и 0 удалений

Просмотреть файл

@ -0,0 +1,93 @@
{
"Name": "vmray_emails_CL",
"Properties": [
{
"Name": "TenantId",
"Type": "String"
},
{
"Name": "SourceSystem",
"Type": "String"
},
{
"Name": "MG",
"Type": "String"
},
{
"Name": "ManagementGroupName",
"Type": "String"
},
{
"Name": "TimeGenerated",
"Type": "DateTime"
},
{
"Name": "Computer",
"Type": "String"
},
{
"Name": "RawData",
"Type": "String"
},
{
"Name": "email_attachments_s",
"Type": "String"
},
{
"Name": "email_headers_s",
"Type": "String"
},
{
"Name": "email_message_id_s",
"Type": "String"
},
{
"Name": "email_received_t",
"Type": "DateTime"
},
{
"Name": "email_recipients_s",
"Type": "String"
},
{
"Name": "email_sender_s",
"Type": "String"
},
{
"Name": "email_sensor_id_d",
"Type": "Double"
},
{
"Name": "email_sent_t",
"Type": "DateTime"
},
{
"Name": "email_subject_s",
"Type": "String"
},
{
"Name": "email_urls_s",
"Type": "String"
},
{
"Name": "email_verdict_s",
"Type": "String"
},
{
"Name": "email_vmray_uuid_g",
"Type": "Guid"
},
{
"Name": "email_webif_url_s",
"Type": "String"
},
{
"Name": "Type",
"Type": "String"
},
{
"Name": "_ResourceId",
"Type": "String"
}
]
}

35
Logos/vmray.svg Executable file
Просмотреть файл

@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg version="1.1" viewBox="0 0 400 400" xml:space="preserve"
xmlns="http://www.w3.org/2000/svg"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<metadata>
<rdf:RDF>
<cc:Work rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
</cc:Work>
</rdf:RDF>
</metadata>
<g transform="matrix(1.3333 0 0 -1.3333 0 400)">
<g transform="scale(.1)">
<path d="m1717.7 430.58h-67.12l-165.49 198.38 28.43 1.418c76.46 2.34 81.59 52.801 83.56 67.461 7.58 54.34-28.3 78.98-72.83 79.691l-75.99 0.321-52.21 44.808 126.1-0.449h1.71c97.34 0 135.53-59.75 127.81-116.42-1.87-40.168-26.57-85.641-79.73-104.33l145.76-170.88" fill="#1e516e"/>
<path d="m799.2 433.6h-69.523l-142.71 392.07h55.816l122.97-344.94 122.97 344.94h53.191l-142.72-392.07" fill="#1e516e"/>
<path d="m1347.3 433.6h-52.13v337.84l-109.02-229.88h-31.07l-108.23 229.88v-337.84h-48.708v392.07h71.098l104.54-218.28 101.11 218.28h72.41v-392.07" fill="#1e516e"/>
<path d="m2114.8 430.58h-55.56l-38.45 109.27h-169.57l-38.45-109.27h-52.92l142.72 392.08h69.51l142.72-392.08m-178.79 346.52-68.99-192.48h137.72l-68.73 192.48" fill="#1e516e"/>
<path d="m2275.1 430.58h-52.14v166.41l-138.51 225.67h57.67l107.18-175.37 108.22 175.37h55.56l-137.98-220.14v-171.94" fill="#1e516e"/>
<path d="m1500 1982.8-308 586.68h616l-308-586.68" fill="#1e516e"/>
<path d="m1023.3 1066.1h-623.33l550 1041.3 315.33-579.33-242-462" fill="#1e516e"/>
<path d="m2600 1066.1h-623.25l-121 231-120.92 231-0.03-0.08-0.04 0.08 315.33 579.33 549.91-1041.3" fill="#1e516e"/>
<path d="m1500 1066.1h-476.67l242 462 234.67-462" fill="#82a1af"/>
<path d="m1976.7 1066.1h-476.67l234.8 461.92 120.95-230.92 120.92-231" fill="#82a1af"/>
<path d="m1855.8 1297.1-120.95 230.92 0.03 0.08 120.92-231" fill="#0f4864"/>
<path d="m1265.3 1528.1-315.33 579.33 242 462.01 308-586.68-234.67-454.66" fill="#82a1af"/>
<path d="m1734.7 1528.1-234.67 454.66 308 586.68 242-462.01-315.33-579.33" fill="#82a1af"/>
<path d="m1500 1066.1-234.67 462 234.67 454.66 234.67-454.66-234.67-462" fill="#ccd9de"/>
<path d="m950 2107.4-242 462.01h484l-242-462.01" fill="#1e516e"/>
<path d="m2050 2107.4-242 462.01h484l-242-462.01" fill="#1e516e"/>
</g>
</g>
</svg>

После

Ширина:  |  Высота:  |  Размер: 2.6 KiB

Просмотреть файл

@ -0,0 +1,98 @@
[
{
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
"SourceSystem": "RestAPI",
"MG": "",
"ManagementGroupName": "",
"TimeGenerated": "2021-08-04T11:14:45Z",
"Computer": "",
"RawData": "",
"email_attachments_s": "[]",
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id 11E59160813;\\tWed, 4 Aug 2021 11:14:45 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============1502096034021184596==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807568462.9.9036724522137395510_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:14:44 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"4 questions answered about computer\"}]",
"email_message_id_s": "<162807568462.9.9036724522137395510_sanitized.com>",
"email_received_t": "2021-08-04T11:14:45Z",
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
"email_sender_s": "sanitized@sanitized.com",
"email_sensor_id_d": 1,
"email_sent_t": "2021-08-04T11:14:44Z",
"email_subject_s": "4 questions answered about computer",
"email_urls_s": "[]",
"email_verdict_s": "clean",
"email_vmray_uuid_g": "337317e3-cddb-4e14-a310-9c4aa4badd01",
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=337317e3-cddb-4e14-a310-9c4aa4badd01",
"Type": "vmray_emails_CL",
"_ResourceId": ""
},
{
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
"SourceSystem": "RestAPI",
"MG": "",
"ManagementGroupName": "",
"TimeGenerated": "2021-08-04T11:19:47Z",
"Computer": "",
"RawData": "",
"email_attachments_s": "[{\"attachment_filename\": \"cover1.jpg\", \"attachment_sha256hash\": \"82899dee46a0e34f9a7f2ecb664522e6cf556527702a7e482928ec6039e6c4b1\", \"attachment_verdict\": \"clean\"}]",
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id 6DCA2160813;\\tWed, 4 Aug 2021 11:19:45 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============0502589840769161396==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807598491.9.15160501248599303023_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:19:44 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"The fastest way to get better at anything\"}]",
"email_message_id_s": "<162807598491.9.15160501248599303023_sanitized.com>",
"email_received_t": "2021-08-04T11:19:45Z",
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
"email_sender_s": "sanitized@sanitized.com",
"email_sensor_id_d": 1,
"email_sent_t": "2021-08-04T11:19:44Z",
"email_subject_s": "The fastest way to get better at anything",
"email_urls_s": "[]",
"email_verdict_s": "clean",
"email_vmray_uuid_g": "23053748-da0e-4cce-81ca-9dd6b533b36e",
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=23053748-da0e-4cce-81ca-9dd6b533b36e",
"Type": "vmray_emails_CL",
"_ResourceId": ""
},
{
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
"SourceSystem": "RestAPI",
"MG": "",
"ManagementGroupName": "",
"TimeGenerated": "2021-08-04T11:22:26Z",
"Computer": "",
"RawData": "",
"email_attachments_s": "[{\"attachment_filename\": \"vs_BuildTools.exe\", \"attachment_sha256hash\": \"1778d923fd40c62c29f85cf4ef63a0c855a61c13b1dd68419bcda2cff38c984f\", \"attachment_verdict\": \"clean\"}, {\"attachment_filename\": \"Firefox Installer.exe\", \"attachment_sha256hash\": \"40cd4bef9a5dcb8a33840367a03932ecf06e82729c10b73d814853818ddad298\", \"attachment_verdict\": \"malicious\"}]",
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id E8AA8160813;\\tWed, 4 Aug 2021 11:22:17 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============7106368624508490837==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807613748.9.17138733245689342939_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:22:17 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"The fastest way to get better at anything\"}]",
"email_message_id_s": "<162807613748.9.17138733245689342939_sanitized.com>",
"email_received_t": "2021-08-04T11:22:18Z",
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
"email_sender_s": "sanitized@sanitized.com",
"email_sensor_id_d": 1,
"email_sent_t": "2021-08-04T11:22:17Z",
"email_subject_s": "The fastest way to get better at anything",
"email_urls_s": "[]",
"email_verdict_s": "malicious",
"email_vmray_uuid_g": "51948389-b7c5-4634-aa21-cf46e33436d2",
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=51948389-b7c5-4634-aa21-cf46e33436d2",
"Type": "vmray_emails_CL",
"_ResourceId": ""
},
{
"TenantId": "f29e0f7c-a590-4c87-b900-eefa6593b830",
"SourceSystem": "RestAPI",
"MG": "",
"ManagementGroupName": "",
"TimeGenerated": "2021-08-04T11:24:01Z",
"Computer": "",
"RawData": "",
"email_attachments_s": "[]",
"email_headers_s": "[{\"header_name\": \"Received\", \"header_value\": \"from [1.2.3.4] (unknown [5.6.7.8])\\tby b591a2cdbf81.localdomain (Postfix) with ESMTPS id DB521160813;\\tWed, 4 Aug 2021 11:24:00 +0000 (UTC)\"}, {\"header_name\": \"Content-Type\", \"header_value\": \"multipart/mixed; boundary=\\\"===============2951710851682953810==\\\"\"}, {\"header_name\": \"Mime-Version\", \"header_value\": \"1.0\"}, {\"header_name\": \"From\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"To\", \"header_value\": \"sanitized@sanitized.com\"}, {\"header_name\": \"Message-Id\", \"header_value\": \"<162807624057.9.8491175543255676034_sanitized.com>\"}, {\"header_name\": \"Date\", \"header_value\": \"Wed, 04 Aug 2021 11:24:00 +0000\"}, {\"header_name\": \"Subject\", \"header_value\": \"The uncomfortable truth about Python\"}]",
"email_message_id_s": "<162807624057.9.8491175543255676034_sanitized.com>",
"email_received_t": "2021-08-04T11:24:01Z",
"email_recipients_s": "[\"sanitized@sanitized.com\"]",
"email_sender_s": "sanitized@sanitized.com",
"email_sensor_id_d": 2,
"email_sent_t": "2021-08-04T11:24:00Z",
"email_subject_s": "The uncomfortable truth about Python",
"email_urls_s": "[{\"url_original_url\": \"https://www.microsofft.com/de-de/\", \"url_verdict\": \"suspicious\"}, {\"url_original_url\": \"https://en.wikipedia.org/wiki/Microsoft_Azure\", \"url_verdict\": \"clean\"}]",
"email_verdict_s": "suspicious",
"email_vmray_uuid_g": "a7c2f692-7611-49fc-abdb-174f9da60c17",
"email_webif_url_s": "https://vmray.example.com/etd/emails?vmray_uuid=a7c2f692-7611-49fc-abdb-174f9da60c17",
"Type": "vmray_emails_CL",
"_ResourceId": ""
}
]

Двоичные данные
Solutions/VMRay ETD/Data Connectors/VMRayETDFeeder.zip Normal file

Двоичный файл не отображается.

Просмотреть файл

@ -0,0 +1,299 @@
import base64
import hmac
import json
import logging
import os
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from email.utils import formatdate
from enum import Enum
from typing import Dict, Iterator, List, NamedTuple, Optional
import azure.functions as func
import requests
LOG_TYPE = "vmray_emails"
class Verdict(Enum):
clean = "clean"
suspicious = "suspicious"
malicious = "malicious"
@dataclass
class Config:
workspace_id: str
workspace_key: str
log_analytics_url: str
vmray_platform_url: str
vmray_api_key: str
poll_additional_fields: List[str]
poll_minimum_verdict: Verdict
verify_tls: bool
@classmethod
def from_environ(cls) -> "Config":
poll_additional_fields = []
for field in os.environ["POLL_ADDITIONAL_FIELDS"].split(","):
field = field.strip()
if field.startswith("email_"):
poll_additional_fields.append(field)
else:
logging.error(
"Ignoring invalid entry in POLL_ADDITIONAL_FIELDS: '%s'", field
)
try:
poll_minimum_verdict = Verdict(os.environ["POLL_MINIMUM_VERDICT"])
except ValueError:
logging.error("POLL_MINIMUM_VERDICT value is invalid; falling back to 'clean'")
poll_minimum_verdict = Verdict.clean
verify_tls_env = os.environ["VERIFY_VMRAY_PLATFORM_TLS"].lower()
try:
verify_tls = {"true": True, "false": False}[verify_tls_env]
except KeyError:
logging.error("VERIFY_VMRAY_PLATFORM_TLS value is invalid; defaulting to 'true'")
verify_tls = True
return cls(
workspace_id=os.environ["WORKSPACE_ID"],
workspace_key=os.environ["WORKSPACE_KEY"],
log_analytics_url=os.environ["LOG_ANALYTICS_URL"],
vmray_platform_url=os.environ["VMRAY_PLATFORM_URL"],
vmray_api_key=os.environ["VMRAY_API_KEY"],
poll_additional_fields=poll_additional_fields,
poll_minimum_verdict=poll_minimum_verdict,
verify_tls=verify_tls,
)
@dataclass
class State:
last_poll: datetime
@classmethod
def load(cls, raw: str) -> "State":
dict_ = json.loads(raw)
return cls(
last_poll=datetime.fromisoformat(dict_["last_poll"]),
)
def dump(self) -> str:
dict_ = {
"last_poll": self.last_poll.isoformat(),
}
return json.dumps(dict_)
class APIError(Exception):
@classmethod
def expected(cls, code: int, message: str) -> "APIError":
return cls(f"({code}) {message}")
@classmethod
def unexpected(cls, resp: requests.Response) -> "APIError":
return cls(f"Unexpected error: ({resp.status_code}) {resp.text}")
class FeederError(Exception):
pass
EmailInfo = Dict[str, object]
REQUIRED_API_FIELDS = [
"email_vmray_uuid",
"email_message_id",
"email_sent",
"email_received",
"email_sensor_id",
"email_verdict",
"email_verdict_reached",
"email_webif_url",
]
class Feeder:
def __init__(self, config: Config, state: State):
self.config = config
self.state = state
@property
def api_url(self) -> str:
return f"{self.config.vmray_platform_url}/rest"
@property
def auth_headers(self) -> Dict[str, str]:
return {"Authorization": f"api_key {self.config.vmray_api_key}"}
def feed(self) -> None:
"""
Feed email data fetched from the ETD API into Log Analytics.
We feed the data in batches to limit our memory usage and to
ensure the requests to the Log Analytics API don't become too
large.
"""
poll_from = self.state.last_poll
poll_until = datetime.now(tz=timezone.utc)
logging.info("Polling from %s until %s", poll_from, poll_until)
batches_iter = self.poll_etd(poll_from, poll_until)
count = 0
while True:
try:
batch = next(batches_iter)
except (requests.RequestException, APIError) as exc:
raise FeederError(f"Error polling from VMRay ETD: {exc}")
except StopIteration:
break
try:
self.push_to_log_analytics(batch)
except (requests.RequestException, APIError) as exc:
raise FeederError(f"Error pushing to Log Analytics: {exc}")
count += len(batch)
logging.info("Pushed %d emails", count)
self.state.last_poll = poll_until
def poll_etd(self, start: datetime, end: datetime) -> Iterator[List[EmailInfo]]:
time_fmt = "%Y-%m-%dT%H:%M:%S.%f"
poll_interval = f"{start.strftime(time_fmt)}~{end.strftime(time_fmt)}"
fields = REQUIRED_API_FIELDS + self.config.poll_additional_fields
fields_spec = "(" + ",".join(fields) + ")"
fixed_params = {
"email_verdict_reached": poll_interval,
"_fields": fields_spec,
}
verdict_filters_map: Dict[Verdict, List[Optional[str]]] = {
Verdict.clean: [None],
Verdict.suspicious: ["suspicious", "malicious"],
Verdict.malicious: ["malicious"],
}
verdict_filters = verdict_filters_map[self.config.poll_minimum_verdict]
for verdict in verdict_filters:
if verdict:
params = {"email_verdict": verdict, **fixed_params}
else:
params = fixed_params
yield from (b for b in self.poll_etd_with_params(params) if b)
def poll_etd_with_params(self, params: Dict[str, str]) -> Iterator[List[EmailInfo]]:
resp = requests.get(
url=f"{self.api_url}/email",
headers=self.auth_headers,
params=params,
verify=self.config.verify_tls,
)
etd_resp = extract_etd_response(resp)
yield etd_resp.emails
while etd_resp.continuation_id:
resp = requests.get(
url=f"{self.api_url}/continuation/{etd_resp.continuation_id}",
headers=self.auth_headers,
verify=self.config.verify_tls,
)
etd_resp = extract_etd_response(resp)
yield etd_resp.emails
def push_to_log_analytics(self, email_data: List[EmailInfo]) -> None:
method = "POST"
resource = "/api/logs"
content_type = "application/json"
data = json.dumps(email_data)
content_length = len(data)
x_ms_date = formatdate(usegmt=True)
def build_signature() -> str:
string_to_sign = "\n".join(
[
method,
str(content_length),
content_type,
f"x-ms-date:{x_ms_date}",
resource,
]
)
bytes_to_sign = string_to_sign.encode("utf-8")
key = base64.b64decode(self.config.workspace_key)
mac = hmac.digest(key, bytes_to_sign, "sha256")
signature = base64.b64encode(mac).decode()
return signature
signature = build_signature()
shared_key = f"{self.config.workspace_id}:{signature}"
resp = requests.request(
method=method,
url=f"{self.config.log_analytics_url}{resource}",
params={"api-version": "2016-04-01"},
headers={
"Authorization": f"SharedKey {shared_key}",
"Content-Type": content_type,
"Log-Type": LOG_TYPE,
"x-ms-date": x_ms_date,
"time-generated-field": "email_verdict_reached",
},
data=data,
)
success = 200 <= resp.status_code < 300
if not success:
raise APIError.unexpected(resp)
class ETDResponse(NamedTuple):
emails: List[EmailInfo]
continuation_id: Optional[int]
def extract_etd_response(resp: requests.Response) -> ETDResponse:
try:
resp_json = resp.json()
result = resp_json["result"]
if result == "ok":
emails = resp_json["data"]
continuation_id = resp_json.get("continuation_id")
else:
error_msg = resp_json["error_msg"]
raise APIError.expected(resp.status_code, error_msg)
except (ValueError, KeyError):
raise APIError.unexpected(resp)
return ETDResponse(emails, continuation_id)
def main(timer: func.TimerRequest, state: func.InputStream) -> str:
if state:
decoded = state.read().decode()
state_ = State.load(decoded)
else:
now = datetime.now(tz=timezone.utc)
state_ = State(
last_poll=now - timedelta(minutes=10),
)
config = Config.from_environ()
try:
feeder = Feeder(config, state_)
feeder.feed()
except FeederError as exc:
logging.error("%s", exc)
raise
return feeder.state.dump()

Просмотреть файл

@ -0,0 +1,23 @@
{
"scriptFile": "__init__.py",
"bindings": [
{
"name": "timer",
"type": "timerTrigger",
"direction": "in",
"schedule": "%POLLING_SCHEDULE%"
},
{
"name": "state",
"type": "blob",
"path": "vmray-etd-feeder/state.json",
"direction": "in"
},
{
"name": "$return",
"type": "blob",
"path": "vmray-etd-feeder/state.json",
"direction": "out"
}
]
}

Просмотреть файл

@ -0,0 +1,125 @@
{
"id": "VMRayETD",
"title": "VMRay Email Threat Defender (ETD)",
"publisher": "VMRay",
"descriptionMarkdown": "This connector ingests email data collected by [VMRay Email Threat Defender (ETD)](https://www.vmray.com/products/email-threat-defender/). It requires a connection to a VMRay Platform 4.3.0 (or later) installation that has ETD enabled.",
"graphQueries": [
{
"metricName": "Total data received",
"legend": "Emails",
"baseQuery": "vmray_emails_CL"
}
],
"sampleQueries": [
{
"description": "All ingested emails",
"query": "vmray_emails_CL\n| order by TimeGenerated desc"
},
{
"description" : "Malicious emails",
"query": "vmray_emails_CL\n| where email_verdict_s == \"malicious\""
},
{
"description": "Recipients of malicious emails",
"query": "vmray_emails_CL\n| where email_verdict_s == \"malicious\"\n| extend recipient=parse_json(email_recipients_s)\n| mv-expand recipient to typeof(string)\n| distinct recipient"
},
{
"description": "Malicious attachments",
"query": "vmray_emails_CL\n| where email_attachments_s has '\"malicious\"'\n| extend attachment=parse_json(email_attachments_s)\n| mv-expand attachment\n| where attachment.attachment_verdict == \"malicious\"\n| extend filename=tostring(attachment.attachment_filename)\n| extend hash=tostring(attachment.attachment_sha256hash)\n| summarize filenames=make_set(filename, 5) by hash"
}
],
"dataTypes": [
{
"name": "vmray_emails_CL",
"lastDataReceivedQuery": "vmray_emails_CL\n| summarize Time = max(TimeGenerated)\n| where isnotempty(Time)"
}
],
"connectivityCriterias": [
{
"type": "IsConnectedQuery",
"value": [
"vmray_emails_CL\n| summarize LastLogReceived = max(TimeGenerated)\n| project IsConnected = LastLogReceived > ago(30d)"
]
}
],
"availability": {
"status": 1,
"isPreview": true
},
"permissions": {
"resourceProvider": [
{
"provider": "Microsoft.OperationalInsights/workspaces",
"permissionsDisplayText": "read and write permissions on the workspace are required.",
"providerDisplayName": "Workspace",
"scope": "Workspace",
"requiredPermissions": {
"write": true,
"read": true,
"delete": true
}
},
{
"provider": "Microsoft.OperationalInsights/workspaces/sharedKeys",
"permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).",
"providerDisplayName": "Keys",
"scope": "Workspace",
"requiredPermissions": {
"action": true
}
}
],
"customs": [
{
"name": "Microsoft.Web/sites permissions",
"description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)."
},
{
"name": "VMRay Platform Access",
"description": "Access to a VMRay Platform 4.3.0 (or later) installation that has ETD enabled. Ensure that the VMRay Platform's REST API is accessible to external clients so that the connector can fetch data through it."
}
]
},
"instructionSteps": [
{
"title": "",
"description": ">**NOTE:** This connector uses Azure Functions to connect to a VMRay Platform API to pull ETD email data into Azure Sentinel. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details."
},
{
"title": "",
"description": ">**(Optional Step)** Securely store Workspace and API keys in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Function App."
},
{
"title": "",
"description": "**STEP 1 - Configuration Steps for the VMRay Platform**\n\nIn the VMRay Platform, create a user that has permissions to access ETD and the email data you want to ingest. If you are using ETD Cloud, see the *Setting up and Managing Users* chapter of the *Cloud Account Manager Guide* for more information. If you are using ETD On Premises, see the *Setting up Users and Assigning Roles* section of the *On Premises Installation Guide* for more information.\n\nNext, create an API key for that user, which will be used by the Azure Sentinel connector. For detailed instructions, refer to the *API Programmer Guide*.\n\nVerify that your API key has the required permissions by running the following command:\n\n\t$ curl -H \"Authorization: api_key <vmray_api_key>\" <vmray_platform_url>/rest/email\nThis should return a list of emails that ETD has analyzed."
},
{
"title": "",
"description": "**STEP 2 - Deploy the Azure Resource Manager (ARM) Template**\n\n>**IMPORTANT:** Before deploying the VMRay ETD connector, have the Azure Sentinel Workspace ID and Primary Key (can be copied from below), as well as the VMRay Platform API key, readily available.",
"instructions": [
{
"parameters": {
"fillWith": [
"WorkspaceId"
],
"label": "Workspace ID"
},
"type": "CopyableLabel"
},
{
"parameters": {
"fillWith": [
"PrimaryKey"
],
"label": "Workspace Primary Key"
},
"type": "CopyableLabel"
}
]
},
{
"title": "",
"description": "1. Click the **Deploy to Azure** button below.\n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-VMRayETD-azuredeploy)\n2. Select the preferred **Subscription**, **Resource group** and **Region**.\n3. Enter the **Workspace ID**, **Workspace Key**, **VMRay Platform URL** and **VMRay API Key**.\n>Note: If you are using Azure Key Vault secrets for any of the values above, use the`@Microsoft.KeyVault(SecretUri={Security Identifier})`schema in place of the string values. Refer to the [Key Vault references documentation](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) for further details.\nAdjust the other connector settings as required.\n\n4. Finish the wizard and wait for the deployment to complete."
}
]
}

Просмотреть файл

@ -0,0 +1,177 @@
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"ResourcePrefix": {
"type": "string",
"defaultValue": "vmrayetd",
"minLength": 1,
"maxLength": 11,
"metadata": {
"description": "Prefix for all resources created by this Azure Resource Manager (ARM) template."
}
},
"WorkspaceID": {
"type": "string",
"metadata": {
"description": "Azure Sentinel Workspace ID."
}
},
"WorkspaceKey": {
"type": "securestring",
"metadata": {
"description": "Azure Sentinel Workspace Primary Key."
}
},
"VmrayPlatformURL": {
"type": "string",
"metadata": {
"description": "URL of the VMRay Platform from which email data should be ingested."
}
},
"VmrayAPIKey": {
"type": "securestring",
"metadata": {
"description": "VMRay Platform API key with sufficient permissions to access email data. For example: 'https://eu.cloud.vmray.com'"
}
},
"PollingSchedule": {
"type": "string",
"defaultValue": "0 * * * * *",
"metadata": {
"description": "Cron expression specifying the schedule for fetching data from the VMRay Platform."
}
},
"PollAdditionalFields": {
"type": "string",
"defaultValue": "email_sender, email_recipients, email_subject, email_headers, email_attachments, email_urls",
"metadata": {
"description": "Comma-separated list of email fields that should be fetched in addition to the required fields. See the ETD API Reference, which is included with the VMRay Platform documentation, for complete information about the /rest/email endpoint, including a list of available email fields."
}
},
"PollMinimumVerdict": {
"type": "string",
"defaultValue": "clean",
"allowedValues": [
"clean",
"suspicious",
"malicious"
],
"metadata": {
"description": "Verdict filter for fetched emails. Specify the least severe verdict you want to fetch. For example, to fetch only suspicious and malicious emails, set this to 'suspicious'. To select all verdicts, specify 'clean'."
}
},
"VerifyVmrayPlatformTLS": {
"type": "bool",
"defaultValue": "true",
"metadata": {
"description": "Set this to 'false' if you want to ignore TLS certificate errors when connecting to the VMRay Platform. This is not recommended for production but can be useful during testing."
}
}
},
"variables": {
"resourceName": "[concat(toLower(parameters('ResourcePrefix')), uniqueString(resourceGroup().id))]",
"storageSuffix": "[tolower(environment().suffixes.storage)]",
"logAnalyticsURL": "[replace(environment().portal, 'https://portal', concat('https://', toLower(parameters('WorkspaceID')), '.ods.opinsights'))]"
},
"resources": [
{
"type": "Microsoft.Storage/storageAccounts",
"apiVersion": "2021-02-01",
"name": "[variables('resourceName')]",
"location": "[resourceGroup().location]",
"kind": "StorageV2",
"sku": {
"name": "Standard_LRS",
"tier": "Standard"
},
"properties": {}
},
{
"type": "Microsoft.Insights/components",
"apiVersion": "2015-05-01",
"name": "[variables('resourceName')]",
"location": "[resourceGroup().location]",
"kind": "web",
"properties": {
"Application_Type": "web",
"ApplicationId": "[variables('resourceName')]"
}
},
{
"type": "Microsoft.Web/sites",
"apiVersion": "2020-12-01",
"name": "[variables('resourceName')]",
"location": "[resourceGroup().location]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts', variables('resourceName'))]",
"[resourceId('Microsoft.Insights/components', variables('resourceName'))]"
],
"kind": "functionapp,linux",
"properties": {
"reserved": true,
"siteConfig": {
"linuxFxVersion": "python|3.8",
"appSettings": [
{
"name": "AzureWebJobsStorage",
"value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('resourceName'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('resourceName')), '2021-02-01').keys[0].value, ';EndpointSuffix=', variables('storageSuffix'))]"
},
{
"name": "APPINSIGHTS_INSTRUMENTATIONKEY",
"value": "[reference(resourceId('Microsoft.insights/components', variables('resourceName'))).InstrumentationKey]"
},
{
"name": "FUNCTIONS_EXTENSION_VERSION",
"value": "~3"
},
{
"name": "FUNCTIONS_WORKER_RUNTIME",
"value": "python"
},
{
"name": "WEBSITE_RUN_FROM_PACKAGE",
"value": "https://aka.ms/sentinel-VMRayETD-functionapp"
},
{
"name": "WORKSPACE_ID",
"value": "[parameters('WorkspaceID')]"
},
{
"name": "WORKSPACE_KEY",
"value": "[parameters('WorkspaceKey')]"
},
{
"name": "LOG_ANALYTICS_URL",
"value": "[variables('logAnalyticsURL')]"
},
{
"name": "VMRAY_PLATFORM_URL",
"value": "[parameters('VmrayPlatformURL')]"
},
{
"name": "VMRAY_API_KEY",
"value": "[parameters('VmrayAPIKey')]"
},
{
"name": "POLLING_SCHEDULE",
"value": "[parameters('PollingSchedule')]"
},
{
"name": "POLL_ADDITIONAL_FIELDS",
"value": "[parameters('PollAdditionalFields')]"
},
{
"name": "POLL_MINIMUM_VERDICT",
"value": "[parameters('PollMinimumVerdict')]"
},
{
"name": "VERIFY_VMRAY_PLATFORM_TLS",
"value": "[parameters('VerifyVmrayPlatformTLS')]"
}
]
}
}
}
]
}

Просмотреть файл

@ -0,0 +1,14 @@
{
"version": "2.0",
"extensionBundle": {
"id": "Microsoft.Azure.Functions.ExtensionBundle",
"version": "[2.*, 3.0.0)"
},
"logging": {
"applicationInsights": {
"samplingSettings": {
"isEnabled": false
}
}
}
}

Просмотреть файл

@ -0,0 +1,2 @@
azure-functions
requests==2.25.1