Azure
/
Azure-Sentinel
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Updated readme and deploy to Azure

This commit is contained in:
Matt Egen 2021-09-29 08:05:17 -07:00
Родитель bdf622b995
Коммит 7a5d57b851
9 изменённых файлов: 346 добавлений и 39 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

Двоичные данные
Tools/RDAP/RDAPQuery/Documentation/Drawing1.vsdx Normal file
Просмотреть файл

Двоичный файл не отображается.

Двоичные данные
Tools/RDAP/RDAPQuery/Documentation/ProcessWorkflow.jpg Normal file
Просмотреть файл

Двоичный файл не отображается.
Режим "рядом"

После

Ширина:  |  Высота:  |  Размер: 106 KiB

2
Tools/RDAP/RDAPQuery/RDAPQuery.sln
Просмотреть файл

@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.31129.286
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RDAPQuery", "RDAPQuery\RDAPQuery.csproj", "{19CEA8D8-3865-4ADD-BFB3-E0325D0F8EB6}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RDAPQuery", "RDAPQuery\RDAPQuery.csproj", "{19CEA8D8-3865-4ADD-BFB3-E0325D0F8EB6}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

Двоичные данные
Tools/RDAP/RDAPQuery/RDAPQuery.zip Normal file
Просмотреть файл

Двоичный файл не отображается.

47
Tools/RDAP/RDAPQuery/RDAPQuery/LogAnalytics.cs
Просмотреть файл

@ -1,18 +1,11 @@
// ***********************************************************************
// Assembly : RDAPQuery
// Author : MattEgen
// Author : Matt Egen @FlyingBlueMonkey
// Created : 04-16-2021
//
// Last Modified By : MattEgen
// Last Modified On : 04-26-2021
// ***********************************************************************
// <copyright file="LogAnalytics.cs" company="">
// Copyright (c) . All rights reserved.
// </copyright>
// <summary></summary>
// ***********************************************************************
// Last Modified By : Matt Egen @FlyingBlueMonkey
// Last Modified On : 05-30-2021
using Newtonsoft.Json;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Net.Http;
@ -68,7 +61,7 @@ namespace RDAPQuery
Token token = null;
HttpClient client = new HttpClient();
#endregion
Console.Write("Received Call to GetbearerToken");
// Set the base address of the HttpClient object
client.BaseAddress = new Uri(baseAddress);
// Add an Accept header for JSON format.
@ -173,6 +166,7 @@ namespace RDAPQuery
/// <returns>QueryResults.</returns>
public static async Task<QueryResults> QueryData(string query)
{
Console.Write("Received Call to QueryData, calling GetBearerToken");
//Get the authorization bearer token
Task<Token> task = GetBearerToken();
Token token = task.Result;
@ -187,21 +181,32 @@ namespace RDAPQuery
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.AccessToken);
var data = new StringContent(query, Encoding.UTF8, "application/json");
HttpResponseMessage queryResponse = await client.PostAsync(baseAddress,data);
if (queryResponse.IsSuccessStatusCode)
try
{
var jsonContent = await queryResponse.Content.ReadAsStringAsync();
var jsonObject = JsonConvert.DeserializeObject<QueryResults>(jsonContent);
Console.WriteLine("Calling LogAnalytics in QueryData");
HttpResponseMessage queryResponse = await client.PostAsync(baseAddress, data);
if (queryResponse.IsSuccessStatusCode)
{
var jsonContent = await queryResponse.Content.ReadAsStringAsync();
var jsonObject = JsonConvert.DeserializeObject<QueryResults>(jsonContent);
client.Dispose();
return jsonObject;
}
else
{
Console.WriteLine("{0} ({1})", (int)queryResponse.StatusCode, queryResponse.ReasonPhrase);
}
// Dispose of the client since all HttpClient calls are complete.
client.Dispose();
return jsonObject;
return null;
}
else
catch (Exception ex)
{
Console.WriteLine("{0} ({1})", (int)queryResponse.StatusCode, queryResponse.ReasonPhrase);
Console.WriteLine(ex.Message);
client.Dispose();
return null;
}
// Dispose of the client since all HttpClient calls are complete.
client.Dispose();
return null;
}
/// <summary>

16
Tools/RDAP/RDAPQuery/RDAPQuery/QueryEngine.cs
Просмотреть файл

@ -1,16 +1,11 @@
// ***********************************************************************
// Assembly : RDAPQuery
// Author : MattEgen
// Author : Matt Egen @FlyingBlueMonkey
// Created : 04-13-2021
//
// Last Modified By : MattEgen
// Last Modified On : 04-27-2021
// ***********************************************************************
// <copyright file="QueryEngine.cs" company="">
// Copyright (c) . All rights reserved.
// </copyright>
// <summary></summary>
// ***********************************************************************
// Last Modified By : Matt Egen @FlyingBlueMonkey
// Last Modified On : 05-30-2021
using System;
using System.Collections.Generic;
using System.IO;
@ -50,7 +45,7 @@ namespace RDAPQuery
/// <param name="myTimer">My timer.</param>
/// <param name="log">The log.</param>
[FunctionName("CheckDomains")]
public static void Run([TimerTrigger("0 */5 * * * *")]TimerInfo myTimer, ILogger log)
public static void Run([TimerTrigger("0 */30 * * * *")]TimerInfo myTimer, ILogger log)
{
//Log the initiation of the function
log.LogInformation($"CheckDomains Timer trigger function executed at: {DateTime.Now}");
@ -58,6 +53,7 @@ namespace RDAPQuery
try
{
string queryBody = GetEnvironmentVariable("query_string");
log.LogInformation(string.Format("Calling QueryData with query '{0}'", queryBody));
Task<QueryResults> task = LogAnalytics.QueryData(queryBody);
QueryResults results = task.Result;
//Ok, now that we have our domains, for each domain returned, call the bootstrap service and get the responsible server

8
Tools/RDAP/RDAPQuery/RDAPQuery/RDAPQuery.csproj
Просмотреть файл

@ -2,15 +2,15 @@
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
<Authors>Matt Egen</Authors>
<Authors>Matt Egen @FlyingBlueMonkey</Authors>
<Company />
<Product />
<Description>RDAP Query is an Azure Function based query engine to retrieve Whois / Registry Data Access Protocol data about domains.</Description>
<Description>RDAP Query is an Azure Function based query engine to retrieve Whois / Registry Data Access Protocol data about domains. The software is provided "as is," with all faults, defects, bugs, and errors</Description>
<PackageReleaseNotes>The software is provided "as is," with all faults, defects, bugs, and errors</PackageReleaseNotes>
<AssemblyVersion>1.0.0.0</AssemblyVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.11" />
<PackageReference Include="RestSharp" Version="106.11.7" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.13" />
</ItemGroup>
<ItemGroup>
<None Update="host.json">

104
Tools/RDAP/RDAPQuery/README.md
Просмотреть файл

@ -1,5 +1,103 @@
# Whois/Registration Data Access Protocol (RDAP) Query Azure Function
# Registration Data Access Protocol (RDAP) Query Engine
----
As top level domains (and domains in general) have increased, there is a need to be able to lookup information about domains such as when they were registered, who owns them, etc. This project is designed to solve this need (in an albeit limited use case for now) by retrieving domain(s) from Azure Sentinel / Log Analytics, querying the RDAP network for registration information, and then writing that resolution information back in to Azure Sentinel / Log Analytics.
Author: Matt Egen
mattegen@microsoft.com
<a href="https://twitter.com/FlyingBlueMonki?ref_src=twsrc%5Etfw" class="twitter-follow-button" data-show-count="true">Follow @FlyingBlueMonki on Twitter</a>
With the ever increasing number of new domains on the Internet as well as all of the new Top Level Domains (TLD), it's often hard to know if a user has gone to a potentially malicious new site that has just popped up online. To help with this, a SOC team or analyst could track for users accessing newly registered domains. One way to do this is to query the Registration Data Access Protocol (RDAP). RDAP allows you to access domain name registration data (much like its predecesor the WHOIS protocol does today) but via an API call and with a better, more machine readable structure to the data. This Azure Function queries an Azure Sentinel environment, finds domain names of interest, and then conducts an RDAP lookup to retrieve information about the domain for investigators and analysts. There is also an Azure Sentinel Analytic rule that can then alert if evidence of a domain that was registered in the last 30 days should be found.
Please note: This version only stores the registration date of the domains successfully resolved, but you could modify it to store more information such as who registered the domain, address information, contact data etc.
Also note: Not all TLD's support RDAP. The current version of the code does not account for this _except_ to ignore TLD's you specifiy. A future version of this function is planned to support traditional WHOIS lookups as well if there is enough interest.
## Setup Steps
When deploying this Azure Function you will need some values to fill in the blanks for the Azure Resource Manager (ARM) Template:
### Function Name
The name you want to give the Azure Function. The Default is "RDAPQuery". A unique string will be attached to this in order to deconflict with any potential pre-existing functions you may have. For example, the template may create a name like "rdapquerytbdem24sevgdq"
### Azure AD App Permissions
The Azure Function needs permission to read the Log Analytics Workspace that your Azure Sentinel instance is attached to. For guidance on creating an Azure AD App Registration, please see [QuickStart: Register App](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app). For this application you will need the following permissions:
````
Log Analytics API
Data.Read
````
After registering your application, you will need the Client ID and Client Secret for the ARM Template.
### Azure Tenant ID
To authenticate, the Azure Function will need to know your Tenant ID to call the proper Azure AD authentication end point
## Azure Log Analytics Resource Location
This value is used in building the Token to read data from your Log Analytics environment. It specifies the service you're trying to access. It is in the format of "https://[location].api.loganalytics.io". For example, a resource in westus2 is "https://westus2.api.loganalytics.io". You should specify the entire path (e.g. "https://westus2.api.loganalytics.io" not just "westus2"
### Workspace ID and Workspace Shared Key
The Azure Function will write data to the Log Analytics Workspace that your Azure Sentinel instance is attached to. This data is available on the Agents Management page of your Log Analytics Workspace. You can get to this page in Sentinel by going to Settings --> Workspace Settings --> Agents Management. While there is only one Workspace ID, there are two possible keys for you to choose from, either one will work.
### Log Analytics Custom Log Name
The name you want to use for your resolved domains. The default is ResolvedDomains. Note: Log Analytics will automatically append "\_CL" to the end of whatever string you enter here.
[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FFlyingBlueMonkey%2FRDAPQuery%2Fmaster%2Fazuredeploy.json)
### Post Template Configuration
After deploying the ARM Template, you should go in to your Azure Sentinel instance and create the GetDomainsForRDAP function. An example is included in this repo, but you can use any function you want so long as it returns a field named "Domain" that has the domain you are looking up information for.
## Description and Workflow
----
RDAP Query Engine is comprised of two main components each of which has some subcomponents
### The Log Analytics Queries and Custom Tables
GetDomainsForRDAP - Log Analytics Function
Since we're looking for information about domains in our Azure Sentinel environment, we have to first get those domains. For this version, I'm only searching in one table: DeviceNetworkEvents. DeviceNetworkEvents is a table from M365 Defender and has the networking events that occurred on the enrolled devices in my environment. To get the domains, I've created a function called GetDomainsForRDAP which looks like this:
````
// A dynamic list of domains and TLDs to not bother searching for
let ExcludedDomains = dynamic(["cn","io", "ms"]);
DeviceNetworkEvents
| where TimeGenerated >= ago(1h)
| where isnotempty(RemoteUrl)
// A little cleanup just in case
| extend parsedDomain = case(RemoteUrl contains "//", parse_url(RemoteUrl).Host, RemoteUrl)
| extend cleanDomain = split(parsedDomain,"/")[0]
| extend splitDomain = split(cleanDomain,".")
| extend Domain = tolower(strcat(splitDomain[array_length(splitDomain)-2],".",splitDomain[array_length(splitDomain)-1]))
| extend TLD = splitDomain[array_length(splitDomain)-1]
| where TLD !in(ExcludedDomains)
| where Domain !in(ExcludedDomains)
| summarize DistinctDomain = dcount(Domain) by Domain
| project Domain
// | join kind=leftanti (ResolvedDomains_CL | where TimeGenerated >= ago(90d)) on $left.Domain == $right.domainName_s //Uncomment this line after the FIRST run of the Azure Function. Otherwise the query will throw an error until the ResolvedDomains_CL table is instantiated
````
This function extracts the domain values from the DeviceNetworkEvents table, de-duplicates them, and then does a leftanti join to the ResolvedDomains_CL table (leftanti meaning "show me everything in the left table (DeviceNetworkEvents) that's not in the right table (ResolvedDomains_CL)"). ResolvedDomains_CL is where the Azure Function stores its results. Thus if we've already looked up this domain in the last 90 days (or longer depending on how long you want to retain the data) we don't bother looking it up again. The resulting domains are used in the Azure Function to resolve their information in RDAP. Note: Using a Log Analytics function like this means that we can change the source data (include more / different sources) by just changing the function query without having to modify the RDAP Query Engine code itself.
### ResolvedDomains - Log Analytics Custom Log
This is the custom log where RDAP Query Engine writes its results to. In my current implementation the tables consists of the domain name (domainName_s) and domain registration date (registrationDate_t). There is a LOT more information available in an RDAP record, however for this particular case I merely wanted to know when a domain was registered so that if a user were to travel to a domain that was registered within the last 30 days I could raise an alert. This is a configurable value.
### Newly Registered Domain Detected - Azure Sentinel Analytic Rule
This rule is based on the information contained in the ResolvedDomains_CL log table. It runs once an hour and looks for any new records in the ResolvedDomains_CL table that have a registration date within the last 30 days
````
ResolvedDomains_CL
| where TimeGenerated >= ago(1h)
| where registrationDate_t >= ago(30d)
````
If so, it then raises an alert in Azure Sentinel for an analyst to investigate
## The Azure Function
Now that we understand where the source data is coming from, time to look at the Azure Function itself. The basic Azure Function triggered every 30 minutes and the first thing it does is call into LogAnalytics and runs the GetDomainsForRDAP Function. For each domain that is returned, it extracts the Top Level Domain (TLD) and calls the RDAP BootStrap server to find the server that handles that particular TLD. For example, if the domain we're looking up is crazyfunnyhats.com the function extracts the TLD as "com", calls the bootstrap server (https://data.iana.org/rdap/dns.json) and discovers that the COM TLD is serviced by "https://rdap.verisign.com/com/v1/". It then calls that service endpoint with the properly constructed URI ("https://rdap.verisign.com/com/v1/domain/crazyfunnyhats.com") parses the results and then calls the Log Analytics API to insert the registration data into the ResolvedDomains_CL table.
![Workflow Diagram](/Documentation/ProcessWorkflow.jpg)
## Gotchas / Issues / Bugs
----
The following are some issues I’ve run into on this project. I am still working on more elegant solutions for them, but for now the workarounds (if available) seem to work.
##### GetDomainsForRDAP returns IP addresses (both IPv4 and IPv6) and they don't resolve
This is an artifact of the way that DeviceNetworkEvents returns data. If the target was an IP address then DeviceNetworkEvents returns that information. Since RDAP Query Engine doesn't process IP addresses at this time there are some remnants left in the GetDomainsForRDAP query. These are ignored / error out in the subsequent RDAP query so they don't cause any issues. Future plans are to clean up the GetDomainsForRDAP function to ignore these scenarios completely.
RDAPQuery is an Azure Function written in c# that runs on a timer (default 10 minutes) and executes a query against your Azure Sentinel environment (GetDomainsForRDAP) to retrieve a list of the domains that have recently been seen in your environment.

208
Tools/RDAP/RDAPQuery/azuredeploy.json Normal file
Просмотреть файл

@ -0,0 +1,208 @@
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"functionName": {
"defaultValue": "RDAPQuery",
"type": "string",
"metadata": {
"description": "Specifies the name of the Function App."
}
},
"WorkspaceID": {
"type": "string",
"metadata": {
"description": "Specifies the Log Analytics Workspace Id."
}
},
"SharedKey": {
"type": "securestring",
"metadata": {
"description": "Specifies the Log Analytics Workspace Key."
}
},
"TenantID": {
"type": "string",
"metadata": {
"description": "Specifies your Azure AD Tenant ID."
}
},
"clientID": {
"type": "string",
"metadata": {
"description": "Specifies the client ID for the application."
}
},
"clientSecret": {
"type": "securestring",
"metadata": {
"description": "Specifies the shared secret used by the application."
}
},
"ResourceLocation": {
"defaultValue": "https://westus2.api.loganalytics.io",
"type": "string",
"metadata": {
"description": "Specifies the root location of your log analytics instance."
}
},
"LogAnalyticsCustomLogName": {
"defaultValue": "ResolvedDomains",
"type": "string",
"metadata": {
"description": "Specifies Azure Log Analytics Workspace table name to store resolved domains."
}
}
},
"variables": {
"functionName": "[concat(toLower(parameters('functionName')), uniqueString(resourceGroup().id))]",
"StorageAccountName": "[substring(variables('functionName'), 0, 22)]",
"HostingPlanName": "[concat('ASP-',substring(variables('functionName'), 0, 22))]",
"StorageSuffix": "[environment().suffixes.storage]",
"LogAnaltyicsUri": "[replace(environment().portal, 'https://portal', concat('https://', toLower(parameters('WorkspaceId')), '.ods.opinsights'))]"
},
"resources": [
{
"type": "Microsoft.Insights/components",
"apiVersion": "2015-05-01",
"name": "[variables('FunctionName')]",
"location": "[resourceGroup().location]",
"kind": "web",
"properties": {
"Application_Type": "web",
"ApplicationId": "[variables('FunctionName')]"
}
},
{
"type": "Microsoft.Storage/storageAccounts",
"apiVersion": "2019-06-01",
"name": "[variables('StorageAccountName')]",
"location": "[resourceGroup().location]",
"sku": {
"name": "Standard_LRS",
"tier": "Standard"
},
"kind": "StorageV2",
"properties": {
"networkAcls": {
"bypass": "AzureServices",
"virtualNetworkRules": [],
"ipRules": [],
"defaultAction": "Allow"
},
"supportsHttpsTrafficOnly": true,
"encryption": {
"services": {
"file": {
"keyType": "Account",
"enabled": true
},
"blob": {
"keyType": "Account",
"enabled": true
}
},
"keySource": "Microsoft.Storage"
}
}
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices",
"apiVersion": "2019-06-01",
"name": "[concat(variables('StorageAccountName'), '/default')]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccountName'))]"
],
"sku": {
"name": "Standard_LRS",
"tier": "Standard"
},
"properties": {
"cors": {
"corsRules": []
},
"deleteRetentionPolicy": {
"enabled": false
}
}
},
{
"type": "Microsoft.Storage/storageAccounts/fileServices",
"apiVersion": "2019-06-01",
"name": "[concat(variables('StorageAccountName'), '/default')]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccountName'))]"
],
"sku": {
"name": "Standard_LRS",
"tier": "Standard"
},
"properties": {
"cors": {
"corsRules": []
}
}
},
{
"apiVersion": "2018-02-01",
"name": "[variables('HostingPlanName')]",
"type": "Microsoft.Web/serverfarms",
"location": "[resourceGroup().location]",
"tags": {},
"dependsOn": [],
"properties": {},
"sku": {
"name": "Y1",
"tier": "Dynamic"
}
},
{
"type": "Microsoft.Web/sites",
"apiVersion": "2021-01-15",
"name": "[variables('functionName')]",
"location": "[resourceGroup().location]",
"kind": "functionapp",
"dependsOn": [
"[resourceId('Microsoft.Web/serverfarms', variables('HostingPlanName'))]",
"[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccountName'))]"
],
"identity": {
"type": "SystemAssigned"
},
"properties": {
"enabled": true,
"httpsOnly": true,
"alwaysOn": true,
"reserved": false,
"serverFarmId": "[concat('/subscriptions/', subscription().subscriptionId,'/resourcegroups/', resourceGroup().name, '/providers/Microsoft.Web/serverfarms/', variables('HostingPlanName'))]"
},
"resources": [
{
"apiVersion": "2018-11-01",
"type": "config",
"name": "appsettings",
"dependsOn": [
"[concat('Microsoft.Web/sites/', variables('FunctionName'))]"
],
"properties": {
"FUNCTIONS_EXTENSION_VERSION": "~3",
"FUNCTIONS_WORKER_RUNTIME": "dotnet",
"APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2015-05-01').InstrumentationKey]",
"APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2015-05-01').ConnectionString]",
"AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('StorageAccountName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccountName')), '2019-06-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]",
"WorkspaceID": "[parameters('WorkspaceID')]",
"SharedKey": "[parameters('SharedKey')]",
"LogName": "[parameters('LogAnalyticsCustomLogName')]",
"resource": "[parameters('ResourceLocation')]",
"client_id": "[parameters('clientID')]",
"client_secret": "[parameters('clientSecret')]",
"grant_type": "client_credentials",
"tenant_id": "[parameters('tenantID')]",
"query_string": "{\"query\": \"GetDomainsForRDAP\"}",
"WEBSITE_RUN_FROM_PACKAGE": "https://github.com/Azure/Azure-Sentinel/Tools/RDAP/RDAPQuery/blob/master/RDAPQuery.zip?raw=true"
}
}
]
}
]
}