M365D tutorials and tools (#3186)
* M365D tutorials and tools Added webcasts and pbi to the right folders in Sentinel repo * Update Episode 1 - KQL Fundamentals.txt * Update Episode 2 - Joins.txt removed en-us from links * Update Episode 4 - Lets Hunt.txt removed en-us from links * Update MCAS - The Hunt.txt removed links with en-us * Update Performance, Json and dynamics operator, external data.txt removed en-us from links * Update MCAS - The Hunt.txt removed en-us * Update Airlift 2021 - Lets Invoke.csl removed en-us
This commit is contained in:
tali-ash
GitHub
Родитель
f0f7e9e594
Коммит
903c6fbe27
Двоичные данные
Tools/M365-PowerBi Dashboard/Microsoft Threat Protection - API Dashboard.pbit
Normal file
Двоичные данные
Tools/M365-PowerBi Dashboard/Microsoft Threat Protection - API Dashboard.pbit
Normal file
Двоичный файл не отображается.
|
|
@ -0,0 +1 @@
|
||||||
|
This folder contains some PowerBi dashboard that can be helpful to visualize data from Microsoft Threat Protection using the built-in APIs
|
||||||
|
|
@ -0,0 +1,464 @@
|
||||||
|
|
||||||
|
|
||||||
|
// ( ( ( ( (( (( (( )
|
||||||
|
// )\ )\ )\ )\ ))\ ))\))\ ())
|
||||||
|
// ((_) ((()_()((_)((_)))((_)(_)))(()))
|
||||||
|
// \ \ / / \ _ \ \| |_ _| \| |/ __|
|
||||||
|
// \ \/\/ /| - | / . || || . | (_ |
|
||||||
|
// \_/\_/ |_|_|_|_\_|\_|___|_|\_|\___|
|
||||||
|
|
||||||
|
// ( ) ) ) ) ( ( (
|
||||||
|
// )\ (\( ( (\( ( (() )\ )\
|
||||||
|
// ((_) )(| )\: )(|)\ (((_) ((_) ((_)
|
||||||
|
// | | ()\((_)_()\| | | | | (/ \ (/ \
|
||||||
|
// | |__/ -_) V / -_) | |_ _| | () | | () |
|
||||||
|
// |____|___|\_/\___|_| |_| \__/ \__/
|
||||||
|
|
||||||
|
/////////////////////
|
||||||
|
// Kusto Functions //
|
||||||
|
/////////////////////
|
||||||
|
|
||||||
|
// Sometimes you write something cool and want to reuse it a bunch of times.
|
||||||
|
// In this webcast we are going to cover some cool ways to use the let statement
|
||||||
|
// to enable query reuse.
|
||||||
|
|
||||||
|
// let - not just for variables :)
|
||||||
|
// https://docs.microsoft.com/azure/data-explorer/kusto/query/letstatement
|
||||||
|
|
||||||
|
let MyFunction = (MyParameter:string)
|
||||||
|
{
|
||||||
|
print MyParameter
|
||||||
|
};
|
||||||
|
MyFunction('foo')
|
||||||
|
|
||||||
|
// ...but that's just getting started. You can also pass in tabular data!
|
||||||
|
// To accomplish this we will use the invoke tabluar operator
|
||||||
|
// https://docs.microsoft.com/azure/data-explorer/kusto/query/invokeoperator
|
||||||
|
|
||||||
|
let MyOtherFunction = (MyParameter:(Value:int))
|
||||||
|
{
|
||||||
|
MyParameter
|
||||||
|
| extend Sine = sin(Value), Cosine = cos(Value), Tangent = tan(Value), Cotangent = cot(Value)
|
||||||
|
};
|
||||||
|
datatable(Value:int)
|
||||||
|
[
|
||||||
|
1,2,3,4,5,6,7,8,9,10
|
||||||
|
]
|
||||||
|
| invoke MyOtherFunction()
|
||||||
|
|
||||||
|
// The column name must match between the parameter and the source data !
|
||||||
|
|
||||||
|
// You can also pass in tabular data and scalar parameters if you choose...
|
||||||
|
let MultiplyByN = (SourceData:(Multiplicand:int), Multiplier:dynamic)
|
||||||
|
{
|
||||||
|
SourceData
|
||||||
|
| extend Multiplier = Multiplier
|
||||||
|
| mv-expand Multiplier to typeof(int)
|
||||||
|
| extend Product = Multiplicand * Multiplier
|
||||||
|
};
|
||||||
|
let Multiplier = range(1,10,1);
|
||||||
|
datatable(Multiplicand:int)
|
||||||
|
[1,2,3,4,5,6,7,8,9,10]
|
||||||
|
| invoke MultiplyByN(Multiplier)
|
||||||
|
|
||||||
|
// range() - generates a range of numbers from the first
|
||||||
|
// parameter to the second parameter step the third parameter
|
||||||
|
// https://docs.microsoft.com/azure/data-explorer/kusto/query/rangeoperator
|
||||||
|
|
||||||
|
// or go all out and use multiple tablular inputs
|
||||||
|
|
||||||
|
let LetsIntersect = (Dataset1:(key:int,value:string), Dataset2:(key:int,value:string))
|
||||||
|
{
|
||||||
|
Dataset1
|
||||||
|
| join kind=inner Dataset2 on key
|
||||||
|
};
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Foo",
|
||||||
|
1, "Bar",
|
||||||
|
2, "Baz",
|
||||||
|
3, "Qux",
|
||||||
|
4, "Quux"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Wibble",
|
||||||
|
1, "Wobble",
|
||||||
|
2, "Wubble",
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| invoke LetsIntersect(RightTable)
|
||||||
|
|
||||||
|
// OK! Enough of the contrived examples! How can I actually use this?
|
||||||
|
|
||||||
|
///////////////////
|
||||||
|
// Path Aliasing //
|
||||||
|
///////////////////
|
||||||
|
|
||||||
|
// Ever try to gather statistics on something in the \users\ directory? You
|
||||||
|
// end up with all kinds of one-off FolderPaths tied to a user's profile.
|
||||||
|
// ... let's fix that.
|
||||||
|
|
||||||
|
let AliasPath = (SourcePath:(FolderPath:string, FileName:string))
|
||||||
|
{
|
||||||
|
SourcePath
|
||||||
|
| extend AliasPath = tolower(
|
||||||
|
case(
|
||||||
|
//Modern style profile
|
||||||
|
FolderPath startswith 'c:\\users\\', strcat('%UserProfile%', substring(FolderPath, indexof(FolderPath,'\\',11), strlen(FolderPath) - 11)),
|
||||||
|
//Legacy style profile
|
||||||
|
FolderPath startswith 'c:\\documents and settings\\', strcat('%UserProfile%', substring(FolderPath, indexof(FolderPath,'\\',27), strlen(FolderPath) - 27)),
|
||||||
|
//Windir
|
||||||
|
FolderPath contains @':\Windows\', strcat('%windir%', substring(FolderPath, 10)),
|
||||||
|
//ProgramData
|
||||||
|
FolderPath contains @':\programdata\', strcat('%programdata%', substring(FolderPath, 14)),
|
||||||
|
// ProgramFiles
|
||||||
|
FolderPath contains @':\Program Files\', strcat('%ProgramFiles%', substring(FolderPath, 16)),
|
||||||
|
// Program Files (x86)
|
||||||
|
FolderPath contains @':\Program Files (x86)\', strcat('%ProgramFilesx86%', substring(FolderPath, 22)),
|
||||||
|
//Other
|
||||||
|
FolderPath)
|
||||||
|
)
|
||||||
|
};
|
||||||
|
DeviceProcessEvents
|
||||||
|
| take 100
|
||||||
|
| invoke AliasPath()
|
||||||
|
| project-reorder AliasPath
|
||||||
|
|
||||||
|
// So lets use this to help determine the impact of a firewall rule...
|
||||||
|
|
||||||
|
let EphemeralRangeStart = 49152;
|
||||||
|
let IncludeInboundRemoteIPs = false;
|
||||||
|
let AliasPath = (SourcePath:(FolderPath:string, FileName:string))
|
||||||
|
{
|
||||||
|
SourcePath
|
||||||
|
| extend AliasPath = tolower(
|
||||||
|
case(
|
||||||
|
//Modern style profile
|
||||||
|
FolderPath startswith 'c:\\users\\', strcat('%UserProfile%', substring(FolderPath, indexof(FolderPath,'\\',11), strlen(FolderPath) - 11)),
|
||||||
|
//Legacy style profile
|
||||||
|
FolderPath startswith 'c:\\documents and settings\\', strcat('%UserProfile%', substring(FolderPath, indexof(FolderPath,'\\',27), strlen(FolderPath) - 27)),
|
||||||
|
//Windir
|
||||||
|
FolderPath contains @':\Windows\', strcat('%windir%', substring(FolderPath, 10)),
|
||||||
|
//ProgramData
|
||||||
|
FolderPath contains @':\programdata\', strcat('%programdata%', substring(FolderPath, 14)),
|
||||||
|
// ProgramFiles
|
||||||
|
FolderPath contains @':\Program Files\', strcat('%ProgramFiles%', substring(FolderPath, 16)),
|
||||||
|
// Program Files (x86)
|
||||||
|
FolderPath contains @':\Program Files (x86)\', strcat('%ProgramFilesx86%', substring(FolderPath, 22)),
|
||||||
|
//Other
|
||||||
|
FolderPath)
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let ServerConnections =
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where ActionType in ('InboundConnectionAccepted','ListeningConnectionCreated')
|
||||||
|
and RemoteIPType != 'Loopback'
|
||||||
|
and LocalIP != RemoteIP
|
||||||
|
and RemoteIP !startswith '169.254'
|
||||||
|
and LocalPort < EphemeralRangeStart
|
||||||
|
| distinct DeviceId, InitiatingProcessFolderPath, LocalPort;
|
||||||
|
union (
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where ActionType in ('InboundConnectionAccepted','ListeningConnectionCreated','ConnectionSuccess','ConnecitonFound','ConnectionRequest')
|
||||||
|
and RemoteIPType != 'Loopback'
|
||||||
|
and LocalIP != RemoteIP
|
||||||
|
and RemoteIP !startswith '169.254'
|
||||||
|
and LocalPort < EphemeralRangeStart
|
||||||
|
| join kind=leftsemi ServerConnections on DeviceId, InitiatingProcessFolderPath, LocalPort
|
||||||
|
| project-rename FolderPath = InitiatingProcessFolderPath, FileName = InitiatingProcessFileName
|
||||||
|
| invoke AliasPath()
|
||||||
|
| extend Directionality = 'Inbound', Port = LocalPort, RemoteIP = iff(IncludeInboundRemoteIPs == true, RemoteIP,'')
|
||||||
|
),(
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where ActionType in ('ConnectionSuccess','ConnecitonFound','ConnectionRequest')
|
||||||
|
and RemoteIPType != 'Loopback'
|
||||||
|
and LocalIP != RemoteIP
|
||||||
|
and RemoteIP !startswith '169.254'
|
||||||
|
and LocalPort >= EphemeralRangeStart
|
||||||
|
| join kind=leftanti ServerConnections on DeviceId, InitiatingProcessFolderPath, LocalPort
|
||||||
|
| project-rename FolderPath = InitiatingProcessFolderPath, FileName = InitiatingProcessFileName
|
||||||
|
| invoke AliasPath()
|
||||||
|
| extend Directionality = 'Outbound', Port = RemotePort
|
||||||
|
)
|
||||||
|
| summarize ConnectionCount = count(), DistinctMachines = dcount(DeviceId), Ports = makeset(Port), RemoteIPs = makeset(RemoteIP) by Directionality, AliasPath
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
//////////////////////////////
|
||||||
|
// Mapping out IP addresses //
|
||||||
|
//////////////////////////////
|
||||||
|
|
||||||
|
// Sometimes you have a mapping of IP addresses that you want to compare to your
|
||||||
|
// query results. This might be a CSV of internal addresses and the associated
|
||||||
|
// physical location, a list of IP addresses by service, or something entirely
|
||||||
|
// different.
|
||||||
|
|
||||||
|
// In this example, we will use the externaldata operator to bring in the list
|
||||||
|
// of Azure IP addresses and compare the RemoteIP from our results to determine
|
||||||
|
// if the IP is in the Azure range.
|
||||||
|
|
||||||
|
let AzureSubnets = toscalar (
|
||||||
|
externaldata (xml:string)
|
||||||
|
[
|
||||||
|
@'https://download.microsoft.com/download/0/1/8/018E208D-54F8-44CD-AA26-CD7BC9524A8C/PublicIPs_20200824.xml'
|
||||||
|
]
|
||||||
|
with (format="txt")
|
||||||
|
| extend Subnet = tostring(parse_xml(xml).IpRange.['@Subnet'])
|
||||||
|
| where isnotempty(Subnet)
|
||||||
|
| summarize make_set(Subnet)
|
||||||
|
);
|
||||||
|
let IsItAzure = (SourceData:(RemoteIP:string)) {
|
||||||
|
SourceData
|
||||||
|
| extend AzureSubnet = AzureSubnets
|
||||||
|
| mv-expand AzureSubnet to typeof(string)
|
||||||
|
| extend IsAzure = ipv4_is_in_range(RemoteIP, AzureSubnet)
|
||||||
|
| summarize IsAzure = max(IsAzure) by RemoteIP
|
||||||
|
};
|
||||||
|
// BEGIN SAMPLE QUERY //
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| take 10000
|
||||||
|
// END SAMPLE QUERY
|
||||||
|
| invoke IsItAzure()
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
/////////////////////////////////
|
||||||
|
// Determining active users //
|
||||||
|
// based on process creations //
|
||||||
|
/////////////////////////////////
|
||||||
|
|
||||||
|
// How many times have you run into the need to find out which user(s) were active
|
||||||
|
// on to a device at a given time? Sure, we have the LoggedOnUsers column in
|
||||||
|
// the DeviceInfo table, but what if we wanted a very specific resolution based on
|
||||||
|
// the identities of processes created around an event?
|
||||||
|
|
||||||
|
// In our ficticious example, we'll hunt for accounts that were active in the same
|
||||||
|
// 5 minute block as a registry event.
|
||||||
|
|
||||||
|
// Notice that we are passing two parameters in this function - and the second one
|
||||||
|
// has a default value of 5m. This is a handy way to enable flexibility. If you
|
||||||
|
// later wanted to change the resolution you can just specify it as a parameter in
|
||||||
|
// your invoke statement.
|
||||||
|
|
||||||
|
let WhoWasActive = (SourceData:(DeviceId:string, Timestamp:datetime), Resolution:timespan = 5m)
|
||||||
|
{
|
||||||
|
SourceData
|
||||||
|
| extend TimeBin = bin(Timestamp, Resolution)
|
||||||
|
| join kind=inner (
|
||||||
|
DeviceProcessEvents
|
||||||
|
| where AccountDomain !in~ ('nt authority','font driver host', 'window manager', 'nt service')
|
||||||
|
| project Timestamp, DeviceId, Account = tolower(strcat(AccountDomain,'\\',AccountName))
|
||||||
|
| summarize ActiveAccounts = make_set(Account) by DeviceId, TimeBin = bin(Timestamp, Resolution)
|
||||||
|
) on DeviceId, TimeBin
|
||||||
|
| project-away TimeBin, TimeBin1
|
||||||
|
};
|
||||||
|
DeviceRegistryEvents
|
||||||
|
| summarize arg_max(Timestamp, *) by DeviceId
|
||||||
|
| take 100
|
||||||
|
| invoke WhoWasActive()
|
||||||
|
| project-reorder ActiveAccounts
|
||||||
|
|
||||||
|
///////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
///////////////////////////////////
|
||||||
|
// Detecting anomalous key-value //
|
||||||
|
// pair combinations in datasets //
|
||||||
|
///////////////////////////////////
|
||||||
|
|
||||||
|
// I developed this function while on the Microsoft DART team to help
|
||||||
|
// solve a specific problem - we needed to find files that masqueraded
|
||||||
|
// as legitimate files, usually based on their filename. This technique
|
||||||
|
// is commonly used by malware as a means of masquerading as a
|
||||||
|
// legitimate file. While most of the time the file names an attacker
|
||||||
|
// would use could be found in the root of %windir%\System32,
|
||||||
|
// occasionally we would find some creativity out there using other
|
||||||
|
// file names.
|
||||||
|
|
||||||
|
// The good news is you can't hide from statistics :)
|
||||||
|
|
||||||
|
// DetectMasqueradeAnomaly
|
||||||
|
// Inputs:
|
||||||
|
// - SourceData: the dataset to detect anomalies in. Must contain
|
||||||
|
a column named 'Key' and another named 'Value'
|
||||||
|
- MaxResults: The number of results to return in descending
|
||||||
|
order of how anomalous the pairing is
|
||||||
|
|
||||||
|
// What it does:
|
||||||
|
// This function will look for common keys that are consistently paired
|
||||||
|
// with the same value. The more common the key and the more commonly it
|
||||||
|
// is paired with the same value the more 'normal' this pairing is
|
||||||
|
// considered.
|
||||||
|
|
||||||
|
// Contrived example:
|
||||||
|
|
||||||
|
let DetectMasqueradeAnomaly = (SourceData:(Key:string, Value:string), MaxResults:int = 10000) {
|
||||||
|
let PairCount = materialize(
|
||||||
|
SourceData
|
||||||
|
| summarize hint.strategy=shuffle Instances = count() by Key, Value
|
||||||
|
);
|
||||||
|
PairCount
|
||||||
|
| summarize hint.strategy=shuffle SampleSize = count(), Average = avg(Instances), DistinctValueCount = dcount(Value) by Key
|
||||||
|
| where DistinctValueCount > 1 and SampleSize > 1 and DistinctValueCount < Average // Remove entries that are always have the same key, have only one instance, or are above the average instances (that would be a normal value for the key by this definition)
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Join back on raw Key to Value statistics so that we can now find anomalies
|
||||||
|
| extend MScore = (SampleSize * (Average - Instances) / (Instances * DistinctValueCount)) // Run the masquerade detection calculation
|
||||||
|
| where MScore > 0 // Remove values that are more normal than the average
|
||||||
|
| top MaxResults by MScore desc // Strip entries beyond the defined result count threshold
|
||||||
|
| join kind=inner hint.strategy=shuffle ( // Join back on PairCount to deteremine most common value for the pair
|
||||||
|
PairCount
|
||||||
|
| summarize MostCommonValueInstances = max(Instances) by Key // In this case, we need to know which key had the max number of instances
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Bring it back together with the Key to Value data
|
||||||
|
| where Instances == MostCommonValueInstances // Select only the rows where Instances == MostCommonValueInstances to figure out which path was most common
|
||||||
|
| project Key, MostCommonValueInstances = Instances, MostCommonValue = Value // Clean up output
|
||||||
|
) on Key
|
||||||
|
| project-away Key1, Key2
|
||||||
|
| top MaxResults by MScore desc
|
||||||
|
};
|
||||||
|
datatable (Key:string, Value:string)
|
||||||
|
[
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '1',
|
||||||
|
'a', '3', // Masquerade anomaly
|
||||||
|
'b', '2',
|
||||||
|
'b', '2',
|
||||||
|
'b', '2',
|
||||||
|
'b', '2',
|
||||||
|
'b', '2',
|
||||||
|
'b', '2',
|
||||||
|
'b', '2',
|
||||||
|
'b', '4', // Masquerade anomaly
|
||||||
|
'c', '1', // c has an even distribution of values
|
||||||
|
'c', '2',
|
||||||
|
'c', '3',
|
||||||
|
'c', '4',
|
||||||
|
'd', '4', // d is always the same
|
||||||
|
'd', '4',
|
||||||
|
'd', '4',
|
||||||
|
'd', '4'
|
||||||
|
]
|
||||||
|
| invoke DetectMasqueradeAnomaly()
|
||||||
|
|
||||||
|
// Notice:
|
||||||
|
// - All common pairings are filtered out
|
||||||
|
// - 'a' was ranked higher from an anomaly perspective
|
||||||
|
// than 'b' because there are more examples of 'a' being paired with '1'
|
||||||
|
// than 'b' paired with '2'
|
||||||
|
// - 'c' did not show up because it was 100% random in this dataset
|
||||||
|
// - 'd' did not show up because it was 100% consistent in this dataset
|
||||||
|
|
||||||
|
// Using this in the real world
|
||||||
|
|
||||||
|
// Finding programs with common names launched from strange directories
|
||||||
|
|
||||||
|
let DetectMasqueradeAnomaly = (SourceData:(Key:string, Value:string), MaxResults:int = 10000) {
|
||||||
|
let PairCount = materialize(
|
||||||
|
SourceData
|
||||||
|
| summarize hint.strategy=shuffle Instances = count() by Key, Value
|
||||||
|
);
|
||||||
|
PairCount
|
||||||
|
| summarize hint.strategy=shuffle SampleSize = count(), Average = avg(Instances), DistinctValueCount = dcount(Value) by Key
|
||||||
|
| where DistinctValueCount > 1 and SampleSize > 1 and DistinctValueCount < Average // Remove entries that are always have the same key, have only one instance, or are above the average instances (that would be a normal value for the key by this definition)
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Join back on raw Key to Value statistics so that we can now find anomalies
|
||||||
|
| extend MScore = (SampleSize * (Average - Instances) / (Instances * DistinctValueCount)) // Run the masquerade detection calculation
|
||||||
|
| where MScore > 0 // Remove values that are more normal than the average
|
||||||
|
| top MaxResults by MScore desc // Strip entries beyond the defined result count threshold
|
||||||
|
| join kind=inner hint.strategy=shuffle ( // Join back on PairCount to deteremine most common value for the pair
|
||||||
|
PairCount
|
||||||
|
| summarize MostCommonValueInstances = max(Instances) by Key // In this case, we need to know which key had the max number of instances
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Bring it back together with the Key to Value data
|
||||||
|
| where Instances == MostCommonValueInstances // Select only the rows where Instances == MostCommonValueInstances to figure out which path was most common
|
||||||
|
| project Key, MostCommonValueInstances = Instances, MostCommonValue = Value // Clean up output
|
||||||
|
) on Key
|
||||||
|
| project-away Key1, Key2
|
||||||
|
| top MaxResults by MScore desc
|
||||||
|
};
|
||||||
|
DeviceProcessEvents
|
||||||
|
| where Timestamp > ago(1d)
|
||||||
|
| project Key = FileName, Value = FolderPath
|
||||||
|
| invoke DetectMasqueradeAnomaly(500)
|
||||||
|
| join DeviceProcessEvents on $left.Key == $right.FileName, $left.Value == $right.FolderPath
|
||||||
|
| project-away Key, Value, SampleSize, Average, Instances, DistinctValueCount, MostCommonValueInstances, MostCommonValue
|
||||||
|
| order by MScore desc
|
||||||
|
|
||||||
|
// Detecting user accounts logging on to anomalous systems
|
||||||
|
|
||||||
|
let DetectMasqueradeAnomaly = (SourceData:(Key:string, Value:string), MaxResults:int = 10000) {
|
||||||
|
let PairCount = materialize(
|
||||||
|
SourceData
|
||||||
|
| summarize hint.strategy=shuffle Instances = count() by Key, Value
|
||||||
|
);
|
||||||
|
PairCount
|
||||||
|
| summarize hint.strategy=shuffle SampleSize = count(), Average = avg(Instances), DistinctValueCount = dcount(Value) by Key
|
||||||
|
| where DistinctValueCount > 1 and SampleSize > 1 and DistinctValueCount < Average // Remove entries that are always have the same key, have only one instance, or are above the average instances (that would be a normal value for the key by this definition)
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Join back on raw Key to Value statistics so that we can now find anomalies
|
||||||
|
| extend MScore = (SampleSize * (Average - Instances) / (Instances * DistinctValueCount)) // Run the masquerade detection calculation
|
||||||
|
| where MScore > 0 // Remove values that are more normal than the average
|
||||||
|
| top MaxResults by MScore desc // Strip entries beyond the defined result count threshold
|
||||||
|
| join kind=inner hint.strategy=shuffle ( // Join back on PairCount to deteremine most common value for the pair
|
||||||
|
PairCount
|
||||||
|
| summarize MostCommonValueInstances = max(Instances) by Key // In this case, we need to know which key had the max number of instances
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Bring it back together with the Key to Value data
|
||||||
|
| where Instances == MostCommonValueInstances // Select only the rows where Instances == MostCommonValueInstances to figure out which path was most common
|
||||||
|
| project Key, MostCommonValueInstances = Instances, MostCommonValue = Value // Clean up output
|
||||||
|
) on Key
|
||||||
|
| project-away Key1, Key2
|
||||||
|
| top MaxResults by MScore desc
|
||||||
|
};
|
||||||
|
DeviceLogonEvents
|
||||||
|
| project Key = strcat(AccountDomain, @'\', AccountName), Value = DeviceId
|
||||||
|
| invoke DetectMasqueradeAnomaly(500)
|
||||||
|
| join (
|
||||||
|
DeviceLogonEvents
|
||||||
|
| extend UserAndDomain = strcat(AccountDomain, @'\', AccountName)
|
||||||
|
) on $left.Key == $right.UserAndDomain, $left.Value == $right.DeviceId
|
||||||
|
| project-away Key, Value, SampleSize, Average, Instances, DistinctValueCount, MostCommonValueInstances, MostCommonValue
|
||||||
|
| project-reorder MScore, Timestamp, DeviceId, UserAndDomain
|
||||||
|
| order by MScore desc
|
||||||
|
|
||||||
|
// Detecting users logging on during strange hours of the day
|
||||||
|
|
||||||
|
print gettype(hourofday(now()))
|
||||||
|
|
||||||
|
// Note that hourofday() is a long, so we will need to change the data type for Value
|
||||||
|
|
||||||
|
let DetectMasqueradeAnomaly = (SourceData:(Key:string, Value:long), MaxResults:int = 10000) {
|
||||||
|
let PairCount = materialize(
|
||||||
|
SourceData
|
||||||
|
| summarize hint.strategy=shuffle Instances = count() by Key, Value
|
||||||
|
);
|
||||||
|
PairCount
|
||||||
|
| summarize hint.strategy=shuffle SampleSize = count(), Average = avg(Instances), DistinctValueCount = dcount(Value) by Key
|
||||||
|
| where DistinctValueCount > 1 and SampleSize > 1 and DistinctValueCount < Average // Remove entries that are always have the same key, have only one instance, or are above the average instances (that would be a normal value for the key by this definition)
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Join back on raw Key to Value statistics so that we can now find anomalies
|
||||||
|
| extend MScore = (SampleSize * (Average - Instances) / (Instances * DistinctValueCount)) // Run the masquerade detection calculation
|
||||||
|
| where MScore > 0 // Remove values that are more normal than the average
|
||||||
|
| top MaxResults by MScore desc // Strip entries beyond the defined result count threshold
|
||||||
|
| join kind=inner hint.strategy=shuffle ( // Join back on PairCount to deteremine most common value for the pair
|
||||||
|
PairCount
|
||||||
|
| summarize MostCommonValueInstances = max(Instances) by Key // In this case, we need to know which key had the max number of instances
|
||||||
|
| join kind=inner hint.strategy=shuffle PairCount on Key // Bring it back together with the Key to Value data
|
||||||
|
| where Instances == MostCommonValueInstances // Select only the rows where Instances == MostCommonValueInstances to figure out which path was most common
|
||||||
|
| project Key, MostCommonValueInstances = Instances, MostCommonValue = Value // Clean up output
|
||||||
|
) on Key
|
||||||
|
| project-away Key1, Key2
|
||||||
|
| top MaxResults by MScore desc
|
||||||
|
};
|
||||||
|
DeviceLogonEvents
|
||||||
|
| project Key = strcat(AccountDomain, @'\', AccountName), Value = hourofday(Timestamp)
|
||||||
|
| invoke DetectMasqueradeAnomaly(500)
|
||||||
|
| join (
|
||||||
|
DeviceLogonEvents
|
||||||
|
| extend UserAndDomain = strcat(AccountDomain, @'\', AccountName), HourOfDay = hourofday(Timestamp)
|
||||||
|
) on $left.Key == $right.UserAndDomain, $left.Value == $right.HourOfDay
|
||||||
|
| project-away Key, Value, SampleSize, Average, Instances, DistinctValueCount, MostCommonValueInstances, MostCommonValue
|
||||||
|
| project-reorder MScore, Timestamp, DeviceName, UserAndDomain
|
||||||
|
| order by MScore desc
|
||||||
|
|
@ -0,0 +1,202 @@
|
||||||
|
print Session = 'Best practices for hunting across domains with Microsoft 365 Defender', Presenter = 'Michael Melone, Tali Ash', Company = 'Microsoft'
|
||||||
|
|
||||||
|
// Schema Reference (upper right corner)
|
||||||
|
|
||||||
|
// Explore identities data
|
||||||
|
// From IdentityDirectoryEvents schema reference click one of the Action type
|
||||||
|
IdentityDirectoryEvents | where ActionType == 'SMB session'
|
||||||
|
|
||||||
|
// using extend to extract information form json column of AdditionalFields
|
||||||
|
// From IdentityDirectoryEvents schema reference click sample query of “Group
|
||||||
|
// modifications”
|
||||||
|
let group = 'Domain Admins';
|
||||||
|
IdentityDirectoryEvents
|
||||||
|
| where ActionType == 'Group Membership changed'
|
||||||
|
| extend AddedToGroup = AdditionalFields['TO.GROUP']
|
||||||
|
| extend RemovedFromGroup = AdditionalFields['FROM.GROUP']
|
||||||
|
| extend TargetAccount = AdditionalFields['TARGET_OBJECT.USER']
|
||||||
|
| where AddedToGroup == group or RemovedFromGroup == group
|
||||||
|
| project-reorder Timestamp, ActionType, AddedToGroup, RemovedFromGroup, TargetAccount
|
||||||
|
| limit 100
|
||||||
|
|
||||||
|
// Explore emails data
|
||||||
|
//Find who sent emails identified with malware/phishing
|
||||||
|
EmailEvents
|
||||||
|
| where (PhishFilterVerdict == "Phish" or MalwareFilterVerdict == "Malware") and DeliveryAction == "Delivered"
|
||||||
|
//| where SenderFromDomain != "gmail.com"
|
||||||
|
| project DeliveryAction, MalwareFilterVerdict, PhishFilterVerdict, Timestamp, SenderFromAddress, RecipientEmailAddress, Subject, AttachmentCount
|
||||||
|
|
||||||
|
// Finds the first appearance of files sent by a malicious sender in your organization
|
||||||
|
let MaliciousSenders = pack_array("mtpdemos@juno.com");
|
||||||
|
EmailAttachmentInfo
|
||||||
|
| where SenderFromAddress in~(MaliciousSenders)
|
||||||
|
| join kind=leftouter(
|
||||||
|
DeviceFileEvents
|
||||||
|
) on SHA256, $left.RecipientObjectId == $right.InitiatingProcessAccountObjectId
|
||||||
|
| summarize FirstAppearance = min(Timestamp1) by SenderFromAddress, RecipientEmailAddress, DeviceName, DeviceId, SHA256, FileName
|
||||||
|
|
||||||
|
// Functions are a special sort of join which let you pull more static data about a file (more are
|
||||||
|
// planned in the future, stay tuned!). This is really helpful when you want to get information about
|
||||||
|
// file prevalence or antimalware detections.
|
||||||
|
// Get more details on the malicous files using FileProfile function enrichment
|
||||||
|
let MaliciousSender = dynamic(["mtpdemos@juno.com"]);
|
||||||
|
EmailAttachmentInfo
|
||||||
|
| where SenderFromAddress in~ (MaliciousSender)
|
||||||
|
| join (
|
||||||
|
DeviceFileEvents
|
||||||
|
) on SHA256
|
||||||
|
| distinct SHA1
|
||||||
|
| invoke FileProfile()
|
||||||
|
| project SHA1, SHA256 , FileSize , GlobalFirstSeen , GlobalLastSeen , GlobalPrevalence , IsExecutable
|
||||||
|
|
||||||
|
//Get alerted every time an email from malicious sender was received
|
||||||
|
let MaliciousSender = "mtpdemos@juno.com";
|
||||||
|
EmailEvents
|
||||||
|
| where SenderFromAddress =~ MaliciousSender and DeliveryAction == "Delivered"
|
||||||
|
|
||||||
|
|
||||||
|
// Detection name - Email from malicious sender
|
||||||
|
// Alert title - Email from malicious sender - mtpdemos@juno.com
|
||||||
|
// Description - Email from malicious sender mtpdemos@juno.com was delivered to // users in the org
|
||||||
|
|
||||||
|
/////////////////////////////
|
||||||
|
// Get to know useful operators
|
||||||
|
/////////////////////////////
|
||||||
|
|
||||||
|
// Dealing with Phishing using Advanced Hunting
|
||||||
|
|
||||||
|
// parse_url()
|
||||||
|
// Breaks down a URL into its individual parts – including each
|
||||||
|
// query parameter
|
||||||
|
|
||||||
|
print url = parse_url("https://www.bing.com/search?q=tracking+the+adversary+mtp+advanced+hunting&qs=AS&pq=tracking+the+adversary+&sc=1-23&cvid=81318E9030D74B31A876FDE99603EE60&FORM=QBRE&sp=1")
|
||||||
|
| evaluate bag_unpack(url)
|
||||||
|
|
||||||
|
// Let’s use parse_url() to analyze some phishing activity
|
||||||
|
|
||||||
|
let Phishurls = toscalar(
|
||||||
|
EmailEvents
|
||||||
|
| where PhishFilterVerdict == "Phish"
|
||||||
|
| join EmailUrlInfo on NetworkMessageId
|
||||||
|
| extend host = parse_url(Url).Host
|
||||||
|
| where isnotempty(host)
|
||||||
|
| summarize makeset(host)
|
||||||
|
);
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where isnotempty(RemoteUrl)
|
||||||
|
| extend NetworkEventHost = parse_url(RemoteUrl).Host
|
||||||
|
| where isnotempty(NetworkEventHost)
|
||||||
|
| extend PhishHost = Phishurls
|
||||||
|
| mvexpand PhishHost to typeof(string)
|
||||||
|
| where NetworkEventHost == PhishHost
|
||||||
|
|
||||||
|
|
||||||
|
// In practice, you're likely to encounter a bunch of false positives
|
||||||
|
// due to common domains being mixed with phish domains. To accommodateTo accomodate that
|
||||||
|
// we can just reduce the dataset based on a threshold
|
||||||
|
|
||||||
|
|
||||||
|
let MaxConnections = 10; // This will be our cutoff threshold
|
||||||
|
let Phishurls = toscalar(
|
||||||
|
EmailEvents
|
||||||
|
| where PhishFilterVerdict == "Phish"
|
||||||
|
| join EmailUrlInfo on NetworkMessageId
|
||||||
|
| extend host = parse_url(Url).Host
|
||||||
|
| where isnotempty(host)
|
||||||
|
EmailEvents
|
||||||
|
| where PhishFilterVerdict == "Phish"
|
||||||
|
| join EmailUrlInfo on NetworkMessageId
|
||||||
|
| extend host = parse_url(Url).Host
|
||||||
|
| where isnotempty(host)
|
||||||
|
| summarize makeset(host)
|
||||||
|
);
|
||||||
|
// We will use this portion of the query twice now - better to make it a variable
|
||||||
|
let DeviceConnections = (
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where isnotempty(RemoteUrl)
|
||||||
|
| extend NetworkEventHost = tostring(parse_url(RemoteUrl).Host)
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where isnotempty(RemoteUrl)
|
||||||
|
| extend NetworkEventHost = tostring(parse_url(RemoteUrl).Host)
|
||||||
|
| where isnotempty(NetworkEventHost)
|
||||||
|
);
|
||||||
|
DeviceConnections
|
||||||
|
| summarize count() by NetworkEventHost // Count the number of connections by FQDN
|
||||||
|
| where count_ < MaxConnections // Filter to only domains with less than MaxConnections connections
|
||||||
|
| join kind=rightsemi DeviceConnections on NetworkEventHost // Filter our dataset to only those FQDNs
|
||||||
|
| extend PhishHost = Phishurls
|
||||||
|
| mvexpand PhishHost to typeof(string)
|
||||||
|
| where NetworkEventHost == PhishHost
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// Using the bin() function you can group events by a period of time.
|
||||||
|
// Let's take a look at some logon statistics on a daily basis
|
||||||
|
|
||||||
|
// Let's look at account logon activity over time on a
|
||||||
|
// daily basis by UPN.
|
||||||
|
|
||||||
|
IdentityLogonEvents
|
||||||
|
| where isnotempty(AccountUpn)
|
||||||
|
| summarize NumberOfLogons = count() by AccountUpn, bin(Timestamp, 1d)
|
||||||
|
| render timechart
|
||||||
|
|
||||||
|
// render - creates a chart
|
||||||
|
|
||||||
|
// We can also use this bin'ed data to determine min, max, and average daily logons
|
||||||
|
|
||||||
|
IdentityLogonEvents
|
||||||
|
| where isnotempty(AccountUpn)
|
||||||
|
| summarize NumberOfLogons = count() // first get a calculation of how many logons the user does per day
|
||||||
|
by AccountUpn
|
||||||
|
, bin(Timestamp, 1d)
|
||||||
|
| summarize TotalLogons = sum(NumberOfLogons) // Then average all of them together to get average daily logons
|
||||||
|
, AverageDailyLogons = avg(NumberOfLogons)
|
||||||
|
, FewestLogonsInADay = min(NumberOfLogons)
|
||||||
|
, MostLogonsInADay = max(NumberOfLogons)
|
||||||
|
by AccountUpn
|
||||||
|
| top 10 by TotalLogons desc
|
||||||
|
| render columnchart
|
||||||
|
|
||||||
|
// New Table - IdentityDirectoryEvents
|
||||||
|
// Contains Active Directory \ domain controller operational information
|
||||||
|
|
||||||
|
IdentityDirectoryEvents
|
||||||
|
| distinct ActionType
|
||||||
|
|
||||||
|
IdentityDirectoryEvents
|
||||||
|
| where ActionType == 'Directory Services replication'
|
||||||
|
| summarize count() by IPAddress, tolower(DeviceName), AccountUpn
|
||||||
|
|
||||||
|
// Interesting - looks like a couple of replication attempts from workstations...
|
||||||
|
|
||||||
|
IdentityDirectoryEvents
|
||||||
|
| where ActionType == 'Directory Services replication' and DeviceName !startswith 'mtp-air-aad'
|
||||||
|
| join kind=inner DeviceNetworkEvents on $left.IPAddress == $right.LocalIP and $left.Port == $right.LocalPort and $left.DestinationIPAddress == $right.RemoteIP and $left .DestinationPort == $right.RemotePort
|
||||||
|
| project-reorder Timestamp1, DeviceName1, InitiatingProcessId, InitiatingProcessCommandLine
|
||||||
|
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
|
||||||
|
// The FileProfile function lets you pull more static data about a file (more are planned in the future, stay tuned!).
|
||||||
|
// This is really helpful when you want to get information about file prevalence or antimalware detections.
|
||||||
|
|
||||||
|
// Let's say we wanted information about rare files involved in a process creation event
|
||||||
|
|
||||||
|
DeviceProcessEvents
|
||||||
|
| invoke FileProfile() // Call the FileProfile function
|
||||||
|
| where isnotempty(GlobalPrevalence) and GlobalPrevalence < 1000 // Note that in the real world you might want to include empty GlobalPrevalence
|
||||||
|
| project-reorder DeviceName, FileName, ProcessCommandLine, FileSize, GlobalPrevalence, GlobalFirstSeen, GlobalLastSeen, ThreatName, Publisher, SoftwareName
|
||||||
|
| top 100 by GlobalPrevalence asc
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
|
||||||
|
// AssignedIPAddresses() function
|
||||||
|
// Lists last known IP addresses that were assigned to a given device around the date specified
|
||||||
|
|
||||||
|
AssignedIPAddresses('6c27842721799deb6420b094044d26e15e87a37b', now())
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
|
||||||
|
// Go hunt from incidents.
|
||||||
|
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Webcasts
|
||||||
|
|
||||||
|
This repository will contain query files used in our public training \ webcasts for reuse within your instance of Microsoft 365 Defender
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tracking the Adversary
|
||||||
|
[Signup Link](https://techcommunity.microsoft.com/t5/microsoft-threat-protection/webinar-series-unleash-the-hunter-in-you/ba-p/1509232?ranMID=24542&ranEAID=msYS1Nvjv4c&ranSiteID=msYS1Nvjv4c-_joxReUxkmQPGUkIGSbqzg&epi=msYS1Nvjv4c-_joxReUxkmQPGUkIGSbqzg&irgwc=1&OCID=AID2000142_aff_7593_1243925&tduid=(ir__inwuq92cqkkft0kikk0sohziz32xi1kvkgq9mksc00)(7593)(1243925)(msYS1Nvjv4c-_joxReUxkmQPGUkIGSbqzg)()&irclickid=_inwuq92cqkkft0kikk0sohziz32xi1kvkgq9mksc00)
|
||||||
|
|
||||||
|
This four-part series provides an introduction to advanced hunting in Microsoft Threat Protection including
|
||||||
|
- An introduction to Kusto Query Language (KQL)
|
||||||
|
- Descriptions of each table available (as of the date of the webcast)
|
||||||
|
- Examples to help maximize your hunting skills in Advanced Hunting
|
||||||
|
- An example incident triage almost exclusively using Advanced Hunting
|
||||||
|
|
@ -0,0 +1,353 @@
|
||||||
|
print Series = 'Tracking the Adversary with MTP Advanced Hunting', EpisodeNumber = 1, Topic = 'KQL Fundamentals', Presenter = 'Michael Melone, Tali Ash', Company = 'Microsoft'
|
||||||
|
|
||||||
|
// Language Reference: https://docs.microsoft.com/azure/kusto/query/
|
||||||
|
// Advanced Hunting Reference: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-schema-tables?view=o365-worldwide
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// What is KQL \ Azure Data Explorer?
|
||||||
|
// - Write Once, Read Many (WORM) dataset
|
||||||
|
// - Used in a variety of Microsoft products including
|
||||||
|
// + Defender ATP Advanced Hunting
|
||||||
|
// + Microsoft Threat Protection Advanced Hunting
|
||||||
|
// + Azure Sentinel
|
||||||
|
// + Azure Data Explorer
|
||||||
|
// - Tuned to work best with log data
|
||||||
|
// - Case sensitive
|
||||||
|
// - Automatically expires records based on a specified interval (up to 10 years)
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// When using Kusto datasources
|
||||||
|
// - If the data source is log-based, try to reduce the timeframe
|
||||||
|
// - More current data is likely to be in hot storage and will return more quickly
|
||||||
|
// - Try to reduce data earlier in the query before joining or manipulating it
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// Getting Started: Query Format
|
||||||
|
//
|
||||||
|
// DataSource
|
||||||
|
// | filters \ modifiers \ limiters
|
||||||
|
|
||||||
|
DeviceProcessEvents
|
||||||
|
| take 100
|
||||||
|
|
||||||
|
// DeviceProcessEvents
|
||||||
|
// ref: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-deviceprocessevents-table?view=o365-worldwide
|
||||||
|
// Process creation and related events
|
||||||
|
// - The newly-launched process
|
||||||
|
// - The process which initiated the process
|
||||||
|
// + The device the process
|
||||||
|
// + The identity the process was launched as
|
||||||
|
|
||||||
|
// take
|
||||||
|
// Returns rows up to a pre-set count. Good for testing out your query at a small scale before use.
|
||||||
|
|
||||||
|
// Note: There is no order or consistency when using take without sorting!
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT TOP 15 * FROM DeviceProcessEvents
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// Data sources can be tables, functions, variables
|
||||||
|
|
||||||
|
let foo = "bar";
|
||||||
|
print foo
|
||||||
|
|
||||||
|
// let
|
||||||
|
// Declares a variable which can be used later in the query.
|
||||||
|
// - Values can be:
|
||||||
|
// + Scalar (single value)
|
||||||
|
// + Tabular (a 2-dimensional table)
|
||||||
|
// + A function
|
||||||
|
// + Dynamic (a JSON-formatted object that can be addressed using dotted notation (this.that))
|
||||||
|
// - A semicolon must exist after every let statement!
|
||||||
|
|
||||||
|
// print
|
||||||
|
// Outputs a scalar value
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
DeviceLogonEvents
|
||||||
|
| count
|
||||||
|
|
||||||
|
// DeviceLogonEvents
|
||||||
|
// A table containing a row for each logon a device enrolled in Defender ATP
|
||||||
|
// Contains
|
||||||
|
// - Account information associated with the logon
|
||||||
|
// - The device which the account logged onto
|
||||||
|
// - The process which performed the logon
|
||||||
|
// - Network information (for network logons)
|
||||||
|
// - Timestamp
|
||||||
|
|
||||||
|
// count
|
||||||
|
// Returns the row count for a tablular dataset
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
AppFileEvents
|
||||||
|
| take 100
|
||||||
|
| sort by Timestamp desc
|
||||||
|
|
||||||
|
|
||||||
|
// AppFileEvents
|
||||||
|
// ref: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-appfileevents-table?view=o365-worldwide
|
||||||
|
// Information regarding activity relating to files stored in cloud services
|
||||||
|
// monitored by Microsoft Cloud App Security (MCAS), including
|
||||||
|
// - The cloud application name
|
||||||
|
// - The type of action performed
|
||||||
|
// - The item the action was performed on
|
||||||
|
// - The identity which performed the action
|
||||||
|
// - The IP address and geolocation
|
||||||
|
|
||||||
|
// sort
|
||||||
|
// Orders the dataset based on the specified column
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT TOP 100 * FROM DeviceFileEvents ORDER BY Timestamp desc
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
DeviceRegistryEvents
|
||||||
|
| top 100 by Timestamp desc
|
||||||
|
|
||||||
|
// DeviceRegistryEvents
|
||||||
|
// Registry changes which occurred on a Windows device monitored by Defender ATP
|
||||||
|
// Contains
|
||||||
|
// - Registry information (Key, Value, Data)
|
||||||
|
// - Device information
|
||||||
|
// - The process which performed the operation
|
||||||
|
// - Timestamp
|
||||||
|
|
||||||
|
// top
|
||||||
|
// Returns an ordered list of rows based on the column specified
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT TOP 100 * FROM DeviceRegistryEvents ORDER BY Timestamp desc
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| take 1000
|
||||||
|
| distinct RemoteIP, RemoteUrl
|
||||||
|
|
||||||
|
// DeviceNetworkEvents
|
||||||
|
// Table containing inbound and outbound network connections and attempts from a device monitored by Defender ATP
|
||||||
|
// Contains
|
||||||
|
// - Networking information (source and destination IP and port, URL, protocol)
|
||||||
|
// - Device information
|
||||||
|
// - The process which made or received the connection
|
||||||
|
// - Timestamp
|
||||||
|
|
||||||
|
// distinct
|
||||||
|
// Returns a table of unique results based on the column(s) specified
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT DISTINCT RemoteIP, RemoteUrl FROM DeviceNetworkEvents
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
DeviceInfo
|
||||||
|
| take 100
|
||||||
|
| project DeviceId, DeviceName, OSPlatform
|
||||||
|
|
||||||
|
// DeviceInfo
|
||||||
|
// Operating information about a device monitored by Defender ATP
|
||||||
|
// Contains
|
||||||
|
// - Device name, ID
|
||||||
|
// - Operating system information
|
||||||
|
// - Public IP address
|
||||||
|
// - Logged on user
|
||||||
|
// - Machine group
|
||||||
|
|
||||||
|
// project
|
||||||
|
// Can be used to
|
||||||
|
// - Reduce columns returned from a dataset
|
||||||
|
// - Rename columns in a dataset
|
||||||
|
// - Create calculated columns
|
||||||
|
|
||||||
|
// DataSource
|
||||||
|
// | project Column1, Column2, Column3 = Column1 + Column2
|
||||||
|
|
||||||
|
// SQL Equivalent: The column list in a query statement
|
||||||
|
// SELECT [this is the project statement] FROM DataSource
|
||||||
|
|
||||||
|
DeviceInfo
|
||||||
|
| project Timestamp, DeviceName, Four = 2 + 2
|
||||||
|
| take 100
|
||||||
|
|
||||||
|
// --------------
|
||||||
|
|
||||||
|
DeviceNetworkInfo
|
||||||
|
| take 100
|
||||||
|
| project-away Timestamp
|
||||||
|
|
||||||
|
// DeviceNetworkInfo
|
||||||
|
// Local network configurations for a device monitored by Defender ATP
|
||||||
|
|
||||||
|
// Other useful project commands:
|
||||||
|
|
||||||
|
// project-away
|
||||||
|
// Removes columns from the dataset
|
||||||
|
|
||||||
|
// project-rename
|
||||||
|
// Renames a column
|
||||||
|
|
||||||
|
// project-reorder
|
||||||
|
// Changes the order of columns in the results making the specified columns first
|
||||||
|
// No real change to the data, just how its represented
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
DeviceImageLoadEvents
|
||||||
|
| take 100
|
||||||
|
| extend DomainAndUser = strcat(InitiatingProcessAccountDomain, '\\', InitiatingProcessAccountName)
|
||||||
|
| project-reorder DomainAndUser, InitiatingProcessAccountDomain, InitiatingProcessAccountName
|
||||||
|
|
||||||
|
// DeviceImageLoadEvents
|
||||||
|
// Identifies any DLLs loaded by a process. Useful for tracking DLL sideloading attacks.
|
||||||
|
// Contains
|
||||||
|
// - The process that loaded the library
|
||||||
|
// - The module loaded by the process
|
||||||
|
// - The device where the load occurred
|
||||||
|
// - Timestamp
|
||||||
|
|
||||||
|
// extend
|
||||||
|
// Adds a column to the current dataset
|
||||||
|
|
||||||
|
// strcat()
|
||||||
|
// Concatenates two or more strings
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
AppFileEvents
|
||||||
|
| where Timestamp > ago(3d)
|
||||||
|
|
||||||
|
// where
|
||||||
|
// Used to filter a tables results based on a Boolean expression
|
||||||
|
|
||||||
|
// DataSource
|
||||||
|
// | where Column == "value"
|
||||||
|
|
||||||
|
// SQL Equivalent
|
||||||
|
// SELECT * FROM SecurityEvent WHERE EventID = 4624
|
||||||
|
|
||||||
|
// ago()
|
||||||
|
// Function used to identify a timespan relative to the current date and time
|
||||||
|
// Used with one of the following quantifiers:
|
||||||
|
// d: days
|
||||||
|
// h: hours
|
||||||
|
// m: minutes
|
||||||
|
// s: seconds
|
||||||
|
// ms: milliseconds
|
||||||
|
// microsecond: microseconds
|
||||||
|
// tick: ticks (100 nanosecond intervals)
|
||||||
|
|
||||||
|
// Important note: The most effective way to improve query performance in KQL
|
||||||
|
// is filtering based on time.
|
||||||
|
|
||||||
|
// ----------------------------------
|
||||||
|
|
||||||
|
// Note that Kusto is a case sensitive language and
|
||||||
|
// many of the operators are case sensitive.
|
||||||
|
|
||||||
|
print IsItEqual = 'TEST' == 'test'
|
||||||
|
|
||||||
|
// For a case insensitive string search, use =~
|
||||||
|
|
||||||
|
print IsItEqual = 'TEST' =~ 'test'
|
||||||
|
|
||||||
|
// Common Operators and their case insensitive counterparts
|
||||||
|
// __________________________________________________________________
|
||||||
|
// | Case Sensitive | Case Insensitive | Operation |
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
// | == | =~ | Equality |
|
||||||
|
// | != | !~ | Inequality |
|
||||||
|
// | has_cs | has | Term comparison (whole word) |
|
||||||
|
// | !has_cs | !has | Term comparison (whole word) |
|
||||||
|
// | hasprefix_cs | hasprefix | Term prefix comparison (any) |
|
||||||
|
// | !hasprefix_cs | !hasprefix | Term prefix comparison (any) |
|
||||||
|
// | hassuffix_cs | hassuffix | Term suffix comparison (any) |
|
||||||
|
// | !hassuffix_cs | !hassuffix | Term suffix comaprison (any) |
|
||||||
|
// | contains_cs | contains | Substring |
|
||||||
|
// | !contains_cs | !contains | Substring |
|
||||||
|
// | startswith_cs | startswith | String prefix |
|
||||||
|
// | !startswith_cs | !startswith | String prefix |
|
||||||
|
// | endswith_cs | endswith | String suffix |
|
||||||
|
// | !endswith_cs | !endswith | String suffix |
|
||||||
|
// | in | in~ | Array element match |
|
||||||
|
// | !in | !in~ | Array element match |
|
||||||
|
// | | has_any | Term array match |
|
||||||
|
// | matches regex | | Regular expression match |
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
|
||||||
|
print IsItEqual = "quick" in ("The", "Quick", "Brown", "Fox")
|
||||||
|
|
||||||
|
print IsItEqual = pack_array("lorem","ipsum","dolor") has "Dolor"
|
||||||
|
|
||||||
|
print IsItEqual = "Microsoft" contains_cs "ICR"
|
||||||
|
|
||||||
|
// For a list of all string operators: https://docs.microsoft.com/azure/kusto/query/datatypes-string-operators
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// Special characters \ escaping
|
||||||
|
|
||||||
|
// In KQL, the '\' character is the escape character. If you want to use a '\'
|
||||||
|
// in your query you will need to either escape it by using '\\', or you can
|
||||||
|
// make it a string literal by prepending '@' before the string
|
||||||
|
|
||||||
|
print '\\ This \\ example \\ uses \\ the \\ escape \\ method \\'
|
||||||
|
|
||||||
|
// Now using the string literal method
|
||||||
|
print @'\ This \ example \ uses \ the \ string \ literal \ method \'
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// Checking for null or blank values
|
||||||
|
|
||||||
|
// isnull(Column) / isnotnull(Column)
|
||||||
|
// - Checks for null values
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT TimeGenerated, EventData FROM SecurityEvent WHERE EventData IS NOT NULL
|
||||||
|
|
||||||
|
print isnull("")
|
||||||
|
|
||||||
|
// isempty(Column) / isnotempty(Column)
|
||||||
|
// - Checks for null values or empty strings
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT TimeGenerated, EventData FROM SecurityEvent WHERE EventData LIKE '%'
|
||||||
|
|
||||||
|
IdentityQueryEvents
|
||||||
|
| where isnotempty(AccountSid)
|
||||||
|
| take 100
|
||||||
|
|
||||||
|
// IdentityQueryEvents
|
||||||
|
// - contains query activities performed against Active Directory objects, such as users, groups, devices, and domains monitored by Azure ATP
|
||||||
|
// - Includes SAMR, DNS and LDAP requests
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
search 'microsoft.com'
|
||||||
|
| take 10
|
||||||
|
| project-reorder RemoteUrl
|
||||||
|
|
||||||
|
// search
|
||||||
|
// Searches the entire dataset for a given value
|
||||||
|
// Can be used to search the entire database (all tables and columns) all at once.
|
||||||
|
// Columns will be an aggregate of every table that brought back 1+ results, with
|
||||||
|
// columns having the same name merged together
|
||||||
|
|
||||||
|
// No true SQL equivalent (aside from indexing every table and searching the index, or unioning every table and column and searching that... yuck)
|
||||||
|
|
||||||
|
IdentityInfo
|
||||||
|
| search "administrator"
|
||||||
|
| take 100
|
||||||
|
| project-reorder AccountUpn, AccountName, AccountDisplayName, Surname, EmailAddress, JobTitle
|
||||||
|
|
||||||
|
// IdentityInfo
|
||||||
|
// - Contains information about users in Azure Active Directory
|
||||||
|
|
||||||
|
// Can be used with string equality comparisons. Comparison is row-based
|
||||||
|
|
||||||
|
search "administrator" and "cmd"
|
||||||
|
| take 100
|
||||||
|
| project-reorder ProcessCommandLine, FileName, AccountName, FolderPath, AccountDisplayName, Surname, EmailAddress, JobTitle, AccountUpn
|
||||||
|
|
@ -0,0 +1,325 @@
|
||||||
|
print Series = 'Tracking the Adversary with MTP Advanced Hunting', EpisodeNumber = 2, Topic = 'Joins', Presenter = 'Michael Melone, Tali Ash', Company = 'Microsoft'
|
||||||
|
|
||||||
|
// Language Reference: https://docs.microsoft.com/azure/kusto/query/
|
||||||
|
// Advanced Hunting Reference: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-schema-tables?view=o365-worldwide
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// Joins
|
||||||
|
// - Links two datasets together based on a common key
|
||||||
|
// - Can heavily impact performance depending on how datasets are joined
|
||||||
|
// - If datasets being joined are too large you may get an error
|
||||||
|
|
||||||
|
// ---------------
|
||||||
|
|
||||||
|
// The Join Statement
|
||||||
|
// In the below example, we will find users in the Finance department and determine where they have logged on.
|
||||||
|
// We'll accomplish this using the IdentityInfo table (user information) and the IdentityLogonEvents
|
||||||
|
// table.
|
||||||
|
|
||||||
|
IdentityLogonEvents
|
||||||
|
| take 100
|
||||||
|
|
||||||
|
// IdentityLogonEvents
|
||||||
|
// - Authentications performed against an on-prem DC or to Microsoft online services.
|
||||||
|
// - Contains success \ fail information, logon type, application, identity information, and client information
|
||||||
|
|
||||||
|
IdentityInfo
|
||||||
|
| where Department == 'Finance'
|
||||||
|
| join IdentityLogonEvents on AccountObjectId
|
||||||
|
|
||||||
|
// Note that we now have duplicate columns.
|
||||||
|
// the duplicates have a '1' at the end of the column name to
|
||||||
|
// avoid errors.
|
||||||
|
|
||||||
|
// This example uses two datasets, identified as "left" and "right"
|
||||||
|
// based on their location relative to the join statement.
|
||||||
|
|
||||||
|
// Left table:
|
||||||
|
IdentityInfo
|
||||||
|
| where Department == 'Finance'
|
||||||
|
|
||||||
|
// Right table:
|
||||||
|
IdentityLogonEvents
|
||||||
|
| take 100
|
||||||
|
|
||||||
|
// As long as the join column names match this should
|
||||||
|
// work nicely. If the column names do not match, we may
|
||||||
|
// need to specify which columns to join...
|
||||||
|
// We accomplish this by using $left. and $right.
|
||||||
|
|
||||||
|
IdentityInfo
|
||||||
|
| where Department == 'Finance'
|
||||||
|
| project-rename objid = AccountObjectId
|
||||||
|
| join IdentityLogonEvents on $left.objid == $right.AccountObjectId
|
||||||
|
|
||||||
|
// --------------------------------------------------------
|
||||||
|
|
||||||
|
// JOIN TYPES
|
||||||
|
// Now comes the fun part - understanding the default Kusto join.
|
||||||
|
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Hello",
|
||||||
|
0, "Hola",
|
||||||
|
1, "Salut",
|
||||||
|
1, "Ciao",
|
||||||
|
2, "Hallo"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "World",
|
||||||
|
0, "Mundo",
|
||||||
|
1, "Monde",
|
||||||
|
1, "Mondo",
|
||||||
|
2, "Welt"
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| join RightTable on key
|
||||||
|
|
||||||
|
// As you can see we are missing data. The default Kusto join
|
||||||
|
// deduplicates the left table based on the join column before
|
||||||
|
// joining the datasets together. Because of this, we lose
|
||||||
|
// "Hola" and "Ciao".
|
||||||
|
|
||||||
|
// This is important since it can directly result in missed
|
||||||
|
// detections! If you want to join data together using the
|
||||||
|
// standard inner join (the default in SQL) you need to specify
|
||||||
|
// kind = inner!
|
||||||
|
|
||||||
|
// The default join can be handy from a performance perspective. For
|
||||||
|
// example, let's say we wanted to produce a list of users who logged
|
||||||
|
// on to Windows 10 devices. The DeviceInfo table has duplicates (one
|
||||||
|
// row for each checkin), but we don't need them represented.
|
||||||
|
|
||||||
|
DeviceInfo
|
||||||
|
| where OSPlatform == 'Windows10'
|
||||||
|
| join DeviceLogonEvents on DeviceId
|
||||||
|
| distinct DeviceId, DeviceName, AccountDomain, AccountName, AccountSid
|
||||||
|
|
||||||
|
// Specifying kind=inner enables us to return all rows from both tables
|
||||||
|
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Hello",
|
||||||
|
0, "Hola",
|
||||||
|
1, "Salut",
|
||||||
|
1, "Ciao",
|
||||||
|
2, "Hallo"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "World",
|
||||||
|
0, "Mundo",
|
||||||
|
1, "Monde",
|
||||||
|
1, "Mondo",
|
||||||
|
2, "Welt"
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| join kind=inner RightTable on key
|
||||||
|
|
||||||
|
// This comes in handy when you want to see every network communication within 5 minutes
|
||||||
|
// of an alert event on the device
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where isnotempty(DeviceId)
|
||||||
|
| project-rename AlertTimestamp = Timestamp
|
||||||
|
| join kind=inner DeviceNetworkEvents on DeviceId
|
||||||
|
| where Timestamp between (datetime_add('minute', -5, AlertTimestamp) .. datetime_add('minute', 5, AlertTimestamp))
|
||||||
|
|
||||||
|
// Other types of joins
|
||||||
|
// - left outer: all rows from the left table regardless if they match on the right
|
||||||
|
// - right outer: all rows from the right table regardless if they match on the left
|
||||||
|
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Foo",
|
||||||
|
1, "Bar",
|
||||||
|
2, "Baz",
|
||||||
|
3, "Qux",
|
||||||
|
4, "Quux"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Wibble",
|
||||||
|
1, "Wobble",
|
||||||
|
2, "Wubble",
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| join kind=leftouter RightTable on key
|
||||||
|
|
||||||
|
// For example, let’s say we wanted a list of all emails that the malware
|
||||||
|
// filter detected as phishing paired with details about their attachments.
|
||||||
|
|
||||||
|
// EmailEvents
|
||||||
|
// ref: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-emailevents-table?view=o365-worldwide
|
||||||
|
// Contains information about e-mails processed through Office ATP, including
|
||||||
|
// - Standard email metadata
|
||||||
|
// - Whether phish or malware detection identified the e-mail as malicious upon receipt
|
||||||
|
// - Actions taken by Office ATP on the e-mail upon receipt
|
||||||
|
|
||||||
|
// EmailAttachmentInfo
|
||||||
|
// ref: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-emailattachmentinfo-table?view=o365-worldwide
|
||||||
|
// Contains information about e-mail attachments
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| where ThreatTypes == "Phish"
|
||||||
|
| join kind=leftouter EmailAttachmentInfo on NetworkMessageId, RecipientObjectId
|
||||||
|
| take 100
|
||||||
|
|
||||||
|
// EmailEvents can tell us what e-mails were picked up as phishing, but we won’t
|
||||||
|
// have an entry in EmailAttachmentInfo for each since many are unlikely to have
|
||||||
|
// an attachment. To accomplish this we used left outer join.
|
||||||
|
|
||||||
|
// ------------------------------------------
|
||||||
|
// - full outer: all rows of both tables despite whether or not they match each other
|
||||||
|
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Foo",
|
||||||
|
1, "Bar",
|
||||||
|
2, "Baz",
|
||||||
|
3, "Qux",
|
||||||
|
4, "Quux"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
2, "Wibble",
|
||||||
|
3, "Wobble",
|
||||||
|
16, "Wubble",
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| join kind=fullouter RightTable on key
|
||||||
|
|
||||||
|
// I use this in a query I use reporting on antimalware signature, engine, and platform versions.
|
||||||
|
|
||||||
|
let StartDate = ago(30d);
|
||||||
|
DeviceFileEvents
|
||||||
|
| where Timestamp > StartDate
|
||||||
|
// Find signature \ engine update activity
|
||||||
|
| where InitiatingProcessFileName =~ 'MpSigStub.exe' and InitiatingProcessCommandLine contains '/stub' and InitiatingProcessCommandLine contains '/payload'
|
||||||
|
| summarize Timestamp = arg_max(Timestamp, InitiatingProcessCommandLine) by DeviceId, DeviceName
|
||||||
|
| extend SplitCommand = split(InitiatingProcessCommandLine, ' ')
|
||||||
|
// Locate stub and payload versions
|
||||||
|
| extend EngineVersionLocation = array_index_of(SplitCommand, "/stub") + 1, DefinitionVersionLocation = array_index_of(SplitCommand, "/payload") + 1
|
||||||
|
| project Timestamp, DeviceName, DeviceId, AMEngineVersion = SplitCommand[EngineVersionLocation], AntivirusSignatureVersion = SplitCommand[DefinitionVersionLocation]
|
||||||
|
| join kind=fullouter (
|
||||||
|
DeviceProcessEvents
|
||||||
|
| where Timestamp > StartDate
|
||||||
|
// Find process creations for MsMpEng from the platform folder
|
||||||
|
| where FileName =~ 'MsMpEng.exe' and FolderPath contains @"\Microsoft\Windows Defender\Platform\"
|
||||||
|
| summarize arg_max(Timestamp, FolderPath) by DeviceId, DeviceName
|
||||||
|
// Go up two levels
|
||||||
|
| project DeviceId, DeviceName, AMServiceVersion = split(FolderPath, '\\')[-2]
|
||||||
|
) on DeviceId
|
||||||
|
// Re-projecting to make the UI happy
|
||||||
|
| project DeviceId, DeviceName, AMEngineVersion, AntivirusSignatureVersion, AMServiceVersion
|
||||||
|
|
||||||
|
// There are also anti joins and semi joins which are designed to quickly reduce datasets
|
||||||
|
|
||||||
|
// anti joins will remove any matching rows and return only the left or right table
|
||||||
|
// - leftanti: removes any rows that match between the two tables, only returns the left table
|
||||||
|
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Foo",
|
||||||
|
1, "Bar",
|
||||||
|
2, "Baz",
|
||||||
|
3, "Qux",
|
||||||
|
4, "Quux"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
2, "Wibble",
|
||||||
|
3, "Wobble",
|
||||||
|
16, "Wubble",
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| join kind=leftanti RightTable on key
|
||||||
|
|
||||||
|
// rightanti - you guessed it. It removes matches and returns values from the right table
|
||||||
|
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Foo",
|
||||||
|
1, "Bar",
|
||||||
|
2, "Baz",
|
||||||
|
3, "Qux",
|
||||||
|
4, "Quux"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
2, "Wibble",
|
||||||
|
3, "Wobble",
|
||||||
|
16, "Wubble",
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| join kind=rightanti RightTable on key
|
||||||
|
// Let’s say you wanted to see e-mails which were identified as either phishing
|
||||||
|
// or malware which were likely still in user’s mailboxes. To achieve this, we
|
||||||
|
// will use EmailEvents to identify the suspicious e-mails and filter the results
|
||||||
|
// using the EmailPostDeliveryEvents table.
|
||||||
|
|
||||||
|
// EmailPostDeliveryEvents
|
||||||
|
// ref: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-emailpostdeliveryevents-table?view=o365-worldwide
|
||||||
|
// contains information about post-delivery remediation actions such as manual administrator
|
||||||
|
// remediation, phish zap, or malware zap
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| where ThreatTypes in ('Phish', 'Malware') and EmailAction !in ('Replace attachment', 'Send to quarantine')
|
||||||
|
| join kind=leftanti EmailPostDeliveryEvents on NetworkMessageId , RecipientEmailAddress
|
||||||
|
|
||||||
|
// For all of the joins, check out: https://docs.microsoft.com/azure/kusto/query/joinoperator
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
|
||||||
|
// union
|
||||||
|
// Sometimes you want to "link" two queries together into one result instead of joining them based on a key.
|
||||||
|
// To accomplish this you would use the union operator. A union merges all rows from each query where the column
|
||||||
|
// name and data type match.
|
||||||
|
|
||||||
|
let LeftTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
0, "Foo",
|
||||||
|
1, "Bar",
|
||||||
|
2, "Baz",
|
||||||
|
3, "Qux",
|
||||||
|
4, "Quux"
|
||||||
|
];
|
||||||
|
let RightTable = datatable (key:int, value:string)
|
||||||
|
[
|
||||||
|
2, "Wibble",
|
||||||
|
3, "Wobble",
|
||||||
|
16, "Wubble",
|
||||||
|
];
|
||||||
|
LeftTable
|
||||||
|
| union RightTable
|
||||||
|
|
||||||
|
// Notice we no longer have the extra columns from a join. This might be useful if you want to track
|
||||||
|
// logon activity with devices (the DeviceLogonEvents table) and Active Directory \ Azure Active Directory
|
||||||
|
// (the IdentityLogonEvents table) in one query.
|
||||||
|
|
||||||
|
DeviceLogonEvents
|
||||||
|
| extend Table = 'DeviceLogonEvents'
|
||||||
|
| take 100
|
||||||
|
| union (
|
||||||
|
IdentityLogonEvents
|
||||||
|
| extend Table = 'IdentityLogonEvents'
|
||||||
|
| take 100
|
||||||
|
)
|
||||||
|
| project-reorder Timestamp, Table, AccountDomain, AccountName, AccountUpn, AccountSid
|
||||||
|
| order by Timestamp asc
|
||||||
|
|
||||||
|
|
||||||
|
// --------------------------------------
|
||||||
|
|
||||||
|
// Functions are a special sort of join which let you pull more static data about a file (more are
|
||||||
|
// planned in the future, stay tuned!). This is really helpful when you want to get information about
|
||||||
|
// file prevalence or antimalware detections.
|
||||||
|
|
||||||
|
// Let's say we wanted information about rare files involved in a process creation event
|
||||||
|
|
||||||
|
DeviceProcessEvents
|
||||||
|
| invoke FileProfile() // Call the FileProfile function
|
||||||
|
| where isnotempty(GlobalPrevalence) and GlobalPrevalence < 1000 // Note that in the real world you might want to include empty GlobalPrevalence
|
||||||
|
| project-reorder DeviceName, FileName, ProcessCommandLine, FileSize, GlobalPrevalence, GlobalFirstSeen, GlobalLastSeen, ThreatName, Publisher, SoftwareName
|
||||||
|
| top 100 by GlobalPrevalence asc
|
||||||
|
|
@ -0,0 +1,186 @@
|
||||||
|
print Series = 'Tracking the Adversary with MTP Advanced Hunting', EpisodeNumber = 3, Topic = 'Summarizing, Pivoting, and Visualizing Data', Presenters = 'Michael Melone, Tali Ash', Company = 'Microsoft'
|
||||||
|
|
||||||
|
// summarize
|
||||||
|
// The summarize operator enables you to perform
|
||||||
|
// a variety of calculations on data.
|
||||||
|
|
||||||
|
// The output of summarize will be a table with one
|
||||||
|
// column for each row value you pivoted on as well
|
||||||
|
// as one column for each pivot you performed.
|
||||||
|
|
||||||
|
// In the following example, we will calculate the number of e-mails based on whether
|
||||||
|
// Office ATP identified them as being malware.
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT MalwareFilterVerdict, Count(*) FROM EmailAttachmentInfo GROUP BY MalwareFilterVerdict
|
||||||
|
|
||||||
|
EmailAttachmentInfo
|
||||||
|
| summarize count() by ThreatTypes
|
||||||
|
|
||||||
|
// --------------------------------------------
|
||||||
|
|
||||||
|
// Summarize can also be used to create 2 column pivots by simply adding another
|
||||||
|
// column name after the "by" clause. For example, we will now count the number
|
||||||
|
// of e-mails received by sender and recipient combo
|
||||||
|
|
||||||
|
// You will also notice in this example that the count_ has been renamed to Emails
|
||||||
|
// to make the query easier to understand.
|
||||||
|
|
||||||
|
// SQL Equivalent: SELECT TOP 100 SenderFromAddress, RecipientEmailAddress, Emails = Count(*) FROM EmailEvents GROUP BY SenderFromAddress, RecipientEmailAddress ORDER BY Emails DESC
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| summarize Emails = count() by SenderFromAddress, RecipientEmailAddress
|
||||||
|
| top 100 by Emails desc
|
||||||
|
|
||||||
|
// --------------------------------------------
|
||||||
|
|
||||||
|
// min() - obtains the minimim value from the set
|
||||||
|
// max() - obtains the maximum value from the set
|
||||||
|
|
||||||
|
// SQL Equivalent:
|
||||||
|
// SELECT
|
||||||
|
// Earliest = min(Timestamp)
|
||||||
|
// , Latest = max(Timestamp)
|
||||||
|
// , Count = count()
|
||||||
|
// , AccountName
|
||||||
|
// FROM AlertEvidence
|
||||||
|
// WHERE AccountName LIKE '%'
|
||||||
|
// GROUP BY AccountName
|
||||||
|
// ORDER BY Count desc
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where isnotempty(AccountName)
|
||||||
|
| summarize Earliest = min(Timestamp), Latest = max(Timestamp), Count = count() by AccountName
|
||||||
|
| order by Count desc
|
||||||
|
|
||||||
|
// AlertEvidence
|
||||||
|
// Contains information on entities and evidence involved in an alert, such as devices, accounts, and emails
|
||||||
|
//--------------------------
|
||||||
|
|
||||||
|
// Now let's get a bit more advanced. Using the bin() function you can group events by a period of time.
|
||||||
|
// Let's take a look at some logon statistics on a daily basis
|
||||||
|
|
||||||
|
// Using render we can automatically create a chart. Let's look at account logon activity over time on a
|
||||||
|
// daily basis by UPN.
|
||||||
|
|
||||||
|
IdentityLogonEvents
|
||||||
|
| where isnotempty(AccountUpn)
|
||||||
|
| summarize NumberOfLogons = count() by AccountUpn, bin(Timestamp, 1d)
|
||||||
|
| render timechart
|
||||||
|
|
||||||
|
// render - creates a chart
|
||||||
|
|
||||||
|
// We can also use this bin'ed data to determine min, max, and average daily logons
|
||||||
|
|
||||||
|
IdentityLogonEvents
|
||||||
|
| where isnotempty(AccountUpn)
|
||||||
|
| summarize NumberOfLogons = count()
|
||||||
|
by AccountUpn
|
||||||
|
, bin(Timestamp, 1d)
|
||||||
|
| summarize TotalLogons = sum(NumberOfLogons)
|
||||||
|
, AverageDailyLogons = avg(NumberOfLogons)
|
||||||
|
, FewestLogonsInADay = min(NumberOfLogons)
|
||||||
|
, MostLogonsInADay = max(NumberOfLogons)
|
||||||
|
by AccountUpn
|
||||||
|
| top 10 by TotalLogons desc
|
||||||
|
| render columnchart
|
||||||
|
|
||||||
|
--------------------------
|
||||||
|
|
||||||
|
// You can also use summarize to get the latest event from each category.
|
||||||
|
// For example, let's say you want to get the latest check-in information
|
||||||
|
// for each device in your instance
|
||||||
|
|
||||||
|
// the arg_max() function will maximize the specified argument in the column
|
||||||
|
// set based on the "by" parameter. You can either specify the columns you
|
||||||
|
// want back as parameters, or just use * to get the entire row.
|
||||||
|
|
||||||
|
DeviceInfo
|
||||||
|
| where isnotempty(OSPlatform) // checkins can be partial or full - this filters out partials
|
||||||
|
| summarize arg_max(Timestamp, *) by DeviceId
|
||||||
|
|
||||||
|
// Let's say you now wanted to use this summarized list to create a report of devices
|
||||||
|
// by operating system - but you didn't want to lose the individual device names.
|
||||||
|
// Good news - we can also use summarize to build arrays!
|
||||||
|
|
||||||
|
DeviceInfo
|
||||||
|
| where isnotempty(OSPlatform) // checkins can be partial or full - this filters out partials
|
||||||
|
| summarize arg_max(Timestamp, *) by DeviceId
|
||||||
|
| summarize Devices = count(), DeviceList = make_set(DeviceName) by OSPlatform
|
||||||
|
|
||||||
|
// -----------------------------
|
||||||
|
|
||||||
|
// Another way to perform aggregations is using make-series. The make-series
|
||||||
|
// command is similar to summarize except it is designed to calculate on
|
||||||
|
// a periodic basis, providing zeros for empty datasets for consistency.
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| make-series count() on Timestamp from ago(30d) to now() step 1d by SenderFromDomain
|
||||||
|
|
||||||
|
// With this we can identify outlier programmatically. Let's see if we can find
|
||||||
|
// any sudden increases or decreases in activity relating to mail from a specific
|
||||||
|
// domain using one of our time series analysis capabilities.
|
||||||
|
|
||||||
|
// geek stuff warning
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| make-series MailCount = count() on Timestamp from ago(30d) to now() step 1d by SenderFromDomain
|
||||||
|
| extend (flag, score, baseline) = series_decompose_anomalies(MailCount)
|
||||||
|
| project-reorder flag, score, baseline
|
||||||
|
|
||||||
|
// series_decompose_anomalies adds three new columns
|
||||||
|
// - flag: is the datapoint normal, an abnormal increase (1), or an abnormal decrease (-1)
|
||||||
|
// - score: how anomalous is this data point?
|
||||||
|
// - baseline: the forecaseted value the algorithm expected
|
||||||
|
|
||||||
|
// Let's look for spikes in e-mail traffic from a domain. To do this, we need to expand
|
||||||
|
// the flag column. Expanding takes an array and creates one row for each value in it.
|
||||||
|
// For SQL people, this is like using CROSS APPLY
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| make-series MailCount = count() on Timestamp from ago(30d) to now() step 1d by SenderFromDomain
|
||||||
|
| extend (flag, score, baseline) = series_decompose_anomalies(MailCount)
|
||||||
|
| project-reorder flag, score, baseline
|
||||||
|
| mv-expand flag
|
||||||
|
|
||||||
|
// now we can filter to only 1's. Note that our lists of values all look like strings. We need
|
||||||
|
// to tell KQL that we want these to be int's for accurate comparison.
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| make-series MailCount = count() on Timestamp from ago(30d) to now() step 1d by SenderFromDomain
|
||||||
|
| extend (flag, score, baseline) = series_decompose_anomalies(MailCount)
|
||||||
|
| mv-expand flag to typeof(int) // expand flag and tell KQL it needs to be an int
|
||||||
|
| where flag == 1 // filter to only rows that have a 1
|
||||||
|
| project-reorder flag, score, baseline
|
||||||
|
|
||||||
|
// Next, we'll look for the top 5 most anomalous domain spikes and graph the result
|
||||||
|
|
||||||
|
let interval = 12h;
|
||||||
|
EmailEvents
|
||||||
|
| make-series MailCount = count() on Timestamp from ago(30d) to now() step interval by SenderFromDomain
|
||||||
|
| extend (flag, score, baseline) = series_decompose_anomalies(MailCount)
|
||||||
|
| mv-expand flag to typeof(int)
|
||||||
|
| where flag == 1 // filter to only incremental anomalies
|
||||||
|
| mv-expand score to typeof(double) // expand the score array to a double
|
||||||
|
| summarize MaxScore = max(score) by SenderFromDomain // get the max score value from each domain
|
||||||
|
| top 5 by MaxScore desc // Get the top 5 highest scoring domains
|
||||||
|
| join kind=rightsemi EmailEvents on SenderFromDomain // Filter EmailEvents to only these domains
|
||||||
|
| summarize count() by SenderFromDomain, bin(Timestamp, interval) // build a new summarization for the graph
|
||||||
|
| render timechart // graph it!
|
||||||
|
|
||||||
|
// Aha! I know someone out there sees my bug. Technically, one of these datasets can have both a spike and
|
||||||
|
// a valley and the valley score could be what we're keying off of. Let's try again using logons, but this time
|
||||||
|
// we'll get the specific score associated with the spike instead of just assuming that they're the same.
|
||||||
|
|
||||||
|
let interval = 12h;
|
||||||
|
IdentityLogonEvents
|
||||||
|
| where isnotempty(AccountUpn)
|
||||||
|
| make-series LogonCount = count() on Timestamp from ago(30d) to now() step interval by AccountUpn
|
||||||
|
| extend (flag, score, baseline) = series_decompose_anomalies(LogonCount)
|
||||||
|
| mv-expand with_itemindex = FlagIndex flag to typeof(int) // Expand, but this time include the index in the array as FlagIndex
|
||||||
|
| where flag == 1 // Once again, filter only to spikes
|
||||||
|
| extend SpikeScore = todouble(score[FlagIndex]) // This will get the specific score associated with the detected spike
|
||||||
|
| summarize MaxScore = max(SpikeScore) by AccountUpn
|
||||||
|
| top 5 by MaxScore desc
|
||||||
|
| join kind=rightsemi IdentityLogonEvents on AccountUpn
|
||||||
|
| summarize count() by AccountUpn, bin(Timestamp, interval)
|
||||||
|
| render timechart
|
||||||
|
|
@ -0,0 +1,256 @@
|
||||||
|
print Series = 'Tracking the Adversary with MTP Advanced Hunting', EpisodeNumber = 4, Topic = 'Lets Hunt! Applying KQL to Incident Tracking', Presenter = 'Michael Melone, Tali Ash', Company = 'Microsoft'
|
||||||
|
|
||||||
|
|
||||||
|
// Schema Reference (upper right corner)
|
||||||
|
|
||||||
|
|
||||||
|
// The ABC's of Security
|
||||||
|
// - Authentication
|
||||||
|
// - Backdoors
|
||||||
|
// - Communication
|
||||||
|
// - Data
|
||||||
|
// Authentication
|
||||||
|
// - How is the attacker establishing identity to the system?
|
||||||
|
// - What identities do we consider compromised?
|
||||||
|
// - What are our administrative identities?
|
||||||
|
// Backdoors
|
||||||
|
// - How is the attacker controlling the system?
|
||||||
|
// - Is the service used by the attacker legitimate or illegitimate?
|
||||||
|
// - Where is this capability or condition present?
|
||||||
|
// Communication
|
||||||
|
// - How is the attacker communicating with the system?
|
||||||
|
// Let's see what the malware fairy has brought us today...
|
||||||
|
|
||||||
|
AlertInfo
|
||||||
|
| take 10
|
||||||
|
|
||||||
|
// AlertInfo
|
||||||
|
// Table containing alerts identified by MTP. By itself does not have the entities and evidence
|
||||||
|
// associated with the alert. To get that we will need the AlertEvidence table.
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| take 10
|
||||||
|
|
||||||
|
// AlertEvidence
|
||||||
|
// Details about alerts including associated entities
|
||||||
|
// Let's find out which of our accounts has the most alerts associated with them
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where Timestamp > ago(19d) and EntityType == "User" and isnotempty(AccountObjectId) // Look for user entities
|
||||||
|
| summarize Alerts = dcount(AlertId) by AccountObjectId, AccountName , AccountDomain
|
||||||
|
| project Alerts, AccountDomain, AccountName, AccountObjectId
|
||||||
|
| order by Alerts desc
|
||||||
|
|
||||||
|
// That's suspicious... Let's see what kinds of alerts these are...
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where Timestamp > ago(19d) and EntityType == "User" and AccountObjectId == 'ab653b2a-d23e-49df-9493-c26590f8f319'
|
||||||
|
| join kind=inner AlertInfo on AlertId
|
||||||
|
| summarize Alerts = count(), First = min(Timestamp), Last = max(Timestamp) by Title
|
||||||
|
| order by Alerts desc
|
||||||
|
|
||||||
|
// That doesn't look good. Let's find out when and where this happened...
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where Timestamp > ago(19d) and AccountObjectId == 'ab653b2a-d23e-49df-9493-c26590f8f319' // associated with the suspicious account
|
||||||
|
| join kind=rightsemi AlertEvidence on AlertId // rejoin with evidence...
|
||||||
|
| where EntityType == 'Machine' // and get the machines.
|
||||||
|
| join kind=leftouter (
|
||||||
|
DeviceInfo
|
||||||
|
| summarize DeviceName = any(DeviceName) by DeviceId // Get the device name
|
||||||
|
) on DeviceId
|
||||||
|
| summarize dcount(AlertId) by DeviceName , bin(Timestamp, 1d) // Plot it in 30 minute intervals
|
||||||
|
| render timechart // Make a timechart
|
||||||
|
|
||||||
|
// OK! We have some boxes of interest and it looks like it started on barbaram-pc.
|
||||||
|
// We can also see an uptick in activity on July 19th
|
||||||
|
// Let's timeline alerts on barbaram-pc.
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where Timestamp > ago(19d) and DeviceId == '87da11a9257988b2fc090c9f05c72f6453bc53de'
|
||||||
|
| join kind=inner AlertInfo on AlertId
|
||||||
|
| summarize min(Timestamp) by Title
|
||||||
|
| order by min_Timestamp asc
|
||||||
|
|
||||||
|
// Looks like we detected something malicious from Office 365... Let's see what it was
|
||||||
|
|
||||||
|
AlertInfo
|
||||||
|
| where Timestamp > ago(19d) and Title == 'Post-delivery detection of suspicious attachment'
|
||||||
|
| join kind=rightsemi AlertEvidence on AlertId
|
||||||
|
| where EntityType == 'File'
|
||||||
|
|
||||||
|
// OK, all of this JSON is great, but how about a table instead
|
||||||
|
|
||||||
|
AlertInfo
|
||||||
|
| where Timestamp > ago(19d) and Title == 'Post-delivery detection of suspicious attachment'
|
||||||
|
| join kind=rightsemi AlertEvidence on AlertId
|
||||||
|
| where EntityType == 'File'
|
||||||
|
| extend AFDynamic = parse_json(AdditionalFields) // Turn JSON into a dynamic column
|
||||||
|
| evaluate bag_unpack(AFDynamic) // ...and turn the JSON into columns
|
||||||
|
| project-reorder Name, Directory, Host, SHA256
|
||||||
|
|
||||||
|
// parse_json() - parses a JSON string and turns it into a dynamic
|
||||||
|
// bag_unpack() - takes the first-level properties from a dynamic and promotes them to columns
|
||||||
|
|
||||||
|
// Looks like the file was called Doodles_SOW_07102020.doc...
|
||||||
|
|
||||||
|
DeviceProcessEvents
|
||||||
|
| where Timestamp > ago(19d)
|
||||||
|
and ProcessCommandLine contains 'UpdatedPolicy_SOW_07182020.doc'
|
||||||
|
and AccountObjectId == 'ab653b2a-d23e-49df-9493-c26590f8f319'
|
||||||
|
|
||||||
|
|
||||||
|
// ...and we can see that Barbara launched it. Process ID 13988
|
||||||
|
|
||||||
|
|
||||||
|
// Looks like Barbara used Word to open it a couple times...
|
||||||
|
// Let's see what happened when she opened it...
|
||||||
|
|
||||||
|
search in (DeviceProcessEvents, DeviceNetworkEvents, DeviceFileEvents, DeviceRegistryEvents, DeviceEvents )
|
||||||
|
Timestamp > ago(19d)
|
||||||
|
and DeviceId == '87da11a9257988b2fc090c9f05c72f6453bc53de'
|
||||||
|
and InitiatingProcessId == 13988
|
||||||
|
| where RegistryKey !contains @'\Software\Microsoft\Office\16.0\Common\Internet\Server Cache' // Filtering out cache registry key changes
|
||||||
|
| order by Timestamp asc
|
||||||
|
| project-reorder Timestamp, $table, ActionType, RemoteIP, RemoteUrl, FileName, SHA256, RegistryKey, RegistryValueData, ActionType, AdditionalFields
|
||||||
|
|
||||||
|
// Interesting. Word is allocating writable and executable memory right after launch, but
|
||||||
|
// nothing too interesting otherwise.
|
||||||
|
// ref: https://docs.microsoft.com/windows-hardware/drivers/ddi/ntifs/nf-ntifs-ntallocatevirtualmemory
|
||||||
|
|
||||||
|
// So that doc is on SharePoint. How did it get there?
|
||||||
|
AppFileEvents
|
||||||
|
| where Timestamp > ago(19d) and FileName =~ 'UpdatedPolicy_SOW_07182020.doc'
|
||||||
|
| project-reorder Timestamp, ActionType, Application, FolderPath, IPAddress, Location, ISP
|
||||||
|
| order by Timestamp asc
|
||||||
|
|
||||||
|
// Looks like we have a couple strange IPs interacting with the file: 178.32.124.142 and 51.83.139.56.
|
||||||
|
// It was uploaded using Barbara's account - that's the Authentication
|
||||||
|
// The "backdoor" is just a publicly available service (SharePoint)
|
||||||
|
// The Communication channel are those IPs. Let's see what else was involved with them...
|
||||||
|
|
||||||
|
search Timestamp > ago(19d) and ('178.32.124.142' or '51.83.139.56')
|
||||||
|
| project-reorder $table, Timestamp, AccountName, AccountDomain, ActionType, FileName, FolderPath
|
||||||
|
|
||||||
|
// ...looks like there was another doc uploaded from that same user and IP (BYODRegistration (1).docm).
|
||||||
|
// Maybe we'll investigate that later.
|
||||||
|
|
||||||
|
// We also had a couple alerts. Let's dig deeper.
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where RemoteIP in ('178.32.124.142', '51.83.139.56')
|
||||||
|
| join kind=rightsemi AlertInfo on AlertId
|
||||||
|
|
||||||
|
// Aha! Those are our Tor addresses.
|
||||||
|
|
||||||
|
// So we know there was credential theft going on. Let's see what other accounts logged on
|
||||||
|
// to that compromised system...
|
||||||
|
|
||||||
|
DeviceLogonEvents
|
||||||
|
| where DeviceName == 'barbaram-pc.mtpdemos.net' and Timestamp > ago(19d) and ActionType == 'LogonSuccess'
|
||||||
|
| where AccountDomain !in ('font driver host', 'window manager') // Ignoring internal system identities at the moment
|
||||||
|
| extend Account = strcat(AccountDomain, '\\', AccountName )
|
||||||
|
| summarize count() by Account, bin(Timestamp, 1h)
|
||||||
|
| render timechart
|
||||||
|
|
||||||
|
// Interesting. What does Eric Gubbels do?
|
||||||
|
|
||||||
|
IdentityInfo
|
||||||
|
| where GivenName =~ 'Eric' and Surname =~ "Gubbels"
|
||||||
|
| take 1
|
||||||
|
|
||||||
|
// OK, so he's the help desk supervisor. He probably has elevated permissions.
|
||||||
|
// Another account. Where else did he log on?
|
||||||
|
|
||||||
|
IdentityLogonEvents
|
||||||
|
| where Timestamp > todatetime('2020-07-17') and AccountObjectId == '993788dd-7c13-4db8-9b0a-6297fcb8d5b3' and isnotempty(DeviceName)
|
||||||
|
| summarize count() by DeviceName, bin(Timestamp, 1d)
|
||||||
|
| render timechart
|
||||||
|
|
||||||
|
// Ok, what alerts do we have with his account?
|
||||||
|
|
||||||
|
let EricGAlerts = (
|
||||||
|
AlertEvidence
|
||||||
|
| where Timestamp > todatetime('2020-07-17') and AccountObjectId == '993788dd-7c13-4db8-9b0a-6297fcb8d5b3'
|
||||||
|
); // Get all alerts for EricG's account
|
||||||
|
EricGAlerts
|
||||||
|
| join kind=rightsemi AlertInfo on AlertId // Get the alertinfo
|
||||||
|
| join AlertEvidence on AlertId // Join back on AlertEvidence to get other evidence
|
||||||
|
| join kind = leftouter (
|
||||||
|
DeviceInfo
|
||||||
|
| summarize DeviceName = any(DeviceName) by DeviceId
|
||||||
|
) on DeviceId // This creates a mapping table between DeviceId and DeviceName since we only have ID in AlertEvidence
|
||||||
|
| extend DomainAndAccount = strcat(AccountDomain, '\\', AccountName)
|
||||||
|
| summarize Timestamp = min(Timestamp)
|
||||||
|
, Device = make_set_if(DeviceName, isnotempty(DeviceName))
|
||||||
|
, SHA1 = make_set_if(SHA1,isnotempty(SHA1))
|
||||||
|
, SHA256 = make_set_if(SHA256, isnotempty(SHA256))
|
||||||
|
, RemoteIP = make_set_if(RemoteIP, isnotempty(RemoteIP))
|
||||||
|
, RemoteUrl = make_set_if(RemoteUrl, isnotempty(RemoteUrl))
|
||||||
|
, Account = make_set_if(DomainAndAccount, DomainAndAccount != '\\') by AlertId, Title // Build a nice JSON report of each alert
|
||||||
|
| order by Timestamp asc
|
||||||
|
|
||||||
|
// make_set_if() - Creates a list of unique values from the specified column when they match the
|
||||||
|
// condition in the second parameter.
|
||||||
|
// makeset() - same thing without the conditional operator
|
||||||
|
// makelist() \ make_list_if() - same as makeset but without deduplication
|
||||||
|
|
||||||
|
// OK! We have some interesting things here
|
||||||
|
// - A new device of interest - robertot-pc
|
||||||
|
// - We've found out that the attacker may have created a malicious inbox forwarding rule (backdoor) set from 52.137.127.6 (communication)
|
||||||
|
// - We can see evidence of a possible skeleton key attack (Authentication)
|
||||||
|
// - A few logons using potentially stolen credentials [mtp-air-aadconnect01 and mtp-air-dc01] (Authentication)
|
||||||
|
// I wonder if that IP address is one of our devices...
|
||||||
|
|
||||||
|
DeviceInfo
|
||||||
|
| where PublicIP == "52.137.127.6"
|
||||||
|
| distinct DeviceName
|
||||||
|
|
||||||
|
// Bingo! Back to barbaram-pc. Yup, we'll have to queue that up for investigation.
|
||||||
|
// Let's look for that other Word doc...
|
||||||
|
|
||||||
|
DeviceFileEvents
|
||||||
|
| where Timestamp > ago(19d) and FileName =~ "BYODRegistration (1).docm"
|
||||||
|
| summarize count() by SHA1, SHA256, MD5
|
||||||
|
|
||||||
|
// Got our file hash - let's see what the world knows about it
|
||||||
|
// Backdoor: c18732c861641a5a91d1578efad6f1a2546dc4bd97c68a5f6a6ba5d4f5d76242
|
||||||
|
|
||||||
|
DeviceFileEvents
|
||||||
|
| where SHA256 == 'c18732c861641a5a91d1578efad6f1a2546dc4bd97c68a5f6a6ba5d4f5d76242'
|
||||||
|
| take 1
|
||||||
|
| invoke FileProfile() // Note you need the SHA1 for this to work
|
||||||
|
| project-reorder GlobalPrevalence, GlobalFirstSeen, GlobalLastSeen , Signer, Issuer, SignerHash, IsCertificateValid, IsRootSignerMicrosoft, IsExecutable, ThreatName, Publisher, SoftwareName
|
||||||
|
|
||||||
|
// Low prevalence, first seen April of 2020. Might be targeted, but it is a Word doc
|
||||||
|
// so global prevalence might be misleading...
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////
|
||||||
|
// As you can see, using the ABC method is a quick way to pivot
|
||||||
|
// through an incident. But Advanced Hunting doesn't stop there.
|
||||||
|
///////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
// It is clear this file is malicious, we don’t want it in our env.
|
||||||
|
// We would like to take action on the malicious file – quarantine it
|
||||||
|
|
||||||
|
DeviceFileEvents
|
||||||
|
| where SHA256 == 'c18732c861641a5a91d1578efad6f1a2546dc4bd97c68a5f6a6ba5d4f5d76242'
|
||||||
|
|
||||||
|
|
||||||
|
// We found several IOCs during this investigation, like IPs and file hashes.
|
||||||
|
// We would like to make sure we will get alerted next time we see one of the IOCs in
|
||||||
|
// our env, therefore we will create a custom detection rule.
|
||||||
|
|
||||||
|
// Custom detection rule to get alerted on every future activity involving IP:
|
||||||
|
// '178.32.124.142', '51.83.139.56'
|
||||||
|
|
||||||
|
search in (DeviceNetworkEvents, DeviceEvents)
|
||||||
|
RemoteIP in ('178.32.124.142', '51.83.139.56') or FileOriginIP in ('178.32.124.142', '51.83.139.56') or IPAddress in ('178.32.124.142', '51.83.139.56')
|
||||||
|
|
||||||
|
// Detection name – Activity involving malicious IP ('178.32.124.142', '51.83.139.56')
|
||||||
|
// Alert title – Activity involving malicious IP
|
||||||
|
// Category – Suspicious activity
|
||||||
|
// MITRE techniques -
|
||||||
|
// Description – Activity with '178.32.124.142', '51.83.139.56' was observed
|
||||||
|
|
||||||
|
// Go Hunt
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
# Tracking The Adversary
|
||||||
|
|
||||||
|
**[Webcast Link](https://techcommunity.microsoft.com/t5/microsoft-threat-protection/webinar-series-unleash-the-hunter-in-you/ba-p/1509232)**
|
||||||
|
|
||||||
|
This webcast is designed to take you from newbie to ninja on advanced hunting in four episodes. This repo contains the query files used in each of the webcasts so that you can hunt in your own MTP instance.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Episode 1: KQL Fundamentals
|
||||||
|
|
||||||
|
In the first episode, we will cover the basics of advanced hunting capabilities in Microsoft Threat Protection (MTP). Learn about available advanced hunting data and basic KQL syntax and operators. The best part? No slides!
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Episode 2: Joins
|
||||||
|
|
||||||
|
In episode 2, we will continue learning about data in advanced hunting and how to join tables together. Learn about inner, outer, unique, and semi joins, as well as the nuances of the default Kusto innerunique join. Make Edgar F. Codd proud!
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Episode 3: Summarizing, pivoting, and visualizing Data
|
||||||
|
|
||||||
|
Now that we’re able to filter, manipulate, and join data, it’s time to start summarizing, quantifying, pivoting, and visualizing. In this episode, we will cover the summarize operator and some of the various calculations you can perform while diving into additional tables within MTP. We will turn our datasets into charts that can help improve analysis.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Episode 4: Let’s hunt! Applying KQL to incident tracking
|
||||||
|
|
||||||
|
Time to track some attacker activity! In this episode, we will use our improved understanding of KQL and advanced hunting in Microsoft Threat Protection to track an attack. Learn some of the tips and tricks used in the field to track attacker activity, including the ABCs of cybersecurity and how to apply them to incident response.
|
||||||
|
|
@ -0,0 +1,225 @@
|
||||||
|
print Topic = "l33tSpeak: Advanced hunting in Microsoft 365 Defender"
|
||||||
|
, Presenters = pack_array("Sebastien Molendijk, Michael Melone, Tali Ash")
|
||||||
|
, Company = "Microsoft"
|
||||||
|
, Date = todatetime("10 MAY 2021")
|
||||||
|
|
||||||
|
///////////////////////
|
||||||
|
// Working with the dynamic type
|
||||||
|
// ref: https://docs.microsoft.com/azure/data-explorer/kusto/query/scalar-data-types/dynamic
|
||||||
|
///////////////////////
|
||||||
|
|
||||||
|
// Dynamic is an object oriented format for storing structured data.
|
||||||
|
// Dynamics are usually converted from JSON strings using either todynamic() or parse_json() (same function)
|
||||||
|
// You can also convert XML into a dynamic by using parse_xml()
|
||||||
|
|
||||||
|
let JsonString = '{"hello": 1337, "world": ["wibble","wobble","wubble"]}';
|
||||||
|
print todynamic(JsonString)
|
||||||
|
|
||||||
|
// There are a number of ways you can interact with elements stored as dynamic() typed objects.
|
||||||
|
// Interacting with child elements
|
||||||
|
// - Column.Child
|
||||||
|
// - Column[“Child”]
|
||||||
|
|
||||||
|
let JsonString = '{"hello": 1337, "world": ["wibble","wobble","wubble"]}';
|
||||||
|
print x = todynamic(JsonString)
|
||||||
|
| extend hello = x.hello, world = x["world"]
|
||||||
|
|
||||||
|
// Interacting with lists \ arrays
|
||||||
|
// Column[ElementNumber]
|
||||||
|
|
||||||
|
let JsonString = '{"hello": 1337, "world": ["wibble","wobble","wubble"]}';
|
||||||
|
print x = todynamic(JsonString)
|
||||||
|
| extend hello = x.hello, world = x["world"]
|
||||||
|
| extend FirstElement = world[0], SecondElement = world[1]
|
||||||
|
|
||||||
|
// The information on the actor initiating the activities can be found in AccountObjectId and AccountDisplayName columns.
|
||||||
|
// To get the target account and the activities were performed we can extract information from RawEventData
|
||||||
|
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType == "AddedToGroup"
|
||||||
|
| take 50
|
||||||
|
| project-reorder AccountObjectId, AccountDisplayName, RawEventData
|
||||||
|
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType == "AddedToGroup"
|
||||||
|
| project Timestamp, Application, IPAddress, Actor = AccountDisplayName, AddedUser = RawEventData.TargetUserOrGroupName
|
||||||
|
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType == "AddedToGroup"
|
||||||
|
| project Timestamp, Application, IPAddress, Actor = AccountDisplayName, AddedUser = RawEventData["TargetUserOrGroupName"]
|
||||||
|
|
||||||
|
|
||||||
|
//////////////////////////////
|
||||||
|
// pack_array()
|
||||||
|
// ref: https://docs.microsoft.com/azure/data-explorer/kusto/query/packarrayfunction
|
||||||
|
// Creates a dynamic array
|
||||||
|
//////////////////////////////
|
||||||
|
|
||||||
|
// Lists, sets, and arrays in KQL are stored as dynamics and can be created
|
||||||
|
// with functions such as pack_array()
|
||||||
|
|
||||||
|
print pack_array('foo','bar','baz')
|
||||||
|
|
||||||
|
// Note that you cannot simply compare dynamic elements in KQL. To do this,
|
||||||
|
// convert them back to another type using functions such as tostring() or toint()
|
||||||
|
|
||||||
|
let JsonDynamic = todynamic('{"hello": 1337, "world": ["wibble","wobble","wubble"]}');
|
||||||
|
print tostring(JsonDynamic.hello) == tostring(JsonDynamic['hello'])
|
||||||
|
|
||||||
|
////////////////////////
|
||||||
|
// bag_unpack()
|
||||||
|
// ref: https://docs.microsoft.com/azure/data-explorer/kusto/query/bag-unpackplugin
|
||||||
|
// Automatically unpacks the first level of a dynamic to a table
|
||||||
|
////////////////////////
|
||||||
|
|
||||||
|
// Another option is to use bag_unpack() to turn JSON data directly into a table.
|
||||||
|
// Note that bag_unpack() only processes the first level of JSON. If you have
|
||||||
|
// multiple nested JSON elements you may need multiple calls to the function.
|
||||||
|
|
||||||
|
let JsonDynamic = todynamic('{"hello": 1337, "world": ["wibble","wobble","wubble"]}');
|
||||||
|
print x = JsonDynamic
|
||||||
|
| evaluate bag_unpack(x)
|
||||||
|
|
||||||
|
/////////////////////////////
|
||||||
|
// mv-expand
|
||||||
|
// ref: https://docs.microsoft.com/azure/data-explorer/kusto/query/mvexpandoperator
|
||||||
|
// Multiplies elements in a dynamic array across a tabular dataset
|
||||||
|
/////////////////////////////
|
||||||
|
|
||||||
|
|
||||||
|
// You can also use functions such as mv-expand to parse elements in a dynamic
|
||||||
|
// across a table. This is very handy for efficiently analyzing lists of
|
||||||
|
// elements at scale
|
||||||
|
|
||||||
|
// Let’s put bag_unpack() and mv-expand together.
|
||||||
|
|
||||||
|
let JsonDynamic = todynamic('{"hello": 1337, "world": ["wibble","wobble","wubble"]}');
|
||||||
|
print x = JsonDynamic
|
||||||
|
| evaluate bag_unpack(x)
|
||||||
|
| mv-expand world
|
||||||
|
|
||||||
|
// With mv-expand we can extract the Group the user was added to.
|
||||||
|
// The easiest is to extract it from ActivityObjects column
|
||||||
|
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType == "AddedToGroup"
|
||||||
|
| mv-expand ActivityObjects
|
||||||
|
| where ActivityObjects['Type'] == ('Group')
|
||||||
|
| project Timestamp, Application, IPAddress, Actor = AccountDisplayName, AddedUser = RawEventData.TargetUserOrGroupName, Group = ActivityObjects.Name
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
|
||||||
|
//////////////////////////////
|
||||||
|
// startofday() function
|
||||||
|
// ref: https://docs.microsoft.com/azure/data-explorer/kusto/query/startofdayfunction
|
||||||
|
// Returns the time at the start of a day, with an optional time offset
|
||||||
|
//////////////////////////////
|
||||||
|
|
||||||
|
// KQL time filters and functions are UTC
|
||||||
|
// Returns the start of the day containing the date, shifted by an offset of days, if provided.
|
||||||
|
|
||||||
|
print startofday(datetime(2021-01-01 10:10:17))
|
||||||
|
|
||||||
|
IdentityInfo | where AccountUpn == 'meganb@seccxp.ninja'
|
||||||
|
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
AADSignInEventsBeta
|
||||||
|
| where AccountObjectId == 'eababd92-9dc7-40e3-9359-6c106522db19' and Timestamp >= timeToSearch
|
||||||
|
| distinct Application, ResourceDisplayName, Country, City, IPAddress, DeviceName, DeviceTrustType, OSPlatform, IsManaged, IsCompliant, AuthenticationRequirement, RiskState, UserAgent, ClientAppUsed
|
||||||
|
|
||||||
|
// Step 1: understand the performed actions
|
||||||
|
let accountId = 'eababd92-9dc7-40e3-9359-6c106522db19';
|
||||||
|
let locations = pack_array('SG', 'DE', 'IE', 'AL', 'UK');
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
CloudAppEvents
|
||||||
|
| where AccountObjectId == accountId and CountryCode in (locations) and Timestamp >= timeToSearch
|
||||||
|
| summarize by ActionType, CountryCode, AccountObjectId
|
||||||
|
| sort by ActionType asc
|
||||||
|
|
||||||
|
// Step 2: review the accessed emails
|
||||||
|
let accountId = 'eababd92-9dc7-40e3-9359-6c106522db19';
|
||||||
|
let locations = pack_array('SG', 'DE', 'IE', 'AL', 'UK');
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType == 'MailItemsAccessed' and CountryCode in (locations) and AccountObjectId == accountId and Timestamp >= timeToSearch
|
||||||
|
| mv-expand todynamic(RawEventData.Folders)
|
||||||
|
| extend Path = todynamic(RawEventData_Folders.Path), SessionId = tostring(RawEventData.SessionId)
|
||||||
|
| mv-expand todynamic(RawEventData_Folders.FolderItems)
|
||||||
|
| project SessionId, Timestamp, AccountObjectId, DeviceType, CountryCode, City, IPAddress, UserAgent, Path, Message = tostring(RawEventData_Folders_FolderItems.InternetMessageId)
|
||||||
|
| join kind=leftouter (
|
||||||
|
EmailEvents
|
||||||
|
| where RecipientObjectId == accountId
|
||||||
|
| project Subject, RecipientEmailAddress, SenderMailFromAddress, DeliveryLocation, ThreatTypes, AttachmentCount, UrlCount, InternetMessageId
|
||||||
|
)
|
||||||
|
on $left.Message == $right.InternetMessageId
|
||||||
|
| sort by Timestamp desc
|
||||||
|
|
||||||
|
|
||||||
|
// BONUS: get message details using Graph:
|
||||||
|
// https://graph.microsoft.com/v1.0/users/meganb@seccxp.ninja/messages?filter=internetMessageId eq '<b4acafd9-d086-4de0-8deb-c83118dae907@az.centralus.production.microsoft.com>'&select=subject,from,hasAttachments
|
||||||
|
|
||||||
|
// Step 3: review the accessed FolderItems
|
||||||
|
let accountId = 'eababd92-9dc7-40e3-9359-6c106522db19';
|
||||||
|
let locations = pack_array('SG', 'DE', 'IE', 'AL', 'UK');
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType == 'FilePreviewed' or ActionType == 'FileDownloaded' and CountryCode in (locations) and AccountObjectId == accountId and Timestamp >= timeToSearch
|
||||||
|
| project Timestamp, CountryCode, IPAddress, ISP, UserAgent, Application, ActivityObjects, AccountObjectId
|
||||||
|
| mv-expand ActivityObjects
|
||||||
|
| where ActivityObjects['Type'] in ('File', 'Folder') and ActivityObjects['Role'] == 'Target object'
|
||||||
|
| evaluate bag_unpack(ActivityObjects)
|
||||||
|
|
||||||
|
|
||||||
|
// Step 4: review deleted emails
|
||||||
|
let accountId = 'eababd92-9dc7-40e3-9359-6c106522db19';
|
||||||
|
let locations = pack_array('SG', 'DE', 'IE', 'AL', 'UK');
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType in~ ('MoveToDeletedItems', 'SoftDelete', 'HardDelete') and CountryCode in (locations) and AccountObjectId == accountId and Timestamp >= timeToSearch
|
||||||
|
| mv-expand ActivityObjects
|
||||||
|
| where ActivityObjects['Type'] in ('Email', 'Folder')
|
||||||
|
| evaluate bag_unpack(ActivityObjects)
|
||||||
|
| distinct Timestamp, AccountObjectId, ActionType, CountryCode, IPAddress, Type, Name, Id
|
||||||
|
| sort by Timestamp desc
|
||||||
|
|
||||||
|
|
||||||
|
// Step 5: review the created inbox rules
|
||||||
|
let accountId = 'eababd92-9dc7-40e3-9359-6c106522db19';
|
||||||
|
let locations = pack_array('SG', 'DE', 'IE', 'AL', 'UK');
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType contains_cs 'InboxRule' and CountryCode in (locations)
|
||||||
|
| extend RuleParameters = RawEventData.Parameters
|
||||||
|
| project Timestamp, CountryCode, IPAddress, ISP, ActionType, ObjectName, RuleParameters
|
||||||
|
| sort by Timestamp desc
|
||||||
|
|
||||||
|
// Step 6: identify potential other victims
|
||||||
|
let accountId = 'eababd92-9dc7-40e3-9359-6c106522db19';
|
||||||
|
let locations = pack_array('SG', 'DE', 'IE', 'AL', 'UK');
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
let ips = (
|
||||||
|
CloudAppEvents
|
||||||
|
| where CountryCode in (locations)
|
||||||
|
| distinct IPAddress, AccountObjectId
|
||||||
|
);
|
||||||
|
ips
|
||||||
|
| join (CloudAppEvents | project ActivityIP = IPAddress, UserId = AccountObjectId) on $left.IPAddress == $right.ActivityIP
|
||||||
|
| distinct UserId
|
||||||
|
| join IdentityInfo on $left.UserId == $right.AccountObjectId
|
||||||
|
| distinct AccountDisplayName, AccountUpn, Department, Country, City, AccountObjectId
|
||||||
|
|
||||||
|
// Bonus: identify details sent by the malicious actorIdentityInfo
|
||||||
|
let accountId = 'eababd92-9dc7-40e3-9359-6c106522db19';
|
||||||
|
let timeToSearch = startofday(datetime('2021-05-04'));
|
||||||
|
CloudAppEvents
|
||||||
|
| where ActionType =~ 'send' and AccountObjectId == accountId // apply the right filter
|
||||||
|
| extend rawData = todynamic(RawEventData)
|
||||||
|
| extend UserKey = rawData.UserKey, MessageId = tostring(rawData.Item.InternetMessageId), Subject = rawData.Item.Subject, Attachments = rawData.Item.Attachments
|
||||||
|
| join (
|
||||||
|
EmailEvents
|
||||||
|
)
|
||||||
|
on $left.MessageId == $right.InternetMessageId
|
||||||
|
| sort by Timestamp desc
|
||||||
|
| project UserKey, Timestamp, AccountObjectId, AccountDisplayName, DeviceType, CountryCode, City, ISP, IPAddress, SenderIPv4, SenderIPv6, UserAgent, Subject, InternetMessageId, Attachments, RecipientEmailAddress, SenderMailFromAddress, DeliveryLocation, ThreatTypes, ConfidenceLevel
|
||||||
|
, AttachmentCount, UrlCount, MessageId
|
||||||
|
|
@ -0,0 +1,366 @@
|
||||||
|
print Topic = "l33tSpeak: Advanced hunting in Microsoft 365 Defender", Presenters = "Michael Melone, Tali Ash", Company = "Microsoft", Date = todatetime("17 NOV 2020")
|
||||||
|
|
||||||
|
// -------------------------
|
||||||
|
// Topic 1: Advanced Hunting, KQL, and performance
|
||||||
|
// Advanced hunting query best practices in Microsoft Threat Protection - Microsoft 365 security | Microsoft Docs
|
||||||
|
// -------------------------
|
||||||
|
|
||||||
|
// Advanced Hunting is built on Azure Data Explorer, which is a Write Once Read Many (WORM)
|
||||||
|
// technology. When you write a query against Advanced Hunting:
|
||||||
|
// - Data is based on recent activity (usually delayed just a few minutes)
|
||||||
|
// - There is never any impact to the endpoint
|
||||||
|
// - Queries may be throttled or limited based on how they're written to limit impact to other sessions
|
||||||
|
// Recommended documentation https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-best-practices?view=o365-worldwide
|
||||||
|
|
||||||
|
// #1 best way to improve query performance: reduce timeframe
|
||||||
|
// Visualized filter at the top right
|
||||||
|
|
||||||
|
IdentityInfo
|
||||||
|
| where Department == "Finance"
|
||||||
|
| distinct AccountObjectId
|
||||||
|
| join IdentityLogonEvents on AccountObjectId
|
||||||
|
| where Application == "Office 365"
|
||||||
|
|
||||||
|
// Filter timestamp in the query using “ago”
|
||||||
|
IdentityInfo
|
||||||
|
| where Department == "Finance"
|
||||||
|
| distinct AccountObjectId
|
||||||
|
| join (IdentityLogonEvents | where Timestamp > ago(10d)) on AccountObjectId
|
||||||
|
| where Application == "Office 365"
|
||||||
|
|
||||||
|
// Filter timestamp in the query using “between”
|
||||||
|
let selectedTimestamp = datetime(2020-11-12T19:35:03.9859771Z);
|
||||||
|
IdentityInfo
|
||||||
|
| where Department == "Finance"
|
||||||
|
| distinct AccountObjectId
|
||||||
|
| join (IdentityLogonEvents | where Timestamp between ((selectedTimestamp - 2h) .. (selectedTimestamp + 2h))) on AccountObjectId
|
||||||
|
| where Application == "Office 365"
|
||||||
|
|
||||||
|
AlertInfo
|
||||||
|
| where AlertId == "caC27D7C90-E9E7-3207-9FF8-94335F0E27D3"
|
||||||
|
|
||||||
|
| project AlertTime = Timestamp , Title, Severity, AlertId
|
||||||
|
| join AlertEvidence on AlertId
|
||||||
|
| where EntityType == "User"
|
||||||
|
| project AccountObjectId , AlertTime
|
||||||
|
| join IdentityLogonEvents on AccountObjectId
|
||||||
|
| where Timestamp between ((AlertTime - 1h) ..(AlertTime + 1h))
|
||||||
|
|
||||||
|
// Has beats contains: When looking for full tokens, has works better, since it doesn't look for substrings.
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where RemoteUrl contains "team"
|
||||||
|
| take 50
|
||||||
|
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where RemoteUrl has "team" //only teams will work as it is the full token
|
||||||
|
| take 50
|
||||||
|
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where RemoteUrl contains "microsoft.com"
|
||||||
|
| take 50
|
||||||
|
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where RemoteUrl has "microsoft.com"
|
||||||
|
| take 50
|
||||||
|
|
||||||
|
// Use case-sensitive operators when possible.
|
||||||
|
// Names of case-sensitive string operators, such as has_cs and contains_cs, generally end with _cs.
|
||||||
|
DeviceNetworkEvents
|
||||||
|
| where RemoteUrl has_cs "microsoft.com"
|
||||||
|
| take 50
|
||||||
|
|
||||||
|
// Use == and not =~, Use in and not in~
|
||||||
|
// Get latest information on user/device
|
||||||
|
DeviceInfo
|
||||||
|
| where DeviceName == "alexw-pc" and isnotempty(OSPlatform)
|
||||||
|
| summarize arg_max(Timestamp, *) by DeviceId
|
||||||
|
|
||||||
|
IdentityInfo
|
||||||
|
| where Department == "Finance"
|
||||||
|
| extend ingestionTime = ingestion_time()
|
||||||
|
| summarize arg_max(ingestionTime, *) by AccountObjectId
|
||||||
|
|
||||||
|
// Optimize the join operator
|
||||||
|
// - If you are using a join, try to reduce the dataset before joining to limit the join size
|
||||||
|
// - If you use too many resources you may be put in 'time out' for a bit
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| project NetworkMessageId, Subject, Timestamp, SenderFromAddress , SenderIPv4 , RecipientEmailAddress , AttachmentCount
|
||||||
|
| join kind=leftouter(EmailAttachmentInfo
|
||||||
|
| project NetworkMessageId,FileName, FileType, ThreatTypes, SHA256, RecipientEmailAddress )
|
||||||
|
on NetworkMessageId
|
||||||
|
|
||||||
|
// filter the left table as much as you can
|
||||||
|
// Key for the join should be accurate as possible
|
||||||
|
EmailEvents
|
||||||
|
| where AttachmentCount > 0
|
||||||
|
|project NetworkMessageId, Subject, Timestamp, SenderFromAddress , SenderIPv4 , RecipientEmailAddress , AttachmentCount
|
||||||
|
| join kind=inner (EmailAttachmentInfo
|
||||||
|
| project NetworkMessageId,FileName, FileType, ThreatTypes, SHA256, RecipientEmailAddress )
|
||||||
|
on NetworkMessageId, RecipientEmailAddress
|
||||||
|
|
||||||
|
// Smaller table on the left side, with kind = inner, as default join (innerunique)
|
||||||
|
// will remove left side duplications, so if a single email has more than one attachments we will miss it
|
||||||
|
EmailAttachmentInfo
|
||||||
|
| project NetworkMessageId, FileName, FileType, ThreatTypes, SHA256, RecipientEmailAddress
|
||||||
|
| join kind=inner
|
||||||
|
(EmailEvents
|
||||||
|
| where AttachmentCount > 0
|
||||||
|
|project NetworkMessageId, Subject, Timestamp, SenderFromAddress , SenderIPv4 , RecipientEmailAddress , AttachmentCount)
|
||||||
|
on NetworkMessageId, RecipientEmailAddress
|
||||||
|
|
||||||
|
// Check for specific alerts
|
||||||
|
AlertInfo
|
||||||
|
| join AlertEvidence on AlertId
|
||||||
|
| where EntityType == "Machine"
|
||||||
|
// Attempts to clear security event logs.
|
||||||
|
| where Title in("Event log was cleared",
|
||||||
|
// List alerts flagging attempts to delete backup files.
|
||||||
|
"File backups were deleted",
|
||||||
|
// Potential Cobalt Strike activity - Note that other threat activity can also
|
||||||
|
// trigger alerts for suspicious decoded content
|
||||||
|
"Suspicious decoded content",
|
||||||
|
// Cobalt Strike activity
|
||||||
|
"\'Atosev\' malware was detected",
|
||||||
|
"\'Ploty\' malware was detected",
|
||||||
|
"\'Bynoco\' malware was detected")
|
||||||
|
| extend AlertTime = Timestamp
|
||||||
|
| distinct DeviceName, AlertTime, AlertId, Title
|
||||||
|
| join DeviceLogonEvents on $left.DeviceName == $right.DeviceName
|
||||||
|
// Creating 10 day Window surrounding alert activity
|
||||||
|
| where Timestamp < AlertTime +5d and Timestamp > AlertTime - 5d
|
||||||
|
// Projecting specific columns
|
||||||
|
| project Title, DeviceName, DeviceId, Timestamp, LogonType, AccountDomain,
|
||||||
|
AccountName, AccountSid, AlertTime, AlertId, RemoteIP, RemoteDeviceName
|
||||||
|
|
||||||
|
|
||||||
|
// Check for specific alerts
|
||||||
|
AlertInfo
|
||||||
|
// Attempts to clear security event logs.
|
||||||
|
| where Title in("Event log was cleared",
|
||||||
|
// List alerts flagging attempts to delete backup files.
|
||||||
|
"File backups were deleted",
|
||||||
|
// Potential Cobalt Strike activity - Note that other threat activity can also
|
||||||
|
// trigger alerts for suspicious decoded content
|
||||||
|
"Suspicious decoded content",
|
||||||
|
// Cobalt Strike activity
|
||||||
|
"\'Atosev\' malware was detected",
|
||||||
|
"\'Ploty\' malware was detected",
|
||||||
|
"\'Bynoco\' malware was detected")
|
||||||
|
| extend AlertTime = Timestamp
|
||||||
|
| join AlertEvidence on AlertId
|
||||||
|
| where EntityType == "Machine"
|
||||||
|
| distinct DeviceName, AlertTime, AlertId, Title
|
||||||
|
| join DeviceLogonEvents on $left.DeviceName == $right.DeviceName
|
||||||
|
// Creating 10 day Window surrounding alert activity
|
||||||
|
| where Timestamp < AlertTime +5d and Timestamp > AlertTime - 5d
|
||||||
|
// Projecting specific columns
|
||||||
|
| project Title, DeviceName, DeviceId, Timestamp, LogonType, AccountDomain,
|
||||||
|
AccountName, AccountSid, AlertTime, AlertId, RemoteIP, RemoteDeviceName
|
||||||
|
|
||||||
|
// - Queries are limited to 10k results through the web UI, 100k results via API
|
||||||
|
|
||||||
|
|
||||||
|
// -----------------------------
|
||||||
|
// Topic 2: Ransomware tips / recommendations (3 examples)
|
||||||
|
|
||||||
|
// Ransomware is a very real challenge in today’s enterprise.
|
||||||
|
|
||||||
|
// -----------------------------
|
||||||
|
|
||||||
|
// Defender for Endpoint provides a bunch of different types of alerts for
|
||||||
|
// known ransomware and ransomware-like behavior
|
||||||
|
// To accomplish these tests, I explicitly excluded the folder where I placed the malware,
|
||||||
|
// disable automatic response by AutoIR, and ensured EDR in block mode was disabled.
|
||||||
|
|
||||||
|
AlertEvidence
|
||||||
|
| where DeviceId == 'eb610cc67fe0b99300a076324f5f4cd409324872'
|
||||||
|
| join kind=rightsemi AlertInfo on AlertId
|
||||||
|
| order by Timestamp asc
|
||||||
|
|
||||||
|
// Much of the ransomware that makes the news is what we refer to as human operated
|
||||||
|
// ransomware. This differs from other forms of ransomware in that it typically begins
|
||||||
|
// with an attacker compromising a vulnerability, performing credential theft until they
|
||||||
|
// attain enough authorization to deploy ransomware broadly, then doing so.
|
||||||
|
|
||||||
|
// …in other words, it is a targeted attack and should be treated as such. To track and
|
||||||
|
// eliminate this activity you will want to use the ABC method discussed in Tracking the
|
||||||
|
// Adversary episode 4. Its not about the malware, but rather the persistence mechanisms
|
||||||
|
// implemented by the attacker and any credentials they control.
|
||||||
|
|
||||||
|
// On the ransomware front, we can try to detect ransomware-like behaviors, for example:
|
||||||
|
// src: https://github.com/microsoft/Microsoft-365-Defender-Hunting-Queries/blob/master/Execution/Possible%20Ransomware%20Related%20Destruction%20Activity.md
|
||||||
|
|
||||||
|
DeviceProcessEvents
|
||||||
|
| where Timestamp > ago(7d)
|
||||||
|
| where (FileName =~ 'vssadmin.exe' and ProcessCommandLine has "delete shadows" and ProcessCommandLine has "/all" and ProcessCommandLine has "/quiet" ) // Clearing shadow copies
|
||||||
|
or (FileName =~ 'cipher.exe' and ProcessCommandLine contains "/w") // Wiping drive free space
|
||||||
|
or (FileName =~ 'schtasks.exe' and ProcessCommandLine has "/change" and ProcessCommandLine has @"\Microsoft\Windows\SystemRestore\SR" and ProcessCommandLine has "/disable") // Disabling system restore task
|
||||||
|
or (FileName =~ 'fsutil.exe' and ProcessCommandLine has "usn" and ProcessCommandLine has "deletejournal" and ProcessCommandLine has "/d") // Deleting USN journal
|
||||||
|
or (FileName =~ 'icacls.exe' and ProcessCommandLine has @'"C:\*"' and ProcessCommandLine contains '/grant Everyone:F') // Attempts to re-ACL all files on the C drive to give everyone full control
|
||||||
|
or (FileName =~ 'powershell.exe' and (
|
||||||
|
ProcessCommandLine matches regex @'\s+-((?i)encod?e?d?c?o?m?m?a?n?d?|e|en|enc|ec)\s+' and replace(@'\x00','', base64_decode_tostring(extract("[A-Za-z0-9+/]{50,}[=]{0,2}",0 , ProcessCommandLine))) matches regex @".*(Win32_Shadowcopy).*(.Delete\(\)).*"
|
||||||
|
) or ProcessCommandLine matches regex @".*(Win32_Shadowcopy).*(.Delete\(\)).*"
|
||||||
|
) // This query looks for PowerShell-based commands used to delete shadow copies
|
||||||
|
|
||||||
|
|
||||||
|
// -----------------------------
|
||||||
|
// Topic 3: Handling JSON and the dynamic type
|
||||||
|
// -----------------------------
|
||||||
|
|
||||||
|
// Many tables in Advanced Hunting are JSON strings. While you can parse these as strings,
|
||||||
|
// turning them into dynamic type columns is usually a lot more effective.
|
||||||
|
DeviceEvents
|
||||||
|
| summarize arg_max(Timestamp, *) by ActionType
|
||||||
|
| project-reorder ActionType, AdditionalFields
|
||||||
|
|
||||||
|
// Let's look at the AdditionalFields column of a PnpDeviceConnected event
|
||||||
|
DeviceEvents
|
||||||
|
| where ActionType == 'PnpDeviceConnected'
|
||||||
|
| take 10
|
||||||
|
| project-reorder AdditionalFields
|
||||||
|
|
||||||
|
|
||||||
|
// Imagine we wanted to audit which plug and play devices were used on the network. To
|
||||||
|
// accomplish this we need to turn this string into (a much more useful) dynamic.
|
||||||
|
// From there we can access individual properties of the object using dotted notation.
|
||||||
|
DeviceEvents
|
||||||
|
| where ActionType == 'PnpDeviceConnected'
|
||||||
|
| project ATDynamic = parse_json(AdditionalFields)
|
||||||
|
| extend ClassName = ATDynamic.ClassName, ClassId = ATDynamic.ClassId, DeviceId = ATDynamic.DeviceId, DeviceDescription = ATDynamic.DeviceDescription
|
||||||
|
| project-reorder ClassName, ClassId, DeviceDescription
|
||||||
|
| take 10
|
||||||
|
|
||||||
|
|
||||||
|
// ...but that takes forever to type, right? Enter bag_unpack()!
|
||||||
|
DeviceEvents
|
||||||
|
| where ActionType == 'PnpDeviceConnected'
|
||||||
|
| take 100
|
||||||
|
| project ATDynamic = parse_json(AdditionalFields)
|
||||||
|
| evaluate bag_unpack(ATDynamic)
|
||||||
|
|
||||||
|
|
||||||
|
// Lists can be indexed too. For example, let's take a look at signature versions reported
|
||||||
|
// by Threat and Vulnerability Management
|
||||||
|
|
||||||
|
|
||||||
|
DeviceTvmSecureConfigurationAssessment
|
||||||
|
| where ConfigurationId == 'scid-2011'
|
||||||
|
| take 100
|
||||||
|
| project-reorder Context
|
||||||
|
|
||||||
|
|
||||||
|
// Here we have a list within a list. The inner list has three values (for machines running
|
||||||
|
// Defender Antimalware):
|
||||||
|
// - Signature version
|
||||||
|
// - Engine version
|
||||||
|
// - Update date
|
||||||
|
// To get them out:
|
||||||
|
|
||||||
|
|
||||||
|
DeviceTvmSecureConfigurationAssessment
|
||||||
|
| where ConfigurationId == 'scid-2011'
|
||||||
|
| take 100
|
||||||
|
| extend x = todynamic(Context)
|
||||||
|
| project DeviceId, DeviceName, SignatureVersion = x[0][0], EngineVerision = x[0][1], UpdateDate = x[0][2]
|
||||||
|
// Now let's say you wanted to generate a report that has a count of the number of different
|
||||||
|
// plug and play devices added per machine, but also lists out all of the ClassId's from
|
||||||
|
// each of the plug and play devices.
|
||||||
|
|
||||||
|
|
||||||
|
// First, we need to change that project to an extend so that we can get the DeviceId back.
|
||||||
|
|
||||||
|
|
||||||
|
DeviceEvents
|
||||||
|
| where ActionType == 'PnpDeviceConnected'
|
||||||
|
| take 100
|
||||||
|
| extend ATDynamic = parse_json(AdditionalFields)
|
||||||
|
| evaluate bag_unpack(ATDynamic)
|
||||||
|
|
||||||
|
|
||||||
|
// Error! It looks like we have a field in the Additional Fields column called DeviceId already.
|
||||||
|
// To work around this we can use the second parameter of bag_unpack() to assign a prefix.
|
||||||
|
|
||||||
|
|
||||||
|
DeviceEvents
|
||||||
|
| where ActionType == 'PnpDeviceConnected'
|
||||||
|
| take 100
|
||||||
|
| extend ATDynamic = parse_json(AdditionalFields)
|
||||||
|
| evaluate bag_unpack(ATDynamic, 'AdditionalFields_')
|
||||||
|
|
||||||
|
|
||||||
|
// Perfect. Now we need to turn the ClassId into a string so that we can use it in summary
|
||||||
|
// operations, such as dcount(). After that, we will use a new operator, make_set(), to create
|
||||||
|
// a new deduplicated list of ClassIds associated with the device.
|
||||||
|
|
||||||
|
|
||||||
|
DeviceEvents
|
||||||
|
| where ActionType == 'PnpDeviceConnected'
|
||||||
|
| take 100
|
||||||
|
| extend ATDynamic = parse_json(AdditionalFields)
|
||||||
|
| evaluate bag_unpack(ATDynamic, 'AdditionalFields_')
|
||||||
|
| extend ClassId = tostring(AdditionalFields_ClassId)
|
||||||
|
|
||||||
|
| summarize PnPEvents = count(), DifferentPnPDevices = dcount(ClassId), makeset(ClassId), (LastPnPEvent, DeviceName) = arg_max(Timestamp, DeviceName) by DeviceId
|
||||||
|
// Now what if we want to make our own JSON objects with properties values? That's where the pack*() series of
|
||||||
|
// functions come in.
|
||||||
|
|
||||||
|
|
||||||
|
// pack
|
||||||
|
// https://docs.microsoft.com/azure/data-explorer/kusto/query/packfunction
|
||||||
|
// The pack function takes specific keys and values and packs them into a JSON object. Parameters are
|
||||||
|
// provided in pairs with the first pair being the key and the second being the value.
|
||||||
|
|
||||||
|
|
||||||
|
print x = pack('foo','bar','wibble','wobble')
|
||||||
|
|
||||||
|
|
||||||
|
print x = pack('foo','bar','wibble','wobble')
|
||||||
|
| evaluate bag_unpack(x)
|
||||||
|
|
||||||
|
|
||||||
|
// ..but let's say instead you wanted to package your query results as a JSON object.
|
||||||
|
// pack_all() is quite handy for this.
|
||||||
|
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| take 100
|
||||||
|
| extend packed = pack_all()
|
||||||
|
| project-reorder packed
|
||||||
|
|
||||||
|
|
||||||
|
// This is all well and good, but you might need to aggregate all of these individual rows
|
||||||
|
// into a single row, right? This is where our aggrigation functions come in.
|
||||||
|
// makelist() is like makeset(), but without deduplication. Let's use makelist() to create
|
||||||
|
// a list of e-mails in JSON format based on senders and their domain.
|
||||||
|
|
||||||
|
|
||||||
|
EmailEvents
|
||||||
|
| take 100
|
||||||
|
| project SenderFromDomain, SenderFromAddress, packed = pack_all()
|
||||||
|
| summarize makelist(packed) by SenderFromDomain, SenderFromAddress
|
||||||
|
|
||||||
|
|
||||||
|
// -----------------------------
|
||||||
|
// Topic 4: externaldata operator
|
||||||
|
// externaldata operator allows importing data from externally stored files and use it inside a query.
|
||||||
|
// externaldata operator - https://docs.microsoft.com/azure/data-explorer/kusto/query/externaldata-operator
|
||||||
|
// In our docs: https://docs.microsoft.com/microsoft-365/security/mtp/advanced-hunting-best-practices?view=o365-worldwide#ingest-data-from-external-sources
|
||||||
|
// -----------------------------
|
||||||
|
|
||||||
|
EmailAttachmentInfo
|
||||||
|
| where SHA256 in (externaldata(TimeGenerated:datetime, SHA256:string)
|
||||||
|
[@"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Sample%20Data/Feeds/Microsoft.Covid19.Indicators.csv"]
|
||||||
|
with (format="csv"))
|
||||||
|
|
||||||
|
// See if any process created a file matching a hash on the list
|
||||||
|
let covidIndicators = (externaldata(TimeGenerated:datetime, FileHashValue:string, FileHashType: string )
|
||||||
|
[@"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Sample%20Data/Feeds/Microsoft.Covid19.Indicators.csv"]
|
||||||
|
with (format="csv"))
|
||||||
|
| where FileHashType == 'sha256'; //and TimeGenerated > ago(1d);
|
||||||
|
covidIndicators
|
||||||
|
| join (DeviceFileEvents
|
||||||
|
| where ActionType == 'FileCreated'
|
||||||
|
| take 100) on $left.FileHashValue == $right.SHA256
|
||||||
|
|
||||||
|
|
||||||
Загрузка…
Ссылка в новой задаче