146 строки
10 KiB
YAML
146 строки
10 KiB
YAML
id: 24e66452-2aaa-455f-b0c6-a0d8216bbe79
|
|
name: Entropy for Processes for a given Host (Normalized Process Events)
|
|
description: |
|
|
'Entropy calculation used to help identify Hosts where they have a high variety of processes(a high entropy process list on a given Host over time).
|
|
This helps us identify rare processes on a given Host. Rare here means a process shows up on the Host relatively few times in the the last 7days.
|
|
The Weight is calculated based on the Entropy, Process Count and Distinct Hosts with that Process. The lower the Weight/ProcessEntropy the, more interesting.
|
|
The Weight calculation increases the Weight if the process executes more than once on the Host or has executed on more than 1 Hosts.
|
|
In general, this should identify processes on a Host that are rare and rare for the environment.
|
|
References: https://medium.com/udacity/shannon-entropy-information-gain-and-picking-balls-from-buckets-5810d35d54b4
|
|
https://en.wiktionary.org/wiki/Shannon_entropy'
|
|
requiredDataConnectors: []
|
|
tactics:
|
|
- Execution
|
|
query: |
|
|
// exclude when over # of machines have the process
|
|
let excludeThreshold = 10;
|
|
// exclude when more than percent (default 10%)
|
|
let ratioHighCount = 0.1;
|
|
// exclude when less than percent (default 3%)
|
|
let ratioMidCount = 0.03;
|
|
// Process count limit in one day per machine, perf improvement (default every 20 minutes for 24 hours - 3*24 = 72)
|
|
let procLimit = 3*24;
|
|
// Decrease possibility of hitting memory limit by removing high process count items across all machines (default every 10 minutes for 24 hours - 6*24 = 144)
|
|
let maxLimit = 6*24;
|
|
let removeHigh = imProcessCreate
|
|
| where TimeGenerated >= ago(1d)
|
|
| extend TargetProcessFilePath = TargetProcessName
|
|
| summarize count() by TargetProcessFilePath = tolower(TargetProcessFilePath) | where count_ > maxLimit
|
|
| summarize make_set(TargetProcessFilePath, 200);
|
|
let SecEvents = imProcessCreate
|
|
| where TimeGenerated >= ago(1d)
|
|
| extend TargetProcessFilePath = TargetProcessName
|
|
| extend ActingProcessFileName = ActingProcessName
|
|
| extend TargetProcessFileName = TargetProcessFileProduct
|
|
| where tolower(TargetProcessFilePath) !in~ (removeHigh)
|
|
// removing common items that may still show up in small environments, add here if you have additional exclusions
|
|
| where TargetProcessFilePath !has ':\\Windows\\System32\\conhost.exe' and ActingProcessFileName !has ':\\Windows\\System32\\conhost.exe'
|
|
| where ActingProcessFileName !has ':\\Windows\\System32\\wuauclt.exe' and TargetProcessFilePath !has':\\Windows\\System32\\wuauclt.exe' and TargetProcessFilePath !startswith 'C:\\Windows\\SoftwareDistribution\\Download\\Install\\AM_Delta_Patch_'
|
|
| where ActingProcessFileName !has ':\\WindowsAzure\\GuestAgent_' and TargetProcessFilePath !has ':\\WindowsAzure\\GuestAgent_'
|
|
| where ActingProcessFileName !has ':\\WindowsAzure\\WindowsAzureNetAgent_' and TargetProcessFilePath !has ':\\WindowsAzure\\WindowsAzureNetAgent_'
|
|
| where ActingProcessFileName !has ':\\ProgramData\\Microsoft\\Windows Defender\\platform\\' and TargetProcessFilePath !has "\\Windows Defender Advanced Threat Protection\\SenseCncProxy.exe" and TargetProcessFilePath !has "\\Windows Defender Advanced Threat Protection\\SenseIR.exe.exe"
|
|
| where TargetProcessFilePath !has ':\\ProgramData\\Microsoft\\Windows Defender\\platform\\'
|
|
| where TargetProcessFilePath !has ':\\Windows\\Microsoft.NET\\Framework' and not(TargetProcessFilePath endswith '\\ngentask.exe' or TargetProcessFilePath endswith '\\ngen.exe')
|
|
| where ActingProcessFileName !has ':\\Windows\\Microsoft.NET\\Framework' and not(ActingProcessFileName endswith '\\ngentask.exe' or ActingProcessFileName endswith '\\ngen.exe')
|
|
| where TargetProcessFilePath !has ':\\Windows\\System32\\taskhostw.exe' and ActingProcessFileName !has ':\\Windows\\System32\\taskhostw.exe'
|
|
| where ActingProcessFileName !has ':\\Windows\\SoftwareDistribution\\Download\\Install\\' and not(TargetProcessFilePath endswith '\\MpSigStub.exe')
|
|
| where TargetProcessFilePath !has ':\\Program Files\\Microsoft Monitoring Agent\\Agent\\Health Service State\\' and ActingProcessFileName !has ':\\Program Files\\Microsoft Monitoring Agent\\Agent\\MonitoringHost.exe'
|
|
| where TargetProcessFilePath !has ':\\Windows\\servicing\\trustedinstaller.exe'
|
|
| where ActingProcessFileName !has ':\\Program Files\\Microsoft Dependency Agent\\bin\\MicrosoftDependencyAgent.exe'
|
|
| where ActingProcessFileName !has ':\\Program Files (x86)\\Microsoft\\EdgeUpdate\\MicrosoftEdgeUpdate.exe'
|
|
| project TimeGenerated, EventOriginalUid, DvcHostname, ActorUserId, User, TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, _ResourceId, DvcId, EventVendor, EventProduct;
|
|
let Exclude = SecEvents
|
|
| summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), ExcludeCompCount = dcount(DvcHostname), ExcludeProcCount = count() by TargetProcessFileName
|
|
// Removing general limit for noise in one day
|
|
| extend timediff = iff(datetime_diff('day', EndTime, StartTime) > 0, datetime_diff('day', EndTime, StartTime), 1)
|
|
// Default exclude of 48 (2 per hour) or more executions in 24 hours on a given machine
|
|
| where ExcludeProcCount > procLimit*timediff
|
|
// Removing noisy processes for an environment, adjust as needed
|
|
| extend compRatio = ExcludeCompCount/toreal(ExcludeProcCount)
|
|
| where compRatio == 0 or (ExcludeCompCount > excludeThreshold and compRatio < ratioHighCount) or (ExcludeCompCount between (2 .. excludeThreshold) and compRatio < ratioMidCount);
|
|
let AllSecEvents =
|
|
SecEvents | project DvcHostname, TargetProcessFileName , EventVendor, EventProduct
|
|
| join kind= leftanti (
|
|
SecEvents
|
|
// Removing general limit for noise in one day
|
|
| summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), procCount = count() by DvcHostname, TargetProcessFileName
|
|
| extend timediff = iff(datetime_diff('day', EndTime, StartTime) > 0, datetime_diff('day', EndTime, StartTime), 1)
|
|
// Default exclude 48 (2 per hour) or more executions in 24 hours on a given machine to remove them from overall comparison list
|
|
| where procCount > procLimit*timediff
|
|
) on DvcHostname, TargetProcessFileName
|
|
| project DvcHostname, TargetProcessFileName , EventVendor, EventProduct;
|
|
// Removing noisy process from full list
|
|
let Include = materialize(AllSecEvents
|
|
| join kind= leftanti (
|
|
Exclude
|
|
) on TargetProcessFileName);
|
|
// Identifying prevalence for a given process in the environment
|
|
let DCwPC = materialize(Include
|
|
| summarize DistinctHostsProcessCount = dcount(DvcHostname) by TargetProcessFileName
|
|
| join kind=inner (
|
|
Include
|
|
) on TargetProcessFileName
|
|
| distinct DvcHostname, TargetProcessFileName, DistinctHostsProcessCount);
|
|
// Getting the Total process count on each host to use as the denominator in the entropy calc
|
|
let AHPC = materialize(Include
|
|
| summarize AllHostsProcessCount = count() by DvcHostname
|
|
| join kind=inner (
|
|
Include
|
|
) on DvcHostname
|
|
| distinct DvcHostname, TargetProcessFileName, AllHostsProcessCount
|
|
//Getting a decimal value for later computation
|
|
| extend AHPCValue = todecimal(AllHostsProcessCount));
|
|
// Need the count of each class in my bucket or also said as count of ProcName(Class) per Host(Bucket) for use in the entropy calc
|
|
let PCoH = Include
|
|
| summarize ProcessCountOnHost = count() by DvcHostname, TargetProcessFileName
|
|
| join kind=inner (
|
|
Include
|
|
) on DvcHostname,TargetProcessFileName
|
|
| distinct DvcHostname, TargetProcessFileName, ProcessCountOnHost
|
|
//Getting a decimal value for later computation
|
|
| extend PCoHValue = todecimal(ProcessCountOnHost);
|
|
let Combined = DCwPC
|
|
| join (
|
|
AHPC
|
|
) on DvcHostname, TargetProcessFileName
|
|
| join (
|
|
PCoH
|
|
) on DvcHostname, TargetProcessFileName;
|
|
let Results = Combined
|
|
// Entropy calculation
|
|
| extend ProcessEntropy = -log2(PCoHValue/AHPCValue)*(PCoHValue/AHPCValue)
|
|
// Calculating Weight, see details in description
|
|
| extend Weight = toreal(ProcessEntropy*ProcessCountOnHost*DistinctHostsProcessCount)
|
|
// Remove or increase value to see processes with low entropy, meaning more common.
|
|
| where Weight <= 100
|
|
| project DvcHostname, TargetProcessFileName, Weight , ProcessEntropy, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount;
|
|
// Join back full entry
|
|
Results
|
|
| join kind= inner (
|
|
SecEvents
|
|
| project TimeGenerated, EventOriginalUid, DvcHostname, ActorUserId, TargetProcessFileName, User, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, _ResourceId, DvcId , EventVendor, EventProduct
|
|
) on DvcHostname, TargetProcessFileName
|
|
| summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), ResultCount = count() by EventOriginalUid, User, DvcHostname, ActorUserId, Weight, ProcessEntropy,
|
|
TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount, _ResourceId, DvcId , EventVendor, EventProduct
|
|
| project-reorder StartTime, EndTime, ResultCount, EventOriginalUid, EventVendor, EventProduct, DvcHostname, User, ActorUserId, Weight, ProcessEntropy,TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount, _ResourceId, DvcId
|
|
| sort by Weight asc, ProcessEntropy asc, TargetProcessFilePath asc
|
|
| extend timestamp = StartTime, NTDomain = tostring(split(Account,'\\',0)[0]), Name = tostring(split(Account,'\\',1)[0])
|
|
| extend HostName = tostring(split(DvcHostname, '.', 0)[0]), DnsDomain = tostring(strcat_array(array_slice(split(DvcHostname, '.'), 1, -1), '.'))
|
|
| extend Account_0_Name = Name
|
|
| extend Account_0_NTDomain = NTDomain
|
|
| extend Host_0_HostName = HostName
|
|
| extend Host_0_DnsDomain = DnsDomain
|
|
entityMappings:
|
|
- entityType: Account
|
|
fieldMappings:
|
|
- identifier: Name
|
|
columnName: Name
|
|
- identifier: NTDomain
|
|
columnName: NTDomain
|
|
- entityType: Host
|
|
fieldMappings:
|
|
- identifier: HostName
|
|
columnName: HostName
|
|
- identifier: DnsDomain
|
|
columnName: DnsDomain
|
|
version: 1.0.0 |