Merge pull request #2623 from Azure/dev/normalization/process

Fix DvcHostName -> DvcHostname

Hunting Queries/ASimProcess/imProcess_ProcessEntropy.yaml

@@ -46,9 +46,9 @@ query: |
   | where TargetProcessFilePath !has ':\\Windows\\servicing\\trustedinstaller.exe' 
   | where ActingProcessFileName !has ':\\Program Files\\Microsoft Dependency Agent\\bin\\MicrosoftDependencyAgent.exe' 
   | where ActingProcessFileName !has ':\\Program Files (x86)\\Microsoft\\EdgeUpdate\\MicrosoftEdgeUpdate.exe'
-  | project TimeGenerated, EventID, DvcHostName, ActorUserId, Account, AccountType, TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, _ResourceId, DvcId, EventVendor, EventProduct;
+  | project TimeGenerated, EventID, DvcHostname, ActorUserId, Account, AccountType, TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, _ResourceId, DvcId, EventVendor, EventProduct;
   let Exclude = SecEvents 
-  | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), ExcludeCompCount = dcount(DvcHostName), ExcludeProcCount = count() by TargetProcessFileName 
+  | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), ExcludeCompCount = dcount(DvcHostname), ExcludeProcCount = count() by TargetProcessFileName 
   // Removing general limit for noise in one day 
   | extend timediff = iff(datetime_diff('day', EndTime, StartTime) > 0, datetime_diff('day', EndTime, StartTime), 1) 
   // Default exclude of 48 (2 per hour) or more executions in 24 hours on a given machine 
@@ -57,16 +57,16 @@ query: |
   | extend compRatio = ExcludeCompCount/toreal(ExcludeProcCount) 
   | where compRatio == 0 or (ExcludeCompCount > excludeThreshold and compRatio < ratioHighCount) or (ExcludeCompCount between (2 .. excludeThreshold) and compRatio < ratioMidCount);
   let AllSecEvents =  
-  SecEvents | project DvcHostName, TargetProcessFileName , EventVendor, EventProduct
+  SecEvents | project DvcHostname, TargetProcessFileName , EventVendor, EventProduct
   | join kind= leftanti (  
   SecEvents 
   // Removing general limit for noise in one day 
-  | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), procCount = count() by DvcHostName, TargetProcessFileName 
+  | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), procCount = count() by DvcHostname, TargetProcessFileName 
   | extend timediff = iff(datetime_diff('day', EndTime, StartTime) > 0, datetime_diff('day', EndTime, StartTime), 1) 
   // Default exclude 48 (2 per hour) or more executions in 24 hours on a given machine to remove them from overall comparison list 
   | where procCount > procLimit*timediff 
-  ) on DvcHostName, TargetProcessFileName 
-  | project DvcHostName, TargetProcessFileName , EventVendor, EventProduct;
+  ) on DvcHostname, TargetProcessFileName 
+  | project DvcHostname, TargetProcessFileName , EventVendor, EventProduct;
   // Removing noisy process from full list 
   let Include = materialize(AllSecEvents 
   | join kind= leftanti ( 
@@ -74,36 +74,36 @@ query: |
   ) on TargetProcessFileName);
   // Identifying prevalence for a given process in the environment 
   let DCwPC = materialize(Include 
-  | summarize DistinctHostsProcessCount = dcount(DvcHostName) by TargetProcessFileName 
+  | summarize DistinctHostsProcessCount = dcount(DvcHostname) by TargetProcessFileName 
   | join kind=inner ( 
   Include 
   ) on TargetProcessFileName 
-  | distinct DvcHostName, TargetProcessFileName, DistinctHostsProcessCount);
+  | distinct DvcHostname, TargetProcessFileName, DistinctHostsProcessCount);
   // Getting the Total process count on each host to use as the denominator in the entropy calc 
   let AHPC = materialize(Include 
-  | summarize AllHostsProcessCount = count() by DvcHostName 
+  | summarize AllHostsProcessCount = count() by DvcHostname 
   | join kind=inner ( 
   Include 
-  ) on DvcHostName 
-  | distinct DvcHostName, TargetProcessFileName, AllHostsProcessCount 
+  ) on DvcHostname 
+  | distinct DvcHostname, TargetProcessFileName, AllHostsProcessCount 
   //Getting a decimal value for later computation 
   | extend AHPCValue = todecimal(AllHostsProcessCount));
   // Need the count of each class in my bucket or also said as count of ProcName(Class) per Host(Bucket) for use in the entropy calc 
   let PCoH = Include 
-  | summarize ProcessCountOnHost = count() by DvcHostName, TargetProcessFileName 
+  | summarize ProcessCountOnHost = count() by DvcHostname, TargetProcessFileName 
   | join kind=inner ( 
   Include 
-  ) on DvcHostName,TargetProcessFileName 
-  | distinct DvcHostName, TargetProcessFileName, ProcessCountOnHost 
+  ) on DvcHostname,TargetProcessFileName 
+  | distinct DvcHostname, TargetProcessFileName, ProcessCountOnHost 
   //Getting a decimal value for later computation 
   | extend PCoHValue = todecimal(ProcessCountOnHost); 
   let Combined = DCwPC 
   | join ( 
   AHPC 
-  ) on DvcHostName, TargetProcessFileName 
+  ) on DvcHostname, TargetProcessFileName 
   | join ( 
   PCoH 
-  ) on DvcHostName, TargetProcessFileName;
+  ) on DvcHostname, TargetProcessFileName;
   let Results = Combined 
   // Entropy calculation 
   | extend ProcessEntropy = -log2(PCoHValue/AHPCValue)*(PCoHValue/AHPCValue) 
@@ -111,17 +111,17 @@ query: |
   | extend Weight = toreal(ProcessEntropy*ProcessCountOnHost*DistinctHostsProcessCount) 
   // Remove or increase value to see processes with low entropy, meaning more common. 
   | where Weight <= 100
-  | project DvcHostName, TargetProcessFileName, Weight , ProcessEntropy, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount; 
+  | project DvcHostname, TargetProcessFileName, Weight , ProcessEntropy, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount; 
   // Join back full entry 
   Results 
   | join kind= inner ( 
   SecEvents
-  | project TimeGenerated, EventID, DvcHostName, ActorUserId, Account, AccountType, TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, _ResourceId, DvcId , EventVendor, EventProduct
-  ) on DvcHostName, TargetProcessFileName 
-  | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), ResultCount = count() by EventID, DvcHostName, ActorUserId, Account, AccountType, Weight, ProcessEntropy,  
+  | project TimeGenerated, EventID, DvcHostname, ActorUserId, Account, AccountType, TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, _ResourceId, DvcId , EventVendor, EventProduct
+  ) on DvcHostname, TargetProcessFileName 
+  | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), ResultCount = count() by EventID, DvcHostname, ActorUserId, Account, AccountType, Weight, ProcessEntropy,  
   TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount, _ResourceId, DvcId , EventVendor, EventProduct
-  | project-reorder StartTime, EndTime, ResultCount, EventID, EventVendor, EventProduct, DvcHostName, ActorUserId, Account, AccountType, Weight, ProcessEntropy,TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount, _ResourceId, DvcId
+  | project-reorder StartTime, EndTime, ResultCount, EventID, EventVendor, EventProduct, DvcHostname, ActorUserId, Account, AccountType, Weight, ProcessEntropy,TargetProcessFileName, TargetProcessFilePath, TargetProcessCommandLine, ActingProcessFileName, AllHostsProcessCount, ProcessCountOnHost, DistinctHostsProcessCount, _ResourceId, DvcId
   | sort by Weight asc, ProcessEntropy asc, TargetProcessFilePath asc 
-  | extend timestamp = StartTime, HostCustomEntity = DvcHostName, AccountCustomEntity = Account
+  | extend timestamp = StartTime, HostCustomEntity = DvcHostname, AccountCustomEntity = Account
   
 

Hunting Queries/ASimProcess/imProcess_SolarWindsInventory.yaml

@@ -13,6 +13,6 @@ tags:
 query: |
   imProcessCreate
   | where Process has 'solarwinds' 
-  | extend MachineName = DvcHostName , Process = TargetProcessFilePath
+  | extend MachineName = DvcHostname , Process = TargetProcessFilePath
   | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), MachineCount = dcount(Dvc), AccountCount = dcount(User), MachineNames = make_set(Dvc),
   Accounts = make_set(User) by Process, EventVendor, EventProduct

Parsers/ASimAuthentication/ProductParsers/AuthenticationM365D.yaml

@@ -0,0 +1,117 @@
+Parser:
+  Title: ASIM Microsoft 365 Defender DeviceLogonEvent Authentication parser
+  Version: '0.1.0'
+  LastUpdated: July 1, 2021
+Product:
+  Name: MMicrosoft 365 Defender
+Normalization:
+  Schema: Authentication
+  Version: '0.1.0'
+References:
+- Title: ASIM Authentication Schema
+  Link: https://aka.ms/AzSentinelAuthenticationDoc
+- Title: ASIM
+  Link: https:/aka.ms/AzSentinelNormalization
+Description: |
+      This Query Parser maps M365 Defender Device Logon Events (DeviceLogonEvents) to the Azure Sentinel Information Model Authenticaion schema.
+ParserName: vimAuthenticationM365Defender
+ParserQuery: |
+  let FaliureReason=datatable(EventOriginalResultDetails:string, EventResultDetails:string)[
+    'InvalidUserNameOrPassword','No such user or password' 
+    // What other will we find here?
+    ];
+    let AuthM365D=(){
+    DeviceLogonEvents 
+    // ---   Already mapped
+    // // TimeGenerated - exists
+    //
+    | project-rename 
+       EventOriginalResultDetails=FailureReason 
+       , EventSubType=LogonType
+    | extend 
+    //
+       // ----  Untouched
+       TimeGenerated
+    // 
+      // ----  Event
+       , EventCount=int(1)
+       , EventStartTime=TimeGenerated
+       , EventEndTime=TimeGenerated
+       , EventOriginalType = EventSubType // Previously:LogonType
+       , EventProduct='M365 Defender for EndPoint'
+      ,  EventResult = case(ActionType =='LogonSuccess', 'Success'
+                            , ActionType=='LogonFailed', 'Failure'
+                            , ActionType=='LogonAttempted', 'NA' //// Not sure what this is
+                            , 'NA')
+      , EventSchemaVersion='0.1.0'
+      , EventType='Logon'
+      , EventVendor ='Microsoft'
+    //    -   No Data
+    //  EventOriginalUid 
+    //  EventProductVersion 
+    //  EventReportUrl
+    //  EventMessage 
+    //  
+    //  
+    //  ----   Target and Actor Users
+     | project-rename 
+         TargetUserId=AccountSid
+       , ActorUserId =InitiatingProcessAccountSid
+      | extend 
+         TargetUserIdType ='SID'
+       , TargetUsername = strcat(AccountDomain,'\\',AccountName)
+       , TargetUsernameType='Windows'
+    //    -  No Data
+      // TargetUserType 
+    // 
+    // ActingAppName 
+    // ActingAppType 
+    // ActorSessionId 
+      , ActorUserIdType='SID'
+    // ActorUserType 
+      , ActorUsername=coalesce(InitiatingProcessAccountUpn, strcat(InitiatingProcessAccountDomain,'\\',InitiatingProcessAccountName)) // InitiatingProcessAccountName
+      , ActorUsernameType=case(isnotempty( InitiatingProcessAccountUpn), 'Upn/Email'
+                                , isnotempty(InitiatingProcessAccountDomain), 'Windows'
+                                , 'Simple')
+     //, AdditionalFields:already exists
+      | project-rename 
+        DvcHostname=DeviceName
+      , LogonProtocol=Protocol
+      , TargetDvcId=DeviceId
+      , TargetDvcIpAddr=RemoteIP
+    // No Data
+    // DvcIpAddr 
+    // HttpUserAgent 
+    // LogonMethod 
+    //
+      , SrcDvcHostname=RemoteDeviceName 
+      // SrcDvcHostnameType == Requires calculation. not sure it is worth it
+    // SrcDvcId 
+    // SrcDvcIpAddr 
+    // SrcDvcOs 
+    // SrcDvcType 
+    // SrcGeo fields 
+    //
+    // TargetAppId 
+    // TargetAppName 
+    // TargetAppType 
+    // TargetDvc 
+      | extend 
+        TargetDvcHostname=DvcHostname
+        ,  TargetDvcHostnameType='FQDN'
+        , TargetDvcIdType='MDE'
+    // TargetDvcOs
+    // TargetDvcType
+      , TargetPortNumber=RemotePort
+      , TargetSessionId =LogonId
+      // EventResultDetails
+      | lookup FaliureReason on EventOriginalResultDetails 
+    // TargetUrl 
+    // -----------  Alias
+      | extend 
+        User=TargetUsername 
+        , LogonTarget=TargetDvcHostname
+        , Dvc=TargetDvcHostname
+    // No data
+        // // _ResourceId - no data 
+        };

Parsers/ASimProcess/ARM/ProcessEventMicrosoftSysmonTerminate/ProcessEventMicrosoftSysmonTerminate.json

@@ -28,7 +28,7 @@
             "displayName": "ASIM Sysmon/Event Process Termination Event Parser",
             "category": "Security",
             "FunctionAlias": "vimProcessTermianteMicrosoftSysmon",
-            "query": "let ParsedProcessEvent=(){\n    // Create the raw table from the raw XML file structure\n    Event \n    | where Source == \"Microsoft-Windows-Sysmon\" and EventID==5\n    | parse EventData with * '<Data Name=\"RuleName\">'RuleName'</Data>'\n             '<Data Name=\"UtcTime\">'UtcTime'</Data>'\n             '<Data Name=\"ProcessGuid\">{'ProcessGuid'}</Data>'\n             '<Data Name=\"ProcessId\">'ProcessId'</Data>'\n             '<Data Name=\"Image\">'Image'</Data>' *\n    | extend \n            EventType = \"ProcessTerminated\",\n            EventStartTime = todatetime(TimeGenerated),\n            EventEndTime = todatetime(TimeGenerated),\n            EventCount = int(1),\n            EventVendor = \"Microsoft\",\n            EventSchemaVersion = \"0.1.0\",\n            EventProduct = \"Event\",\n            DvcOs = \"Windows\",\n            ActorUsernameType = \"Windows\"\n    | project-rename\n            DvcHostName = Computer,\n            ActorUsername = UserName,\n\n            TargetProcessName = Image,\n            TargetProcessId = ProcessId,\n            TargetProcessGuid = ProcessGuid,\n            EventOriginalId=EventID\n\n     //***** Aliases ******\n    | extend\n           User = ActorUsername,\n           Process = TargetProcessName,\n           Dvc = DvcHostName\n    }; ParsedProcessEvent\n",
+            "query": "let ParsedProcessEvent=(){\n    // Create the raw table from the raw XML file structure\n    Event \n    | where Source == \"Microsoft-Windows-Sysmon\" and EventID==5\n    | parse EventData with * '<Data Name=\"RuleName\">'RuleName'</Data>'\n             '<Data Name=\"UtcTime\">'UtcTime'</Data>'\n             '<Data Name=\"ProcessGuid\">{'ProcessGuid'}</Data>'\n             '<Data Name=\"ProcessId\">'ProcessId'</Data>'\n             '<Data Name=\"Image\">'Image'</Data>' *\n    | extend \n            EventType = \"ProcessTerminated\",\n            EventStartTime = todatetime(TimeGenerated),\n            EventEndTime = todatetime(TimeGenerated),\n            EventCount = int(1),\n            EventVendor = \"Microsoft\",\n            EventSchemaVersion = \"0.1.0\",\n            EventProduct = \"Event\",\n            DvcOs = \"Windows\",\n            ActorUsernameType = \"Windows\"\n    | project-rename\n            DvcHostname = Computer,\n            ActorUsername = UserName,\n\n            TargetProcessName = Image,\n            TargetProcessId = ProcessId,\n            TargetProcessGuid = ProcessGuid,\n            EventOriginalId=EventID\n\n     //***** Aliases ******\n    | extend\n           User = ActorUsername,\n           Process = TargetProcessName,\n           Dvc = DvcHostname\n    }; ParsedProcessEvent\n",
             "version": 1
           }
         }

Parsers/ASimProcess/ProductParsers/ProcessEventMicrosoftSSysmonCreate.yaml

@@ -65,7 +65,7 @@ ParserQuery: |
  
                     | project-rename 
                                 EventMessage = RenderedDescription,
-                                DvcHostName = Computer,
+                                DvcHostname = Computer,
 
                                 EventOriginalId = EventID,
                                 TargetUserSessionGuid = LogonGuid, 
@@ -90,6 +90,6 @@ ParserQuery: |
                     | extend // aliases
                                 // User = TargetUserName,
                                 Process = TargetProcessName,
-                                Dvc = DvcHostName,
+                                Dvc = DvcHostname,
                                 Hash = coalesce(TargetProcessSHA256, TargetProcessSHA1, TargetProcessMD5) // which appears first - will be aliases to "Hash"
                     }; ParsedProcessEvent

Parsers/ASimProcess/ProductParsers/ProcessEventMicrosoftSysmonTerminate.yaml

@@ -35,7 +35,7 @@ ParserQuery: |
                         DvcOs = "Windows",
                         ActorUsernameType = "Windows"
                 | project-rename
-                        DvcHostName = Computer,
+                        DvcHostname = Computer,
                         ActorUsername = UserName,
 
                         TargetProcessName = Image,
@@ -47,5 +47,5 @@ ParserQuery: |
                 | extend
                        User = ActorUsername,
                        Process = TargetProcessName,
-                       Dvc = DvcHostName
+                       Dvc = DvcHostname
                 }; ParsedProcessEvent