Updating TI queries based on feedback and discussions on this PR - #3477 - and I don't want preferences for a specific environment to be included. This includes generic changes that need to be done.

Detections/ThreatIntelligenceIndicator/DomainEntity_CommonSecurityLog.yaml

@@ -17,47 +17,49 @@ triggerThreshold: 0
 tactics:
   - Impact
 query: |
-    let dt_lookBack = 1h;
-    let ioc_lookBack = 14d;
-    //Create a list of TLDs in our threat feed for later validation of extracted domains
-    let list_tlds = ThreatIntelligenceIndicator
-        | where TimeGenerated > ago(ioc_lookBack)
-        | where isnotempty(DomainName)
-        | extend DomainName = tolower(DomainName)
-        | extend parts = split(DomainName, '.')
-        | extend tld = parts[(array_length(parts)-1)]
-        | summarize count() by tostring(tld)
-        | summarize make_list(tld);
-        ThreatIntelligenceIndicator
-        | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-        | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-        | where Active == true
-        // Picking up only IOC's that contain the entities we want
-        | where isnotempty(DomainName)
-        | join (
-            CommonSecurityLog
-            | extend IngestionTime = ingestion_time()
-            | where IngestionTime > ago(dt_lookBack)
-            | where DeviceEventClassID =~ 'url'
-            //Uncomment the line below to only alert on allowed connections
-            //| where DeviceAction !~ "block-url"
-            //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions
-            | extend PA_Url = columnifexists("RequestURL", "None")
-            | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url))
-            | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url))
-            | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host))
-            | where isnotempty(Domain)
-            | extend Domain = tolower(Domain)
-            | extend parts = split(Domain, '.')
-            //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on
-            | extend tld = parts[(array_length(parts)-1)]
-            //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match
-            | where tld in~ (list_tlds)
-            | extend CommonSecurityLog_TimeGenerated = TimeGenerated
-        ) on $left.DomainName==$right.Domain
-        | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime
-        | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
-        | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
+  let dt_lookBack = 1h;
+  let ioc_lookBack = 14d;
+  //Create a list of TLDs in our threat feed for later validation of extracted domains
+  let list_tlds = ThreatIntelligenceIndicator
+  | where TimeGenerated > ago(ioc_lookBack)
+  | where isnotempty(DomainName)
+  | extend DomainName = tolower(DomainName)
+  | extend parts = split(DomainName, '.')
+  | extend tld = parts[(array_length(parts)-1)]
+  | summarize count() by tostring(tld)
+  | summarize make_list(tld);
+  ThreatIntelligenceIndicator
+   where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
+   | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+   | where Active == true
+   // Picking up only IOC's that contain the entities we want
+   | where isnotempty(DomainName)
+   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+   | join kind=innerunique (
+     CommonSecurityLog
+     | extend IngestionTime = ingestion_time()
+     | where IngestionTime > ago(dt_lookBack)
+     | where DeviceEventClassID =~ 'url'
+     //Uncomment the line below to only alert on allowed connections
+     //| where DeviceAction !~ "block-url"
+     //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions
+     | extend PA_Url = columnifexists("RequestURL", "None")
+     | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url))
+     | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url))
+     | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host))
+     | where isnotempty(Domain)
+     | extend Domain = tolower(Domain)
+     | extend parts = split(Domain, '.')
+     //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on
+     | extend tld = parts[(array_length(parts)-1)]
+     //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match
+     | where tld in~ (list_tlds)
+     | extend CommonSecurityLog_TimeGenerated = TimeGenerated
+     ) on $left.DomainName==$right.Domain
+  | where CommonSecurityLog_TimeGenerated < ExpirationDateTime
+  | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId, FileHashValue
+  | project CommonSecurityLog_TimeGenerated, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
+  | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
 entityMappings:
   - entityType: Host
     fieldMappings:
@@ -67,6 +69,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: DestinationIP
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/DomainEntity_DnsEvents.yaml

@@ -37,7 +37,8 @@ query: |
     | where Active == true
     // Picking up only IOC's that contain the entities we want
     | where isnotempty(DomainName)
-    | join (
+    // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+    | join kind=innerunique (
          DnsEvents
         | where TimeGenerated > ago(dt_lookBack)
         //Extract domain patterns from syslog message
@@ -49,9 +50,9 @@ query: |
         | where tld in~ (list_tlds)
         | extend DNS_TimeGenerated = TimeGenerated
     ) on $left.DomainName==$right.Name
-    | where DNS_TimeGenerated >= TimeGenerated and DNS_TimeGenerated < ExpirationDateTime
-    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, DNS_TimeGenerated, Computer, ClientIP, Name, QueryType
+    | where DNS_TimeGenerated < ExpirationDateTime
+    | summarize DNS_TimeGenerated  = arg_max(DNS_TimeGenerated , *) by IndicatorId, Name
+    | project DNS_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, Computer, ClientIP, Name, QueryType
     | extend timestamp = DNS_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = ClientIP, URLCustomEntity = Url
 entityMappings:
   - entityType: Host

Detections/ThreatIntelligenceIndicator/DomainEntity_PaloAlto.yaml

@@ -38,7 +38,8 @@ query: |
         | where Active == true
         // Picking up only IOC's that contain the entities we want
         | where isnotempty(DomainName)
-        | join (
+        // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+        | join kind=innerunique (
             CommonSecurityLog
             | extend IngestionTime = ingestion_time()
             | where IngestionTime > ago(dt_lookBack)
@@ -60,8 +61,10 @@ query: |
             | where tld in~ (list_tlds)
             | extend CommonSecurityLog_TimeGenerated = TimeGenerated
         ) on $left.DomainName==$right.Domain
-        | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime
-        | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
+        | where CommonSecurityLog_TimeGenerated < ExpirationDateTime
+        | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId, Domain
+        | project CommonSecurityLog_TimeGenerated, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, 
+        DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
         | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
 entityMappings:
   - entityType: Host
@@ -72,6 +75,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: DestinationIP
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/DomainEntity_SecurityAlert.yaml

@@ -40,7 +40,8 @@ query: |
     | where Active == true
     // Picking up only IOC's that contain the entities we want
     | where isnotempty(DomainName)
-    | join (
+    // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+    | join kind=innerunique (
         SecurityAlert
         | where TimeGenerated > ago(dt_lookBack)
         | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false)
@@ -62,9 +63,9 @@ query: |
         | extend Alert_TimeGenerated = TimeGenerated
         | extend Alert_Description = Description
     ) on $left.DomainName==$right.domain
-    | where Alert_TimeGenerated >= TimeGenerated and Alert_TimeGenerated < ExpirationDateTime
-    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Alert_TimeGenerated, AlertName, Alert_Description, ProviderName, AlertSeverity, ConfidenceLevel, HostName, IP_addr, Url
+    | where Alert_TimeGenerated < ExpirationDateTime
+    | summarize Alert_TimeGenerated = arg_max(Alert_TimeGenerated, *) by IndicatorId, AlertName
+    | project Alert_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, AlertName, Alert_Description, ProviderName, AlertSeverity, ConfidenceLevel, HostName, IP_addr, Url
     | extend timestamp = Alert_TimeGenerated, HostCustomEntity = HostName, IPCustomEntity = IP_addr, URLCustomEntity = Url
 entityMappings:
   - entityType: Host

Detections/ThreatIntelligenceIndicator/DomainEntity_Syslog.yaml

@@ -37,7 +37,8 @@ query: |
     | where Active == true
     // Picking up only IOC's that contain the entities we want
     | where isnotempty(DomainName)
-    | join (
+    // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+    | join kind=innerunique (
         Syslog
         | where TimeGenerated > ago(dt_lookBack)
         //Extract domain patterns from syslog message
@@ -50,9 +51,9 @@ query: |
         | where tld in~ (list_tlds)
         | extend Syslog_TimeGenerated = TimeGenerated
     ) on $left.DomainName==$right.domain
-    | where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime
-    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, domain, HostIP, Url
+    | where Syslog_TimeGenerated < ExpirationDateTime
+    | summarize Syslog_TimeGenerated  = arg_max(Syslog_TimeGenerated , *) by IndicatorId, domain
+    | project Syslog_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, SyslogMessage, Computer, ProcessName, domain, HostIP, Url
     | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url
 entityMappings:
   - entityType: Host

Detections/ThreatIntelligenceIndicator/EmailEntity_AzureActivity.yaml

@@ -30,18 +30,19 @@ query: |
   | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       AzureActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(Caller)
       | extend Caller = tolower(Caller)
       | where Caller matches regex emailregex
       | extend AzureActivity_TimeGenerated = TimeGenerated
   )
   on $left.EmailSenderAddress == $right.Caller
-  | where AzureActivity_TimeGenerated >= TimeGenerated and AzureActivity_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, AzureActivity_TimeGenerated,
-  EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, Caller, Level, CallerIpAddress, CategoryValue,
-  OperationNameValue, ActivityStatusValue, ResourceGroup, SubscriptionId
+  | where AzureActivity_TimeGenerated < ExpirationDateTime
+  | summarize AzureActivity_TimeGenerated = arg_max(AzureActivity_TimeGenerated, *) by IndicatorId, Caller
+  | project AzureActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, EmailSenderName, EmailRecipient, 
+  EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, Caller, Level, CallerIpAddress, CategoryValue, OperationNameValue, ActivityStatusValue, 
+  ResourceGroup, SubscriptionId
   | extend timestamp = AzureActivity_TimeGenerated, AccountCustomEntity = Caller, IPCustomEntity = CallerIpAddress, URLCustomEntity = Url
 entityMappings:
   - entityType: Account

Detections/ThreatIntelligenceIndicator/EmailEntity_OfficeActivity.yaml

@@ -30,15 +30,16 @@ query: |
   | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       OfficeActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserId)
       | where UserId matches regex emailregex
       | extend OfficeActivity_TimeGenerated = TimeGenerated
   )
   on $left.EmailSenderAddress == $right.UserId
-  | where OfficeActivity_TimeGenerated >= TimeGenerated and OfficeActivity_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, OfficeActivity_TimeGenerated,
+  | where OfficeActivity_TimeGenerated < ExpirationDateTime
+  | summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId, UserId
+  | project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, UserId, ClientIP, Operation, UserType, RecordType, OfficeWorkload, Parameters
   | extend timestamp = OfficeActivity_TimeGenerated, AccountCustomEntity = UserId, IPCustomEntity = ClientIP, URLCustomEntity = Url
 entityMappings:

Detections/ThreatIntelligenceIndicator/EmailEntity_PaloAlto.yaml

@@ -30,7 +30,8 @@ query: |
   | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) and isnotempty(DestinationUserID)
       // Filtering PAN Logs for specific event type to match relevant email entities
       | where DeviceVendor == "Palo Alto Networks" and  DeviceEventClassID == "wildfire" and ApplicationProtocol in ("smtp","pop3")
@@ -39,11 +40,11 @@ query: |
       | extend CommonSecurityLog_TimeGenerated = TimeGenerated
   )
   on $left.EmailSenderAddress == $right.DestinationUserID
-  | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated,
-  EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, DestinationUserID, DeviceEventClassID, LogSeverity, DeviceAction,
-  SourceIP, SourcePort, DestinationIP, DestinationPort, Protocol, ApplicationProtocol
+  | where CommonSecurityLog_TimeGenerated < ExpirationDateTime
+  | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId, DestinationUserID
+  | project CommonSecurityLog_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, EmailSenderName, EmailRecipient, 
+  EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, DestinationUserID, DeviceEventClassID, LogSeverity, DeviceAction, SourceIP, SourcePort, 
+  DestinationIP, DestinationPort, Protocol, ApplicationProtocol
   | extend timestamp = CommonSecurityLog_TimeGenerated, AccountCustomEntity = DestinationUserID, IPCustomEntity = SourceIP, URLCustomEntity = Url
 entityMappings:
   - entityType: Account
@@ -54,6 +55,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: DestinationIP
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityAlert.yaml

@@ -30,7 +30,8 @@ query: |
   | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       SecurityAlert 
       | where TimeGenerated >= ago(dt_lookBack)
       | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false)
@@ -46,9 +47,9 @@ query: |
       | extend Alert_TimeGenerated = TimeGenerated
   )
   on $left.EmailSenderAddress == $right.EntityEmail
-  | where Alert_TimeGenerated >= TimeGenerated and Alert_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, Alert_TimeGenerated,
+  | where Alert_TimeGenerated < ExpirationDateTime
+  | summarize Alert_TimeGenerated = arg_max(Alert_TimeGenerated, *) by IndicatorId, AlertName
+  | project Alert_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, 
   EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, EntityEmail, AlertName, AlertType,
   AlertSeverity, Entities, ProviderName, VendorName
   | extend timestamp = Alert_TimeGenerated, AccountCustomEntity = EntityEmail, URLCustomEntity = Url

Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityEvent.yaml

@@ -30,7 +30,8 @@ query: |
   | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       SecurityEvent | where TimeGenerated >= ago(dt_lookBack) and isnotempty(TargetUserName)
       //Normalizing the column to lower case for exact match with EmailSenderAddress column
       | extend TargetUserName = tolower(TargetUserName)
@@ -38,9 +39,9 @@ query: |
       | extend SecurityEvent_TimeGenerated = TimeGenerated
   )
   on $left.EmailSenderAddress == $right.TargetUserName
-  | where SecurityEvent_TimeGenerated >= TimeGenerated and SecurityEvent_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, SecurityEvent_TimeGenerated,
+  | where SecurityEvent_TimeGenerated < ExpirationDateTime
+  | summarize SecurityEvent_TimeGenerated = arg_max(SecurityEvent_TimeGenerated, *) by IndicatorId, TargetUserName
+  | project SecurityEvent_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, Computer, EventID, TargetUserName, Activity, IpAddress, AccountType,
   LogonTypeName, LogonProcessName, Status, SubStatus
   | extend timestamp = SecurityEvent_TimeGenerated, AccountCustomEntity = TargetUserName, IPCustomEntity = IpAddress, HostCustomEntity = Computer, URLCustomEntity = Url

Detections/ThreatIntelligenceIndicator/EmailEntity_SigninLogs.yaml

@@ -34,7 +34,8 @@ query: |
   | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       table(tableName) | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserPrincipalName)
       //Normalizing the column to lower case for exact match with EmailSenderAddress column
       | extend UserPrincipalName = tolower(UserPrincipalName)
@@ -46,9 +47,9 @@ query: |
       | extend SigninLogs_TimeGenerated = TimeGenerated, Type = Type
   )
   on $left.EmailSenderAddress == $right.UserPrincipalName
-  | where SigninLogs_TimeGenerated >= TimeGenerated and SigninLogs_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, SigninLogs_TimeGenerated,
+  | where SigninLogs_TimeGenerated < ExpirationDateTime
+  | summarize SigninLogs_TimeGenerated = arg_max(SigninLogs_TimeGenerated, *) by IndicatorId, UserPrincipalName
+  | project SigninLogs_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, IPAddress, UserPrincipalName, AppDisplayName,
   StatusCode, StatusDetails, NetworkIP, NetworkDestinationIP, NetworkSourceIP, Type
   | extend timestamp = SigninLogs_TimeGenerated, AccountCustomEntity = UserPrincipalName, IPCustomEntity = IPAddress, URLCustomEntity = Url

Detections/ThreatIntelligenceIndicator/FileHashEntity_CommonSecurityLog.yaml

@@ -29,18 +29,19 @@ query: |
   | where Active == true
   | where isnotempty(FileHashValue);
   // Handle matches against both lower case and uppercase versions of the hash:
-  ( fileHashIndicators | extend  FileHashValue = tolower(FileHashValue)
-    |union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue)))
-  |  join (
-     CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack)
-     | where isnotempty(FileHash)
-     | extend CommonSecurityLog_TimeGenerated = TimeGenerated
-  )
+  (fileHashIndicators | extend  FileHashValue = tolower(FileHashValue)
+  | union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue)))
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  |  join kind=innerunique (
+    CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack)
+    | where isnotempty(FileHash)
+    | extend CommonSecurityLog_TimeGenerated = TimeGenerated
+    )
   on $left.FileHashValue == $right.FileHash
-  | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  CommonSecurityLog_TimeGenerated, SourceIP, SourcePort, DestinationIP, DestinationPort, SourceUserID, SourceUserName, DeviceName, DeviceAction,
+  | where CommonSecurityLog_TimeGenerated < ExpirationDateTime
+  | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId, FileHashValue
+  | project CommonSecurityLog_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  SourceIP, SourcePort, DestinationIP, DestinationPort, SourceUserID, SourceUserName, DeviceName, DeviceAction,
   RequestURL, DestinationUserName, DestinationUserID, ApplicationProtocol, Activity
   | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, AccountCustomEntity = SourceUserName, URLCustomEntity = Url
 entityMappings:
@@ -56,6 +57,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: DestinationIP
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/FileHashEntity_Covid19_CommonSecurityLog.yaml

@@ -21,17 +21,19 @@ query: |
   let fileHashIndicators = covidIndicators
   | where isnotempty(FileHashValue);
   // Handle matches against both lower case and uppercase versions of the hash:
-  ( fileHashIndicators | extend FileHashValue = tolower(FileHashValue)
-    | union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue)))
-  |  join (
+  (fileHashIndicators | extend FileHashValue = tolower(FileHashValue)
+  | union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue)))
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  |  join kind=innerunique (
      CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) 
      | where isnotempty(FileHash)
      | extend CommonSecurityLog_TimeGenerated = TimeGenerated
-  )
+     )
   on $left.FileHashValue == $right.FileHash
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by FileHashValue
-  | project LatestIndicatorTime, FileHashValue, FileHashType, Description, ThreatType,  
-  CommonSecurityLog_TimeGenerated, SourceIP, SourcePort, DestinationIP, DestinationPort, SourceUserID, SourceUserName, DeviceName, DeviceAction, 
+  | where CommonSecurityLog_TimeGenerated < ExpirationDateTime
+  | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId, FileHashValue
+  | project CommonSecurityLog_TimeGenerated, FileHashValue, FileHashType, Description, ThreatType,  
+  SourceIP, SourcePort, DestinationIP, DestinationPort, SourceUserID, SourceUserName, DeviceName, DeviceAction, 
   RequestURL, DestinationUserName, DestinationUserID, ApplicationProtocol, Activity
   | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, AccountCustomEntity = SourceUserName
 entityMappings:
@@ -47,5 +49,7 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: DestinationIP
 version: 1.0.0
 kind: Scheduled

Detections/ThreatIntelligenceIndicator/FileHashEntity_SecurityEvent.yaml

@@ -28,17 +28,18 @@ query: |
   | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
   | where Active == true
   | where isnotempty(FileHashValue)
-  |  join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
     SecurityEvent | where TimeGenerated >= ago(dt_lookBack)
         | where EventID in ("8003","8002","8005")
         | where isnotempty(FileHash)
         | extend SecurityEvent_TimeGenerated = TimeGenerated, Event = EventID
   )
   on $left.FileHashValue == $right.FileHash
-  | where SecurityEvent_TimeGenerated >= TimeGenerated and SecurityEvent_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  SecurityEvent_TimeGenerated, Process, FileHash, Computer, Account, Event
+  | where SecurityEvent_TimeGenerated < ExpirationDateTime
+  | summarize SecurityEvent_TimeGenerated = arg_max(SecurityEvent_TimeGenerated, *) by IndicatorId, FileHash
+  | project SecurityEvent_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  Process, FileHash, Computer, Account, Event
   | extend timestamp = SecurityEvent_TimeGenerated, AccountCustomEntity = Account, HostCustomEntity = Computer, URLCustomEntity = Url
 entityMappings:
   - entityType: Account

Detections/ThreatIntelligenceIndicator/IPEntity_AWSCloudTrail.yaml

@@ -34,15 +34,16 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       AWSCloudTrail | where TimeGenerated >= ago(dt_lookBack)
       // renaming time column so it is clear the log this came from
       | extend AWSCloudTrail_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.SourceIpAddress
-  | where AWSCloudTrail_TimeGenerated >= TimeGenerated and AWSCloudTrail_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AWSCloudTrail_TimeGenerated,
+  | where AWSCloudTrail_TimeGenerated < ExpirationDateTime
+  | summarize AWSCloudTrail_TimeGenerated = arg_max(AWSCloudTrail_TimeGenerated, *) by IndicatorId, SourceIpAddress
+  | project AWSCloudTrail_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   TI_ipEntity, EventName, EventTypeName, UserIdentityAccountId, UserIdentityPrincipalid, UserIdentityUserName, SourceIpAddress,
   NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = AWSCloudTrail_TimeGenerated, IPCustomEntity = SourceIpAddress, AccountCustomEntity = UserIdentityUserName, URLCustomEntity = Url

Detections/ThreatIntelligenceIndicator/IPEntity_AppServiceHTTPLogs.yaml

@@ -31,16 +31,19 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
-      AppServiceHTTPLogs | where TimeGenerated >= ago(dt_lookBack)
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+    AppServiceHTTPLogs | where TimeGenerated >= ago(dt_lookBack)
     | where isnotempty(CIp)
     | extend WebApp = split(_ResourceId, '/')[8]
     // renaming time column so it is clear the log this came from
     | extend AppService_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.CIp
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AppService_TimeGenerated, TI_ipEntity, CsUsername, WebApp = split(_ResourceId, '/')[8], CIp, CsHost, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
+  | where AppService_TimeGenerated < ExpirationDateTime
+  | summarize AppService_TimeGenerated = arg_max(AppService_TimeGenerated, *) by IndicatorId, CIp
+  | project AppService_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, TI_ipEntity, CsUsername, 
+  WebApp = split(_ResourceId, '/')[8], CIp, CsHost, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress, _ResourceId
   | extend timestamp = AppService_TimeGenerated, AccountCustomEntity = CsUsername, IPCustomEntity = CIp, URLCustomEntity = CsHost
 entityMappings:
   - entityType: Host
@@ -59,5 +62,9 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: URLCustomEntity
+  - entityType: AzureResource
+    fieldMappings:
+      - identifier: ResourceId
+        columnName: _ResourceId
 version: 1.1.1
 kind: Scheduled

Detections/ThreatIntelligenceIndicator/IPEntity_AzureActivity.yaml

@@ -34,16 +34,17 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       AzureActivity | where TimeGenerated >= ago(dt_lookBack)
       // renaming time column so it is clear the log this came from
       | extend AzureActivity_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.CallerIpAddress
-  | where AzureActivity_TimeGenerated >= TimeGenerated and AzureActivity_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AzureActivity_TimeGenerated,
-  TI_ipEntity, CallerIpAddress, Caller, OperationNameValue, ActivityStatusValue, CategoryValue, ResourceId, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
+  | where AzureActivity_TimeGenerated < ExpirationDateTime
+  | summarize AzureActivity_TimeGenerated = arg_max(AzureActivity_TimeGenerated, *) by IndicatorId, CallerIpAddress
+  | project AzureActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, TI_ipEntity, CallerIpAddress, 
+  Caller, OperationNameValue, ActivityStatusValue, CategoryValue, ResourceId, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = AzureActivity_TimeGenerated, IPCustomEntity = CallerIpAddress, AccountCustomEntity = Caller, URLCustomEntity = Url
 entityMappings:
   - entityType: Account
@@ -58,5 +59,9 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: URLCustomEntity
+  - entityType: AzureResource
+    fieldMappings:
+      - identifier: ResourceId
+        columnName: ResourceId
 version: 1.1.1
 kind: Scheduled

Detections/ThreatIntelligenceIndicator/IPEntity_AzureFirewall.yaml

@@ -34,7 +34,8 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       AzureDiagnostics
       | where TimeGenerated >= ago(dt_lookBack)
       | where OperationName in ("AzureFirewallApplicationRuleLog","AzureFirewallNetworkRuleLog")
@@ -46,7 +47,7 @@ query: |
   )
   on $left.TI_ipEntity == $right.DestinationAddress
   | where AzureFirewall_TimeGenerated < ExpirationDateTime
-  | summarize AzureFirewall_TimeGenerated = arg_max(AzureFirewall_TimeGenerated, *) by IndicatorId, SourceAddress
+  | summarize AzureFirewall_TimeGenerated = arg_max(AzureFirewall_TimeGenerated, *) by IndicatorId, DestinationAddress
   | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, DomainName, ExpirationDateTime, ConfidenceScore, AzureFirewall_TimeGenerated,
   TI_ipEntity, Resource, Category, msg_s, SourceAddress, DestinationAddress, Action, Protocol, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = AzureFirewall_TimeGenerated, IPCustomEntity = TI_ipEntity, URLCustomEntity = Url
@@ -55,6 +56,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: SourceAddress
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/IPEntity_AzureKeyVault.yaml

@@ -39,7 +39,7 @@ query: |
   on $left.TI_ipEntity == $right.ClientIP
   | where KeyVaultEvents_TimeGenerated < ExpirationDateTime
   | summarize KeyVaultEvents_TimeGenerated = arg_max(KeyVaultEvents_TimeGenerated, *) by IndicatorId, ClientIP
-  | project KeyVaultEvents_TimeGenerated , Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  | project KeyVaultEvents_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   TI_ipEntity, ClientIP, ResourceId, SubscriptionId, OperationName, ResultType, CorrelationId, id_s, clientInfo_s, httpStatusCode_d, identity_claim_appid_g, identity_claim_http_schemas_microsoft_com_identity_claims_objectidentifier_g
   | extend timestamp = KeyVaultEvents_TimeGenerated
 entityMappings:

Detections/ThreatIntelligenceIndicator/IPEntity_AzureNetworkAnalytics.yaml

@@ -31,7 +31,8 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       AzureNetworkAnalytics_CL
       | where TimeGenerated >= ago(dt_lookBack)
       // renaming time column so it is clear the log this came from
@@ -41,11 +42,11 @@ query: |
       | extend PIP = tostring(PIPs[0])
   )
   on $left.TI_ipEntity == $right.PIP
-  | where AzureNetworkAnalytics_CL_TimeGenerated >= TimeGenerated and AzureNetworkAnalytics_CL_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where AzureNetworkAnalytics_CL_TimeGenerated < ExpirationDateTime
+  | summarize AzureNetworkAnalytics_CL_TimeGenerated = arg_max(AzureNetworkAnalytics_CL_TimeGenerated, *) by IndicatorId, PIP
   // Set to alert on Allowed NSG Flows from TI Public IP IOC
   | where FlowStatus_s == "A"
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AzureNetworkAnalytics_CL_TimeGenerated,
+  | project AzureNetworkAnalytics_CL_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   TI_ipEntity, Computer, FlowDirection_s, FlowStatus_s, FlowType_s, SrcPublicIPs_s, DestPublicIPs_s, PublicIPs_s, L7Protocol_s, DestPort_d, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = AzureNetworkAnalytics_CL_TimeGenerated, IPCustomEntity = TI_ipEntity, HostCustomEntity = Computer, URLCustomEntity = Url
 entityMappings:

Detections/ThreatIntelligenceIndicator/IPEntity_DnsEvents.yaml

@@ -34,7 +34,8 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       DnsEvents | where TimeGenerated >= ago(dt_lookBack)
       | where SubType =~ "LookupQuery" and isnotempty(IPAddresses)
       | extend SingleIP = split(IPAddresses, ",")
@@ -44,9 +45,9 @@ query: |
       | extend DNS_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.SingleIP
-  | where DNS_TimeGenerated >= TimeGenerated and DNS_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, DomainName, ExpirationDateTime, ConfidenceScore, DNS_TimeGenerated,
+  | where DNS_TimeGenerated < ExpirationDateTime
+  | summarize DNS_TimeGenerated = arg_max(DNS_TimeGenerated , *) by IndicatorId, SingleIP
+  | project DNS_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, DomainName, ExpirationDateTime, ConfidenceScore,
   TI_ipEntity, Computer, EventId, SubType, ClientIP, Name, IPAddresses, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = DNS_TimeGenerated, IPCustomEntity = ClientIP, HostCustomEntity = Computer, URLCustomEntity = Url
 entityMappings:

Detections/ThreatIntelligenceIndicator/IPEntity_OfficeActivity.yaml

@@ -34,15 +34,16 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       OfficeActivity | where TimeGenerated >= ago(dt_lookBack)
       // renaming time column so it is clear the log this came from
       | extend OfficeActivity_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.ClientIP
-  | where OfficeActivity_TimeGenerated >= TimeGenerated and OfficeActivity_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, OfficeActivity_TimeGenerated,
+  | where OfficeActivity_TimeGenerated < ExpirationDateTime
+  | summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId
+  | project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   TI_ipEntity, ClientIP, UserId, Operation, ResultStatus, RecordType, OfficeObjectId, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = OfficeActivity_TimeGenerated, IPCustomEntity = ClientIP, AccountCustomEntity = UserId, URLCustomEntity = Url
 entityMappings:

Detections/ThreatIntelligenceIndicator/IPEntity_VMConnection.yaml

@@ -34,16 +34,17 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       VMConnection
       | where TimeGenerated >= ago(dt_lookBack)
       // renaming time column so it is clear the log this came from
       | extend VMConnection_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.RemoteIp
-  | where VMConnection_TimeGenerated >= TimeGenerated and VMConnection_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, VMConnection_TimeGenerated,
+  | where VMConnection_TimeGenerated < ExpirationDateTime
+  | summarize VMConnection_TimeGenerated = arg_max(VMConnection_TimeGenerated, *) by IndicatorId, RemoteIp
+  | project VMConnection_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   TI_ipEntity, Computer, Direction, ProcessName, SourceIp, DestinationIp, RemoteIp, Protocol, DestinationPort, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = VMConnection_TimeGenerated, IPCustomEntity = RemoteIp, HostCustomEntity = Computer, URLCustomEntity = Url
 entityMappings:
@@ -55,6 +56,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: SourceIp
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/IPEntity_W3CIISLog.yaml

@@ -34,7 +34,8 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       W3CIISLog
       | where TimeGenerated >= ago(dt_lookBack)
       | where isnotempty(cIP)
@@ -42,10 +43,10 @@ query: |
       | extend W3CIISLog_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.cIP
-  | where W3CIISLog_TimeGenerated >= TimeGenerated and W3CIISLog_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  W3CIISLog_TimeGenerated, TI_ipEntity, Computer, sSiteName, cIP, sIP, sPort, csMethod, csUserName, scStatus, scSubStatus, scWin32Status,
+  | where W3CIISLog_TimeGenerated < ExpirationDateTime
+  | summarize W3CIISLog_TimeGenerated = arg_max(W3CIISLog_TimeGenerated, *) by IndicatorId, cIP
+  | project W3CIISLog_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  TI_ipEntity, Computer, sSiteName, cIP, sIP, sPort, csMethod, csUserName, scStatus, scSubStatus, scWin32Status,
   NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = W3CIISLog_TimeGenerated, IPCustomEntity = cIP, HostCustomEntity = Computer, AccountCustomEntity = csUserName, URLCustomEntity = Url
 entityMappings:
@@ -61,6 +62,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: sIP
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/IPEntity_WireData.yaml

@@ -34,16 +34,17 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
       WireData | where TimeGenerated >= ago(dt_lookBack)
       | where isnotempty(RemoteIP)
       // renaming time column so it is clear the log this came from
       | extend WireData_TimeGenerated = TimeGenerated
   )
   on $left.TI_ipEntity == $right.RemoteIP
-  | where WireData_TimeGenerated >= TimeGenerated and WireData_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, WireData_TimeGenerated,
+  | where WireData_TimeGenerated < ExpirationDateTime
+  | summarize WireData_TimeGenerated = arg_max(WireData_TimeGenerated, *) by IndicatorId, RemoteIP
+  | project WireData_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
   TI_ipEntity, Computer, LocalIP, RemoteIP, ProcessName, ApplicationProtocol, LocalPortNumber, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress
   | extend timestamp = WireData_TimeGenerated, IPCustomEntity = RemoteIP, HostCustomEntity = Computer, URLCustomEntity = Url
 entityMappings:
@@ -55,6 +56,8 @@ entityMappings:
     fieldMappings:
       - identifier: Address
         columnName: IPCustomEntity
+      - identifier: Address
+        columnName: LocalIP
   - entityType: URL
     fieldMappings:
       - identifier: Url

Detections/ThreatIntelligenceIndicator/IPentity_SigninLogs.yaml

@@ -38,6 +38,7 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
       table(tableName) | where TimeGenerated >= ago(dt_lookBack)
       | extend Status = todynamic(Status), LocationDetails = todynamic(LocationDetails)

Detections/ThreatIntelligenceIndicator/URLEntity_AuditLogs.yaml

@@ -29,7 +29,8 @@ query: |
   | where Active == true
   // Picking up only IOC's that contain the entities we want
   | where isnotempty(Url)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
     AuditLogs
     | where TimeGenerated >= ago(dt_lookBack)
     // Extract the URL that is contained within the JSON data
@@ -39,10 +40,10 @@ query: |
     | extend TargetResourceDisplayName = tostring(TargetResources[0].displayName)
     | extend Audit_TimeGenerated = TimeGenerated
   ) on Url
-  | where Audit_TimeGenerated >= TimeGenerated and Audit_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore,
-  Audit_TimeGenerated, OperationName, Identity, userPrincipalName, TargetResourceDisplayName, Url
+  | where Audit_TimeGenerated < ExpirationDateTime
+  | summarize Audit_TimeGenerated = arg_max(Audit_TimeGenerated, *) by IndicatorId, Url
+  | project Audit_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore,
+  OperationName, Identity, userPrincipalName, TargetResourceDisplayName, Url
   | extend timestamp = Audit_TimeGenerated, AccountCustomEntity = userPrincipalName, HostCustomEntity = TargetResourceDisplayName, URLCustomEntity = Url
 entityMappings:
   - entityType: Account

Detections/ThreatIntelligenceIndicator/URLEntity_OfficeActivity.yaml

@@ -26,7 +26,8 @@ query: |
   | where Active == true
   // Picking up only IOC's that contain the entities we want
   | where isnotempty(Url)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
    OfficeActivity
     | where TimeGenerated >= ago(dt_lookBack)
     //Extract the Url from a number of potential fields
@@ -38,10 +39,10 @@ query: |
     // Project a single user identity that we can use for entity mapping
     | extend User = iif(isnotempty(UserId), UserId, iif(isnotempty(Actor), tostring(parse_json(Actor)[0].ID), tostring(parse_json(Parameters)[0].Vlaue))) 
   ) on Url
-  | where OfficeActivity_TimeGenerated >= TimeGenerated and OfficeActivity_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Operation, 
-  UserType, OfficeWorkload, Parameters, OfficeActivity_TimeGenerated, Url, User
+  | where OfficeActivity_TimeGenerated < ExpirationDateTime
+  | summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId, Url
+  | project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Operation, 
+  UserType, OfficeWorkload, Parameters, Url, User
   | extend timestamp = OfficeActivity_TimeGenerated, AccountCustomEntity = User, URLCustomEntity = Url
 entityMappings:
   - entityType: Account

Detections/ThreatIntelligenceIndicator/URLEntity_PaloAlto.yaml

@@ -29,7 +29,8 @@ query: |
   | where Active == true
   // Picking up only IOC's that contain the entities we want
   | where isnotempty(Url)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
     CommonSecurityLog
     | extend IngestionTime = ingestion_time()
     | where IngestionTime > ago(dt_lookBack)
@@ -45,8 +46,9 @@ query: |
     | where isnotempty(PA_Url)
     | extend CommonSecurityLog_TimeGenerated = TimeGenerated
   ) on $left.Url == $right.PA_Url
-  | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, DeviceAction, SourceIP, CommonSecurityLog_TimeGenerated, PA_Url, DeviceName
+  | where CommonSecurityLog_TimeGenerated < ExpirationDateTime
+  | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId, PA_Url
+  | project CommonSecurityLog_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, DeviceAction, SourceIP, PA_Url, DeviceName
   | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
 entityMappings:
   - entityType: Host

Detections/ThreatIntelligenceIndicator/URLEntity_SecurityAlerts.yaml

@@ -32,7 +32,8 @@ query: |
   | where Active == true
   // Picking up only IOC's that contain the entities we want
   | where isnotempty(Url)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
     SecurityAlert
     | where TimeGenerated >= ago(dt_lookBack)
     | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false)
@@ -45,10 +46,9 @@ query: |
     | extend Compromised_Host = tostring(parse_json(ExtendedProperties).["Compromised Host"])
     | extend Alert_TimeGenerated = TimeGenerated
   ) on Url
-  | where Alert_TimeGenerated >= TimeGenerated and Alert_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Alert_TimeGenerated,
-  AlertName, AlertSeverity, Description, Url, Compromised_Host
+  | where Alert_TimeGenerated < ExpirationDateTime
+  | summarize Alert_TimeGenerated = arg_max(Alert_TimeGenerated, *) by IndicatorId, AlertName
+  | project Alert_TimeGenerated, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, AlertName, AlertSeverity, Description, Url, Compromised_Host
   | extend timestamp = Alert_TimeGenerated, HostCustomEntity = Compromised_Host, URLCustomEntity = Url
 entityMappings:
   - entityType: Host

Detections/ThreatIntelligenceIndicator/URLEntity_Syslog.yaml

@@ -29,7 +29,8 @@ query: |
   | where Active == true
   // Picking up only IOC's that contain the entities we want
   | where isnotempty(Url)
-  | join (
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
     Syslog
     | where TimeGenerated >= ago(dt_lookBack)
     // Extract URL from the Syslog message but only take messages that include URLs
@@ -37,9 +38,9 @@ query: |
     | where isnotempty(Url)
     | extend Syslog_TimeGenerated = TimeGenerated
   ) on Url
-  | where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, Url, HostIP
+  | where Syslog_TimeGenerated < ExpirationDateTime
+  | summarize Syslog_TimeGenerated  = arg_max(Syslog_TimeGenerated , *) by IndicatorId, Url
+  | project Syslog_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, SyslogMessage, Computer, ProcessName, Url, HostIP
   | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url
 entityMappings:
   - entityType: Host

Hunting Queries/ThreatIntelligenceIndicator/FileEntity_OfficeActivity.yaml

@@ -24,14 +24,28 @@ query: |
   | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
   | where Active == true
   | where isnotempty(FileName)
-  |  join (
-  OfficeActivity| where TimeGenerated between(starttime..endtime)
-        | where isnotempty(SourceFileName)
-        | extend OfficeActivity_TimeGenerated = TimeGenerated
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+    OfficeActivity
+    | where TimeGenerated between(starttime..endtime)
+    | where isnotempty(SourceFileName)
+    | extend OfficeActivity_TimeGenerated = TimeGenerated
   )
   on $left.FileName == $right.SourceFileName
-  | where OfficeActivity_TimeGenerated >= TimeGenerated and OfficeActivity_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  OfficeActivity_TimeGenerated, FileName, UserId, ClientIP, OfficeObjectId
-  | extend timestamp = OfficeActivity_TimeGenerated, AccountCustomEntity = UserId, IPCustomEntity = ClientIP, URLCustomEntity = Url
+  | where OfficeActivity_TimeGenerated < ExpirationDateTime
+  | summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId, SourceFileName
+  | project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, FileName, UserId, ClientIP, OfficeObjectId
+  | extend timestamp = OfficeActivity_TimeGenerated, AccountCustomEntity = UserId, IPCustomEntity = ClientIP, URLCustomEntity = Url
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: FullName
+        columnName: AccountCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: IPCustomEntity
+  - entityType: URL
+    fieldMappings:
+      - identifier: Url
+        columnName: URLCustomEntity

Hunting Queries/ThreatIntelligenceIndicator/FileEntity_SecurityEvent.yaml

@@ -24,15 +24,34 @@ query: |
   | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
   | where Active == true
   | where isnotempty(FileName)
-  |  join (
-    SecurityEvent | where TimeGenerated between(starttime..endtime)
-        | where EventID in ("4688","8002","4648","4673")
-                   | where isnotempty(Process)
-        | extend SecurityEvent_TimeGenerated = TimeGenerated, Event = EventID
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+    SecurityEvent 
+    | where TimeGenerated between(starttime..endtime)
+    | where EventID in ("4688","8002","4648","4673")
+    | where isnotempty(Process)
+    | extend SecurityEvent_TimeGenerated = TimeGenerated, Event = EventID
   )
   on $left.FileName == $right.Process
-  | where SecurityEvent_TimeGenerated >= TimeGenerated and SecurityEvent_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  SecurityEvent_TimeGenerated, FileName, Computer, IpAddress, Account, Event, Activity
-  | extend timestamp = SecurityEvent_TimeGenerated, AccountCustomEntity = Account, HostCustomEntity = Computer, IPCustomEntity = IpAddress, URLCustomEntity = Url
+  | where SecurityEvent_TimeGenerated < ExpirationDateTime
+  | summarize SecurityEvent_TimeGenerated = arg_max(SecurityEvent_TimeGenerated, *) by IndicatorId, Process
+  | project SecurityEvent_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  FileName, Computer, IpAddress, Account, Event, Activity
+  | extend timestamp = SecurityEvent_TimeGenerated, AccountCustomEntity = Account, HostCustomEntity = Computer, IPCustomEntity = IpAddress, URLCustomEntity = Url
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: FullName
+        columnName: AccountCustomEntity
+  - entityType: Host
+    fieldMappings:
+      - identifier: FullName
+        columnName: HostCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: IPCustomEntity
+  - entityType: URL
+    fieldMappings:
+      - identifier: Url
+        columnName: URLCustomEntity

Hunting Queries/ThreatIntelligenceIndicator/FileEntity_Syslog.yaml

@@ -25,14 +25,29 @@ query: |
   | where Active == true
   | where isnotempty(FileName)
   | extend TI_ProcessEntity = tostring(split(FileName, ".")[-2])
-  |  join (
-       Syslog | where TimeGenerated between(starttime..endtime)
-       | where isnotempty(ProcessName)
-       | extend Syslog_TimeGenerated = TimeGenerated
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+    Syslog
+    | where TimeGenerated between(starttime..endtime)
+    | where isnotempty(ProcessName)
+    | extend Syslog_TimeGenerated = TimeGenerated
   )
   on $left.TI_ProcessEntity == $right.ProcessName
-  | where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  Syslog_TimeGenerated, FileName, Computer, HostIP, SyslogMessage
-  | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url
+  | where Syslog_TimeGenerated < ExpirationDateTime
+  | summarize Syslog_TimeGenerated = arg_max(Syslog_TimeGenerated, *) by IndicatorId, ProcessName
+  | project Syslog_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  FileName, Computer, HostIP, SyslogMessage
+  | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url
+entityMappings:
+  - entityType: Host
+    fieldMappings:
+      - identifier: FullName
+        columnName: HostCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: IPCustomEntity
+  - entityType: URL
+    fieldMappings:
+      - identifier: Url
+        columnName: URLCustomEntity

Hunting Queries/ThreatIntelligenceIndicator/FileEntity_VMConnection.yaml

@@ -25,14 +25,31 @@ query: |
   | where Active == true
   | where isnotempty(FileName)
   | extend TI_ProcessEntity = tostring(split(FileName, ".")[-2])
-  |  join (
-     VMConnection | where TimeGenerated between(starttime..endtime)
-     | where isnotempty(ProcessName)
-     | extend VMConnection_TimeGenerated = TimeGenerated
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+    VMConnection 
+    | where TimeGenerated between(starttime..endtime)
+    | where isnotempty(ProcessName)
+    | extend VMConnection_TimeGenerated = TimeGenerated
   )
   on $left.TI_ProcessEntity == $right.ProcessName
-  | where VMConnection_TimeGenerated >= TimeGenerated and VMConnection_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  VMConnection_TimeGenerated, FileName, Computer, Direction, SourceIp, DestinationIp, RemoteIp, DestinationPort, Protocol
-  | extend timestamp = VMConnection_TimeGenerated, IPCustomEntity = RemoteIp, HostCustomEntity = Computer, URLCustomEntity = Url
+  | where VMConnection_TimeGenerated < ExpirationDateTime
+  | summarize VMConnection_TimeGenerated = arg_max(VMConnection_TimeGenerated, *) by IndicatorId, ProcessName
+  | project VMConnection_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  FileName, Computer, Direction, SourceIp, DestinationIp, RemoteIp, DestinationPort, Protocol
+  | extend timestamp = VMConnection_TimeGenerated, IPCustomEntity = RemoteIp, HostCustomEntity = Computer, URLCustomEntity = Url
+entityMappings:
+  - entityType: Host
+    fieldMappings:
+      - identifier: FullName
+        columnName: HostCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: IPCustomEntity
+      - identifier: Address
+        columnName: SourceIp
+  - entityType: URL
+    fieldMappings:
+      - identifier: Url
+        columnName: URLCustomEntity

Hunting Queries/ThreatIntelligenceIndicator/FileEntity_WireData.yaml

@@ -24,15 +24,32 @@ query: |
   | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
   | where Active == true
   | where isnotempty(FileName)
-  |  join (
-   WireData | where TimeGenerated between(starttime..endtime)
-            | where isnotempty(ProcessName)
-            | extend Process =reverse(substring(reverse(ProcessName), 0, indexof(reverse(ProcessName), "\\")))
-        | extend WireData_TimeGenerated = TimeGenerated
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+    WireData 
+    | where TimeGenerated between(starttime..endtime)
+    | where isnotempty(ProcessName)
+    | extend Process =reverse(substring(reverse(ProcessName), 0, indexof(reverse(ProcessName), "\\")))
+    | extend WireData_TimeGenerated = TimeGenerated
   )
   on $left.FileName == $right.Process
-  | where WireData_TimeGenerated >= TimeGenerated and WireData_TimeGenerated < ExpirationDateTime
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
-  WireData_TimeGenerated, FileName, Computer, Direction, LocalIP, RemoteIP, LocalPortNumber, RemotePortNumber
-  | extend timestamp = WireData_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = RemoteIP, URLCustomEntity = Url
+  | where WireData_TimeGenerated < ExpirationDateTime
+  | summarize WireData_TimeGenerated = arg_max(WireData_TimeGenerated, *) by IndicatorId, Process
+  | project WireData_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,
+  FileName, Computer, Direction, LocalIP, RemoteIP, LocalPortNumber, RemotePortNumber
+  | extend timestamp = WireData_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = RemoteIP, URLCustomEntity = Url
+entityMappings:
+  - entityType: Host
+    fieldMappings:
+      - identifier: FullName
+        columnName: HostCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: IPCustomEntity
+      - identifier: Address
+        columnName: LocalIP
+  - entityType: URL
+    fieldMappings:
+      - identifier: Url
+        columnName: URLCustomEntity

Hunting Queries/ThreatIntelligenceIndicator/Sample-DNSEventsMatchToThreatIntel.yaml

@@ -1,26 +0,0 @@
-id: a411fe4c-2ee0-4ee0-b579-55d74b6e7371
-name: Preview - DNS Events that match threat intelligence
-description: |
-  'This sample hunting query demonstrates how to utilize the threat intelligence data with the DNS event logs'
-severity: Medium
-requiredDataConnectors:
-  - connectorId: DNS
-    dataTypes:
-      - DnsEvents
-  - connectorId: ThreatIntelligence
-    dataTypes:
-      - ThreatIntelligenceIndicator
-  - connectorId: ThreatIntelligenceTaxii
-    dataTypes:
-      - ThreatIntelligenceIndicator
-tactics:
-  - Impact
-query: |
-
-  DnsEvents
-  | join (ThreatIntelligenceIndicator
-    | summarize arg_max(TimeGenerated, *) by IndicatorId
-    | summarize by Url) on $left.Name == $right.Url
-  | summarize StartTimeUtc = min(TimeGenerated), EndTimeUtc = max(TimeGenerated), count()
-  by Computer, ClientIP, ThreatIntel_Related_Domain = Name, Url
-  | extend timestamp = StartTimeUtc, HostCustomEntity = Computer, IPCustomEntity = ClientIP, URLCustomEntity = Url