Merge pull request #4712 from Azure/4R9UN--Private-Ip-Address-exclusion

Update GitLab_MaliciousIP.yaml

Detections/ThreatIntelligenceIndicator/IPEntity_AppServiceHTTPLogs.yaml

@@ -31,6 +31,8 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
+  //Exclude local addresses, using the ipv4_is_private operator
+  | where ipv4_is_private(TI_ipEntity) == false and  TI_ipEntity !startswith "fe80" and TI_ipEntity !startswith "::" and TI_ipEntity !startswith "127."
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
     AppServiceHTTPLogs | where TimeGenerated >= ago(dt_lookBack)
@@ -66,5 +68,5 @@ entityMappings:
     fieldMappings:
       - identifier: ResourceId
         columnName: _ResourceId
-version: 1.2.2
+version: 1.2.3
 kind: Scheduled

Detections/ThreatIntelligenceIndicator/IPEntity_W3CIISLog.yaml

@@ -34,11 +34,15 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
+  //Exclude local addresses, using the ipv4_is_private operator
+  | where ipv4_is_private(TI_ipEntity) == false and  TI_ipEntity !startswith "fe80" and TI_ipEntity !startswith "::" and TI_ipEntity !startswith "127."
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
       W3CIISLog
       | where TimeGenerated >= ago(dt_lookBack)
       | where isnotempty(cIP)
+      //Exclude local addresses, using the ipv4_is_private operator
+      | where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."
       // renaming time column so it is clear the log this came from
       | extend W3CIISLog_TimeGenerated = TimeGenerated
   )
@@ -66,5 +70,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: URLCustomEntity
-version: 1.2.1
-kind: Scheduled
+version: 1.2.2
+kind: Scheduled

Hunting Queries/W3CIISLog/ClientIPwithManyUserAgents.yaml

@@ -18,6 +18,8 @@ query: |
 
   W3CIISLog
   | where scStatus !startswith "20" and scStatus !startswith "30" and cIP !startswith "192.168." and cIP != sIP and cIP != "::1"
+  //Exclude local addresses, using ipv4_is_private operator
+  |where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."
   | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), makeset(csUserAgent), ConnectionCount = count() 
   by Computer, sSiteName, sIP, sPort, cIP, csMethod
   | extend csUserAgentPerIPCount = arraylength(set_csUserAgent)

Hunting Queries/W3CIISLog/ExchangeServerProxyLogonURI.yaml

@@ -15,7 +15,8 @@ tags:
 query: |
 
   W3CIISLog
-  | where not(ipv4_is_private(cIP))
+  //Exclude local addresses, using the ipv4_is_private operator
+  | where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."
   | where (csUriStem matches regex @"\/owa\/auth\/[A-Za-z0-9]{1,30}\.js") or (csUriStem matches regex @"\/ecp\/[A-Za-z0-9]{1,30}\.(js|flt|css)")
   | project TimeGenerated, sSiteName, csMethod, csUriStem, sPort, sIP, cIP, csUserAgent
   | extend timestamp = TimeGenerated
@@ -27,4 +28,4 @@ entityMappings:
     - identifier: DestinationPort
       columnName: sPort
     - identifier: SourceAddress
-      columnName: cIP
+      columnName: cIP

Hunting Queries/W3CIISLog/RareUserAgentStrings.yaml

@@ -14,8 +14,8 @@ relevantTechniques:
 query: |
 
   W3CIISLog
-  // The below line can be used to exclude local IPs if these create noise
-  //| where cIP !startswith "192.168." and cIP != "::1"
+  //Exclude local addresses, using ipv4_is_private operator
+  |where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."  
   | where isnotempty(csUserAgent) and csUserAgent !in~ ("-", "MSRPC")
   | extend csUserAgent_size = string_size(csUserAgent)
   | project TimeGenerated, sSiteName, sPort, csUserAgent, csUserAgent_size, csUserName , csMethod, csUriStem, sIP, cIP, scStatus, 
@@ -23,7 +23,7 @@ query: |
   | join (
       W3CIISLog
   	  // The below line can be used to exclude local IPs if these create noise
-      //| where cIP !startswith "192.168." and cIP != "::1"
+      |where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."  
       | where isnotempty(csUserAgent) and csUserAgent !in~ ("-", "MSRPC") 
       | extend csUserAgent_size = string_size(csUserAgent)
       | summarize csUserAgent_count = count() by bin(csUserAgent_size, 1)
@@ -32,4 +32,4 @@ query: |
   | project TimeGenerated, sSiteName, sPort, sIP, cIP, csUserAgent, csUserAgent_size, csUserAgent_count, csUserName , csMethod, csUriStem, 
   scStatus, scSubStatus, scWin32Status, csHost
   | extend timestamp = TimeGenerated, IPCustomEntity = cIP, HostCustomEntity = csHost, AccountCustomEntity = csUserName 
-  
+  

Hunting Queries/W3CIISLog/SuspectedMailBoxExportHostonOWA.yaml

@@ -33,6 +33,8 @@ query: |
   | extend file_name = tostring(split(csUriStem, "/")[-1])
   | where file_name != ""
   | where cIP !in~ (excludeIps)
+  //Exclude local addresses, using the ipv4_is_private operator
+  | where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."
   | project file_ext, csUriStem, file_name, Computer, cIP, sIP, TenantId, TimeGenerated
   | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), dcount(cIP), AccessingIPs=make_set(cIP), AccessTimes=make_set(TimeGenerated), Access=count() by TenantId, file_name, Computer, csUriStem
   //Collection of the exfiltration will occur only once, lets check for 2 accesses in case they mess up

Hunting Queries/W3CIISLog/SuspectedProxyTokenExploitation.yaml

@@ -13,7 +13,8 @@ relevantTechniques:
   - T1190
 query: |
   W3CIISLog
-  | where not(ipv4_is_private(cIP))
+  //Exclude local addresses, using the ipv4_is_private operator
+  | where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."
   | where csMethod =~ "POST"
   | where csUriStem has "/ecp"
   | where isnotempty(csCookie) and csCookie has "SecurityToken"

Hunting Queries/W3CIISLog/WebShellActivity.yaml

@@ -25,7 +25,8 @@ query: |
   let lookback_period = (
   W3CIISLog
   | where TimeGenerated between (lookback .. starttime)
-  | where not(ipv4_is_private(cIP)) and cIP != "127.0.0.1"
+  //Exclude local addresses, using the ipv4_is_private operator
+  | where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."
   | summarize count() by cIP, csUserAgent
   | project cIP, csUserAgent
   );
@@ -36,7 +37,8 @@ query: |
   | where csUriStem !in~ (ignore_uristems) // Remove noisy uri stems in the final results by editing the ignore_uristems variable
   | extend suffix = strcat(".", split(split(csUriStem, "/")[-1], ".")[-1])
   | extend is_script = iff(suffix in (script_extensions), 1, 0)
-  | where not(ipv4_is_private(cIP)) and cIP != "127.0.0.1"
+  //Exclude local addresses using ipv4_is_private operator
+  |where ipv4_is_private(cIP) == false and  cIP !startswith "fe80" and cIP !startswith "::" and cIP !startswith "127."
   | extend status_xx = strcat(substring(tostring(scStatus), 0, 1), 'XX')
   | serialize cIP, csUserAgent, TimeGenerated
   | extend SessionStarted = row_window_session(TimeGenerated, 30s, 3s, (cIP != prev(cIP)) and (csUserAgent != prev(csUserAgent))));
@@ -56,4 +58,4 @@ query: |
   | sort by dyn_to_non_dyn_ratio desc, num_dynamic_scripts desc
   | extend summary = pack('IPCustomEntity', cIP, 'user_agent', csUserAgent, 'num_dynamic_scripts', num_dynamic_scripts, 'set_dynamic_scripts', set_dynamic_scripts, 'num_non_dyn_scripts', num_non_dyn_scripts, 'set_non_dynamic_scripts', set_non_dynamic_scripts, 'ratio', dyn_to_non_dyn_ratio, 'Session_StartTime', SessionStarted)
   | summarize summaries=make_list(summary), num_of_sessions_on_day = count() by cIP, csUserAgent
-  | sort by num_of_sessions_on_day asc
+  | sort by num_of_sessions_on_day asc

Solutions/GitLab/Analytic Rules/GitLab_MaliciousIP.yaml

@@ -30,6 +30,8 @@ query: |
   | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
   | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
+  //Exclude local addresses using ipv4_is_private operator
+  |where ipv4_is_private(TI_ipEntity) == false and  TI_ipEntity !startswith "fe80" and TI_ipEntity !startswith "::" and TI_ipEntity !startswith "127."
   | join (GitLabAccess) on $left.TI_ipEntity == $right.IPAddress
   | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
   | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, TimeGenerated = EventTime, TI_ipEntity, IPAddress, URI
@@ -42,5 +44,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.0.0
+version: 1.0.1
 kind: Scheduled