fix kql

Solutions/ThreatConnect/Analytic Rules/ThreatConnect_DomainEntity_DnsEvents.yaml

@@ -32,7 +32,7 @@ query: |
   let Domain_Indicators = ThreatIntelligenceIndicator
     | where TimeGenerated >= ago(ioc_lookBack)
     | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | where Active == true and ExpirationDateTime > now()
+    | where ExpirationDateTime > now() and Active == true
     // Filter out non ThreatConnect TI Sources
     | where SourceSystem startswith "ThreatConnect-"
     // Filter out indicators without domain names

Solutions/ThreatConnect/Analytic Rules/ThreatConnect_EmailEntity_OfficeActivity.yaml

@@ -29,9 +29,9 @@ query: |
   let ioc_lookBack = 14d;
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
+    | where TimeGenerated >= ago(ioc_lookBack)
+    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+    | where ExpirationDateTime > now() and Active == true
   // Filter out non ThreatConnect TI Sources
   | where SourceSystem startswith "ThreatConnect-"
   //Filtering the table for Email related IOCs

Solutions/ThreatConnect/Analytic Rules/ThreatConnect_EmailEntity_SigninLogs.yaml

@@ -33,9 +33,9 @@ query: |
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   let aadFunc = (tableName:string){
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
+    | where TimeGenerated >= ago(ioc_lookBack)
+    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+    | where ExpirationDateTime > now() and Active == true
   // Filter out non ThreatConnect TI Sources
   | where SourceSystem startswith "ThreatConnect-"
   //Filtering the table for Email related IOCs

Solutions/ThreatConnect/Package/mainTemplate.json

@@ -182,7 +182,7 @@
                 "description": "Identifies a match in DnsEvents from any ThreatConnect Domain IOC from TI",
                 "displayName": "Threat Connect TI map Domain entity to DnsEvents",
                 "enabled": false,
-                "query": "// Define the lookback periods for time-based filters\nlet dt_lookBack = 1h; // Look back 1 hour for DNS events\nlet ioc_lookBack = 7d; // Look back 7 days for threat intelligence indicators\n// Fetch threat intelligence indicators related to domains\nlet Domain_Indicators = ThreatIntelligenceIndicator\n  | where TimeGenerated >= ago(ioc_lookBack)\n  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n  | where Active == true and ExpirationDateTime > now()\n  // Filter out non ThreatConnect TI Sources\n  | where SourceSystem startswith \"ThreatConnect-\"\n  // Filter out indicators without domain names\n  | where isnotempty(DomainName)\n  | extend TI_DomainEntity = DomainName;\n// Create a list of TLDs in our threat feed for later validation\nlet maxListSize = 100000; // Define the maximum allowed size for each list\nlet list_tlds = Domain_Indicators\n  | extend parts = split(DomainName, '.')\n  | extend tld = parts[(array_length(parts)-1)]\n  | summarize count() by tostring(tld)\n  | project tld\n  | summarize make_list(tld, maxListSize);\n// Perform a join between domain indicators and DNS events to identify potential malicious activity\nDomain_Indicators\n  // Use innerunique to keep performance fast and result set low, as we only need one match to indicate potential malicious activity that needs investigation\n  | join kind=innerunique (\n    DnsEvents\n    | where TimeGenerated > ago(dt_lookBack)\n    // Extract domain patterns from syslog message\n    | where isnotempty(Name)\n    | extend parts = split(Name, '.')\n    | extend tld = parts[(array_length(parts)-1)]\n    // Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed\n    | where tld in~ (list_tlds)\n    | extend DNS_TimeGenerated = TimeGenerated\n  ) on $left.TI_DomainEntity==$right.Name\n  // Filter out DNS events that occurred after the expiration of the corresponding indicator\n  | where DNS_TimeGenerated < ExpirationDateTime\n  // Group the results by IndicatorId and Name, and keep the DNS event with the latest timestamp\n  | summarize DNS_TimeGenerated = arg_max(DNS_TimeGenerated, *) by IndicatorId, Name\n  // Select the desired output fields\n  | project DNS_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, Computer, ClientIP, Name, QueryType, Type, TI_DomainEntity\n  // Extract hostname and DNS domain from the Computer field\n  | extend HostName = tostring(split(Computer, '.', 0)[0]), DnsDomain = tostring(strcat_array(array_slice(split(Computer, '.'), 1, -1), '.'))\n  // Rename the timestamp field\n  | extend timestamp = DNS_TimeGenerated\n",
+                "query": "// Define the lookback periods for time-based filters\nlet dt_lookBack = 1h; // Look back 1 hour for DNS events\nlet ioc_lookBack = 7d; // Look back 7 days for threat intelligence indicators\n// Fetch threat intelligence indicators related to domains\nlet Domain_Indicators = ThreatIntelligenceIndicator\n  | where TimeGenerated >= ago(ioc_lookBack)\n  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n  | where ExpirationDateTime > now() and Active == true\n  // Filter out non ThreatConnect TI Sources\n  | where SourceSystem startswith \"ThreatConnect-\"\n  // Filter out indicators without domain names\n  | where isnotempty(DomainName)\n  | extend TI_DomainEntity = DomainName;\n// Create a list of TLDs in our threat feed for later validation\nlet maxListSize = 100000; // Define the maximum allowed size for each list\nlet list_tlds = Domain_Indicators\n  | extend parts = split(DomainName, '.')\n  | extend tld = parts[(array_length(parts)-1)]\n  | summarize count() by tostring(tld)\n  | project tld\n  | summarize make_list(tld, maxListSize);\n// Perform a join between domain indicators and DNS events to identify potential malicious activity\nDomain_Indicators\n  // Use innerunique to keep performance fast and result set low, as we only need one match to indicate potential malicious activity that needs investigation\n  | join kind=innerunique (\n    DnsEvents\n    | where TimeGenerated > ago(dt_lookBack)\n    // Extract domain patterns from syslog message\n    | where isnotempty(Name)\n    | extend parts = split(Name, '.')\n    | extend tld = parts[(array_length(parts)-1)]\n    // Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed\n    | where tld in~ (list_tlds)\n    | extend DNS_TimeGenerated = TimeGenerated\n  ) on $left.TI_DomainEntity==$right.Name\n  // Filter out DNS events that occurred after the expiration of the corresponding indicator\n  | where DNS_TimeGenerated < ExpirationDateTime\n  // Group the results by IndicatorId and Name, and keep the DNS event with the latest timestamp\n  | summarize DNS_TimeGenerated = arg_max(DNS_TimeGenerated, *) by IndicatorId, Name\n  // Select the desired output fields\n  | project DNS_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, Computer, ClientIP, Name, QueryType, Type, TI_DomainEntity\n  // Extract hostname and DNS domain from the Computer field\n  | extend HostName = tostring(split(Computer, '.', 0)[0]), DnsDomain = tostring(strcat_array(array_slice(split(Computer, '.'), 1, -1), '.'))\n  // Rename the timestamp field\n  | extend timestamp = DNS_TimeGenerated\n",
                 "queryFrequency": "PT1H",
                 "queryPeriod": "P7D",
                 "severity": "Medium",
@@ -322,7 +322,7 @@
                 "description": "Identifies a match in OfficeActivity table from any Email IOC from ThreatConnect TI",
                 "displayName": "ThreatConnect TI map Email entity to OfficeActivity",
                 "enabled": false,
-                "query": "let dt_lookBack = 1h;\nlet ioc_lookBack = 14d;\nlet emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$';\nThreatIntelligenceIndicator\n| where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()\n| summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n| where Active == true\n// Filter out non ThreatConnect TI Sources\n| where SourceSystem startswith \"ThreatConnect-\"\n//Filtering the table for Email related IOCs\n| where isnotempty(EmailSenderAddress)\n// using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated\n| join kind=innerunique (\n    OfficeActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserId)\n    | where UserId matches regex emailregex\n    | extend OfficeActivity_TimeGenerated = TimeGenerated\n)\non $left.EmailSenderAddress == $right.UserId\n| where OfficeActivity_TimeGenerated < ExpirationDateTime\n| summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId, UserId\n| project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,\nEmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, UserId, ClientIP, Operation, UserType, RecordType, OfficeWorkload, Parameters\n| extend Name = tostring(split(UserId, '@', 0)[0]), UPNSuffix = tostring(split(UserId, '@', 1)[0])\n| extend timestamp = OfficeActivity_TimeGenerated\n",
+                "query": "let dt_lookBack = 1h;\nlet ioc_lookBack = 14d;\nlet emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$';\nThreatIntelligenceIndicator\n  | where TimeGenerated >= ago(ioc_lookBack)\n  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n  | where ExpirationDateTime > now() and Active == true\n// Filter out non ThreatConnect TI Sources\n| where SourceSystem startswith \"ThreatConnect-\"\n//Filtering the table for Email related IOCs\n| where isnotempty(EmailSenderAddress)\n// using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated\n| join kind=innerunique (\n    OfficeActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserId)\n    | where UserId matches regex emailregex\n    | extend OfficeActivity_TimeGenerated = TimeGenerated\n)\non $left.EmailSenderAddress == $right.UserId\n| where OfficeActivity_TimeGenerated < ExpirationDateTime\n| summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId, UserId\n| project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,\nEmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, UserId, ClientIP, Operation, UserType, RecordType, OfficeWorkload, Parameters\n| extend Name = tostring(split(UserId, '@', 0)[0]), UPNSuffix = tostring(split(UserId, '@', 1)[0])\n| extend timestamp = OfficeActivity_TimeGenerated\n",
                 "queryFrequency": "PT1H",
                 "queryPeriod": "P14D",
                 "severity": "Medium",
@@ -462,7 +462,7 @@
                 "description": "Identifies a match in SigninLogs table from any Email IOC from ThreatConnect TI",
                 "displayName": "ThreatConnect TI map Email entity to SigninLogs",
                 "enabled": false,
-                "query": "let dt_lookBack = 1h;\nlet ioc_lookBack = 14d;\nlet emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$';\nlet aadFunc = (tableName:string){\nThreatIntelligenceIndicator\n| where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()\n| summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n| where Active == true\n// Filter out non ThreatConnect TI Sources\n| where SourceSystem startswith \"ThreatConnect-\"\n//Filtering the table for Email related IOCs\n| where isnotempty(EmailSenderAddress)\n// using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated\n| join kind=innerunique (\n    table(tableName) | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserPrincipalName)\n    //Normalizing the column to lower case for exact match with EmailSenderAddress column\n    | extend UserPrincipalName = tolower(UserPrincipalName)\n    | where UserPrincipalName matches regex emailregex\n    | extend Status = todynamic(DeviceDetail), LocationDetails = todynamic(LocationDetails)\n    | extend StatusCode = tostring(Status.errorCode), StatusDetails = tostring(Status.additionalDetails)\n    | extend State = tostring(LocationDetails.state), City = tostring(LocationDetails.city), Region = tostring(LocationDetails.countryOrRegion)\n    // renaming timestamp column so it is clear the log this came from SigninLogs table\n    | extend SigninLogs_TimeGenerated = TimeGenerated, Type = Type\n)\non $left.EmailSenderAddress == $right.UserPrincipalName\n| where SigninLogs_TimeGenerated < ExpirationDateTime\n| summarize SigninLogs_TimeGenerated = arg_max(SigninLogs_TimeGenerated, *) by IndicatorId, UserPrincipalName\n| project SigninLogs_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,\nEmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, IPAddress, UserPrincipalName, AppDisplayName,\nStatusCode, StatusDetails, NetworkIP, NetworkDestinationIP, NetworkSourceIP, Type\n| extend Name = tostring(split(UserPrincipalName, '@', 0)[0]), UPNSuffix = tostring(split(UserPrincipalName, '@', 1)[0])\n| extend timestamp = SigninLogs_TimeGenerated\n};\nlet aadSignin = aadFunc(\"SigninLogs\");\nlet aadNonInt = aadFunc(\"AADNonInteractiveUserSignInLogs\");\nunion isfuzzy=true aadSignin, aadNonInt\n",
+                "query": "let dt_lookBack = 1h;\nlet ioc_lookBack = 14d;\nlet emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$';\nlet aadFunc = (tableName:string){\nThreatIntelligenceIndicator\n  | where TimeGenerated >= ago(ioc_lookBack)\n  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n  | where ExpirationDateTime > now() and Active == true\n// Filter out non ThreatConnect TI Sources\n| where SourceSystem startswith \"ThreatConnect-\"\n//Filtering the table for Email related IOCs\n| where isnotempty(EmailSenderAddress)\n// using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated\n| join kind=innerunique (\n    table(tableName) | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserPrincipalName)\n    //Normalizing the column to lower case for exact match with EmailSenderAddress column\n    | extend UserPrincipalName = tolower(UserPrincipalName)\n    | where UserPrincipalName matches regex emailregex\n    | extend Status = todynamic(DeviceDetail), LocationDetails = todynamic(LocationDetails)\n    | extend StatusCode = tostring(Status.errorCode), StatusDetails = tostring(Status.additionalDetails)\n    | extend State = tostring(LocationDetails.state), City = tostring(LocationDetails.city), Region = tostring(LocationDetails.countryOrRegion)\n    // renaming timestamp column so it is clear the log this came from SigninLogs table\n    | extend SigninLogs_TimeGenerated = TimeGenerated, Type = Type\n)\non $left.EmailSenderAddress == $right.UserPrincipalName\n| where SigninLogs_TimeGenerated < ExpirationDateTime\n| summarize SigninLogs_TimeGenerated = arg_max(SigninLogs_TimeGenerated, *) by IndicatorId, UserPrincipalName\n| project SigninLogs_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore,\nEmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, IPAddress, UserPrincipalName, AppDisplayName,\nStatusCode, StatusDetails, NetworkIP, NetworkDestinationIP, NetworkSourceIP, Type\n| extend Name = tostring(split(UserPrincipalName, '@', 0)[0]), UPNSuffix = tostring(split(UserPrincipalName, '@', 1)[0])\n| extend timestamp = SigninLogs_TimeGenerated\n};\nlet aadSignin = aadFunc(\"SigninLogs\");\nlet aadNonInt = aadFunc(\"AADNonInteractiveUserSignInLogs\");\nunion isfuzzy=true aadSignin, aadNonInt\n",
                 "queryFrequency": "PT1H",
                 "queryPeriod": "P14D",
                 "severity": "Medium",
@@ -735,7 +735,7 @@
                 "description": "This query identifies any URL indicators of compromise (IOCs) from threat intelligence (TI) by searching for matches in OfficeActivity data.",
                 "displayName": "ThreatConnect TI Map URL Entity to OfficeActivity Data",
                 "enabled": false,
-                "query": "let dt_lookBack = 1h;\nlet ioc_lookBack = 14d;\nThreatIntelligenceIndicator\n| where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()\n| summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n| where Active == true\n// Filter out non ThreatConnect TI Sources\n| where SourceSystem startswith \"ThreatConnect-\"\n// Picking up only IOC's that contain the entities we want\n| where isnotempty(Url)\n// using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated\n| join kind=innerunique (\n  OfficeActivity\n  | where TimeGenerated >= ago(dt_lookBack)\n  //Extract the Url from a number of potential fields\n  | extend Url = iif(OfficeWorkload == \"AzureActiveDirectory\",extract(\"(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\\\(\\\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+);\", 1,ModifiedProperties),tostring(parse_json(ModifiedProperties)[12].NewValue))\n  | where isnotempty(Url)\n  // Ensure we get a clean URL\n  | extend Url = tostring(split(Url, ';')[0])\n  | extend OfficeActivity_TimeGenerated = TimeGenerated\n  // Project a single user identity that we can use for entity mapping\n  | extend User = iif(isnotempty(UserId), UserId, iif(isnotempty(Actor), tostring(parse_json(Actor)[0].ID), tostring(parse_json(Parameters)[0].Value)))\n) on Url\n| where OfficeActivity_TimeGenerated < ExpirationDateTime\n| summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId, Url\n| project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Operation,\nUserType, OfficeWorkload, Parameters, Url, User\n| extend timestamp = OfficeActivity_TimeGenerated, Name = tostring(split(User, '@', 0)[0]), UPNSuffix = tostring(split(User, '@', 1)[0])\n",
+                "query": "let dt_lookBack = 1h;\nlet ioc_lookBack = 14d;\n  ThreatIntelligenceIndicator\n  | where TimeGenerated >= ago(ioc_lookBack)\n  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n  | where ExpirationDateTime > now() and Active == true\n// Filter out non ThreatConnect TI Sources\n| where SourceSystem startswith \"ThreatConnect-\"\n// Picking up only IOC's that contain the entities we want\n| where isnotempty(Url)\n// using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated\n| join kind=innerunique (\n  OfficeActivity\n  | where TimeGenerated >= ago(dt_lookBack)\n  //Extract the Url from a number of potential fields\n  | extend Url = iif(OfficeWorkload == \"AzureActiveDirectory\",extract(\"(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\\\(\\\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+);\", 1,ModifiedProperties),tostring(parse_json(ModifiedProperties)[12].NewValue))\n  | where isnotempty(Url)\n  // Ensure we get a clean URL\n  | extend Url = tostring(split(Url, ';')[0])\n  | extend OfficeActivity_TimeGenerated = TimeGenerated\n  // Project a single user identity that we can use for entity mapping\n  | extend User = iif(isnotempty(UserId), UserId, iif(isnotempty(Actor), tostring(parse_json(Actor)[0].ID), tostring(parse_json(Parameters)[0].Value)))\n) on Url\n| where OfficeActivity_TimeGenerated < ExpirationDateTime\n| summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId, Url\n| project OfficeActivity_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Operation,\nUserType, OfficeWorkload, Parameters, Url, User\n| extend timestamp = OfficeActivity_TimeGenerated, Name = tostring(split(User, '@', 0)[0]), UPNSuffix = tostring(split(User, '@', 1)[0])\n",
                 "queryFrequency": "PT1H",
                 "queryPeriod": "P14D",
                 "severity": "Medium",