Merge pull request #10693 from jayeshprajapaticrest/CofenseTriage400Fix

Updated PR with the fix of 400 Bad Request issue while creating the Threat Indicator in Sentinel.

.script/tests/KqlvalidationsTests/CustomTables/Cofense_Triage_failed_indicators_CL.json

@@ -0,0 +1,81 @@
+{
+    "Name":"Cofense_Triage_failed_indicators_CL",
+    "Properties":[
+    {
+      "Name": "TenantId",
+      "Type": "string"
+    },
+    {
+      "Name": "SourceSystem",
+      "Type": "string"
+    },
+    {
+      "Name": "MG",
+      "Type": "string"
+    },
+    {
+      "Name": "Computer",
+      "Type": "string"
+    },
+    {
+      "Name": "RawData",
+      "Type": "string"
+    },
+    {
+      "Name": "ManagementGroupName",
+      "Type": "string"
+    },
+    {
+      "Name": "TimeGenerated",
+      "Type": "datetime"
+    },
+    {
+      "Name": "kind_s",
+      "Type": "string"
+    },
+    {
+      "Name": "properties_source_s",
+      "Type": "string"
+    },
+    {
+      "Name": "properties_displayName_s",
+      "Type": "string"
+    },
+    {
+      "Name": "properties_confidence_d",
+      "Type": "real"
+    },
+    {
+      "Name": "properties_patternType_s",
+      "Type": "string"
+    },
+    {
+      "Name": "properties_threatTypes_s",
+      "Type": "string"
+    },
+    {
+      "Name": "properties_pattern_s",
+      "Type": "string"
+    },
+    {
+      "Name": "properties_created_t",
+      "Type": "datetime"
+    },
+    {
+      "Name": "properties_externalLastUpdatedTimeUtc_t",
+      "Type": "datetime"
+    },
+    {
+      "Name": "report_link_s",
+      "Type": "string"
+    },
+    {
+      "Name": "Type",
+      "Type": "string"
+    },
+    {
+      "Name": "_ResourceId",
+      "Type": "string"
+    }
+  ]
+  }

Sample Data/Custom/CofenseTriage/Cofense_Triage_failed_indicators_CL.csv

@@ -0,0 +1,3 @@
+TimeGenerated [UTC],kind_s,properties_source_s,properties_displayName_s,properties_confidence_d,properties_patternType_s,properties_threatTypes_s,properties_pattern_s,properties_created_t [UTC],properties_externalLastUpdatedTimeUtc_t [UTC],report_link_s,Type
+"6/18/2024, 5:52:08.370 PM",indicator,Cofense :  newww,Cofense Triage : 123,1,domain-name,"[""domain-name""]",[domain-name:value = 'new.sh'],"10/5/2023, 9:14:18.558 AM","2/22/2024, 9:52:53.639 AM",https://test.com/api/public/v2/threat_indicators/21289/reports,Cofense_Triage_failed_indicators_CL
+"6/18/2024, 5:54:12.843 PM",indicator,Cofense : vvvv,Cofense Triage : 12,1,file,"[""file""]",[file:hashes.'SHA-256' = '12345f30c1fe514991b7054889a3633f3c679fc524367811d8be1e44b08809an'],"11/18/2020, 9:32:53.225 PM","4/29/2024, 10:18:05.352 AM",https://test.com/api/public/v2/threat_indicators/54/reports,Cofense_Triage_failed_indicators_CL

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/CofenseBasedIndicatorCreator/cofense_to_sentinel_mapping.py

@@ -1,4 +1,5 @@
 """This file contains implementation of mapping of cofense data to sentinel indicator."""
+
 import json
 import inspect
 import datetime
@@ -15,6 +16,7 @@ from ..SharedCode.consts import (
     COFENSE_PAGE_NUMBER,
     COFENSE_PAGE_SIZE,
     DATETIMEFORMAT,
+    FAILED_INDICATORS_FILE_SHARE,
 )
 from ..SharedCode.logger import applogger
 from ..SharedCode.cofense_exception import CofenseException
@@ -32,20 +34,54 @@ class CofenseToSentinelMapping:
         self.cofense_triage_obj = CofenseTriage()
         self.microsoft_obj = MicrosoftSentinel()
         self.log_type = REPORTS_TABLE_NAME
-        self.state_obj = StateManager(
-            connection_string=CONNECTION_STRING, file_path="cofense"
-        )
+        self.state_obj = StateManager(connection_string=CONNECTION_STRING, file_path="cofense")
         self.cofense_page_number = "page[number]"
+        self.state_obj_failed_indicators = StateManager(
+            connection_string=CONNECTION_STRING,
+            file_path=str(int(datetime.datetime.now(datetime.timezone.utc).timestamp())),
+            share_name=FAILED_INDICATORS_FILE_SHARE,
+        )
+
+    def save_failed_indicators(self, indicator_data):
+        """Save failed indicators data to checkpoint.
+
+        Args:
+            indicators_data (list): Failed indicators data.
+        """
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            failed_indicators = self.state_obj_failed_indicators.get(COFENSE_TO_SENTINEL)
+            indicator_id = indicator_data.get("id", "")
+            if failed_indicators is None or failed_indicators == "":
+                self.state_obj_failed_indicators.post(json.dumps([indicator_data]))
+            else:
+                json_failed_indicators = json.loads(failed_indicators)
+                json_failed_indicators.extend([indicator_data])
+                self.state_obj_failed_indicators.post(json.dumps(json_failed_indicators))
+
+            applogger.info(
+                "{}(method={}) : {} : saved failed indicator successful, indicatorId : {}.".format(
+                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, indicator_id
+                )
+            )
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : error while saving failed indicator, indicatorId : {}, error : {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    COFENSE_TO_SENTINEL,
+                    indicator_id,
+                    error,
+                )
+            )
+            raise CofenseException()
 
     def source_mapping(self, indicator):
         """To map cofense source with sentinel source for microsoft sentinel indicator data."""
         response_source = indicator.get("attributes", {}).get("threat_source", "")
         splitted_source = response_source.split(":")
         source = None
-        if (
-            splitted_source[0].lower().strip()
-            != SENTINEL_SOURCE_PREFIX.split(":")[0].lower().strip()
-        ):
+        if splitted_source[0].lower().strip() != SENTINEL_SOURCE_PREFIX.split(":")[0].lower().strip():
             source = COFENSE_SOURCE_PREFIX + response_source
         return source
 
@@ -61,9 +97,7 @@ class CofenseToSentinelMapping:
             threat_type = "file"
         return threat_type
 
-    def pattern_mapping(
-        self, threat_type, indicator_threat_value, indicator_threat_type
-    ):
+    def pattern_mapping(self, threat_type, indicator_threat_value, indicator_threat_type):
         """To map threat value with pattern for microsoft sentinel indicator data."""
         pattern = ""
         if threat_type == "url":
@@ -75,21 +109,30 @@ class CofenseToSentinelMapping:
             pattern = "[domain-name:value = '{}']".format(indicator_threat_value)
         elif threat_type == "file":
             if indicator_threat_type == "SHA256":
-                pattern = "[file:hashes.'SHA-256' = '{}']".format(
-                    indicator_threat_value
-                )
+                pattern = "[file:hashes.'SHA-256' = '{}']".format(indicator_threat_value)
             elif indicator_threat_type == "MD5":
                 pattern = "[file:hashes.'MD5' = '{}']".format(indicator_threat_value)
         return pattern
 
     def report_link_mapping(self, indicator):
         """To get the report link from the cofense indicator."""
-        report_link = (
-            indicator.get("relationships", {})
-            .get("reports", {})
-            .get("links", {})
-            .get("related", "")
-        )
+        __method_name = inspect.currentframe().f_code.co_name
+        report_link = indicator.get("relationships", {}).get("reports", {}).get("links", {}).get("related", "")
+        if report_link:
+            applogger.debug(
+                "{}(method={}) : {} : report link : {}".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    COFENSE_TO_SENTINEL,
+                    report_link,
+                )
+            )
+        else:
+            applogger.warning(
+                "{}(method={}) : {} : no report link found.".format(
+                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL
+                )
+            )
         return report_link
 
     def data_mapping(self, indicator_data):
@@ -117,9 +160,7 @@ class CofenseToSentinelMapping:
                     data = {}
                     continue
                 indicator_id = indicator.get("id", "")
-                data["properties"]["displayName"] = "Cofense Triage : {}".format(
-                    indicator_id
-                )
+                data["properties"]["displayName"] = "Cofense Triage : {}".format(indicator_id)
                 indicator_threat_level = indicator.get("attributes", {}).get("threat_level", "")
                 threat_level = cofense_to_sentinel_threat_level_mapping(indicator_threat_level, COFENSE_TO_SENTINEL)
                 data["properties"]["confidence"] = threat_level
@@ -142,9 +183,7 @@ class CofenseToSentinelMapping:
                     continue
                 data["properties"]["threatTypes"] = [threat_type]
                 indicator_threat_value = indicator.get("attributes", {}).get("threat_value", "")
-                pattern = self.pattern_mapping(
-                    threat_type, indicator_threat_value, indicator_threat_type
-                )
+                pattern = self.pattern_mapping(threat_type, indicator_threat_value, indicator_threat_type)
                 if pattern:
                     data["properties"]["pattern"] = pattern
                 else:
@@ -158,29 +197,14 @@ class CofenseToSentinelMapping:
                     )
                     data = {}
                     continue
-                data["properties"]["created"] = indicator.get("attributes", {}).get(
-                    "created_at", ""
-                )
-                data["properties"]["externalLastUpdatedTimeUtc"] = indicator.get(
-                    "attributes", {}
-                ).get("updated_at", "")
+                data["properties"]["created"] = indicator.get("attributes", {}).get("created_at", "")
+                data["properties"]["externalLastUpdatedTimeUtc"] = indicator.get("attributes", {}).get("updated_at", "")
                 report_link = self.report_link_mapping(indicator)
-                if report_link:
-                    applogger.debug(
-                        "{}(method={}) : {} : report link : {}".format(
-                            LOGS_STARTS_WITH,
-                            __method_name,
-                            COFENSE_TO_SENTINEL,
-                            report_link,
-                        )
-                    )
-                else:
-                    applogger.warning(
-                        "{}(method={}) : {} : no report link found.".format(
-                            LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL
-                        )
-                    )
-                indicator_response = self.microsoft_obj.create_indicator(data)
+                indicator_response = self.microsoft_obj.create_indicator(data, indicator_id)
+                if indicator_response is None:
+                    failed_indicator = {"indicator": data, "report_link": report_link, "id": indicator_id}
+                    self.save_failed_indicators(failed_indicator)
+                    continue
                 indicator_externalId = indicator_response.get("properties", {}).get("externalId", "")
                 applogger.debug(
                     "{}(method={}) : {}: indicator created successfully.".format(
@@ -192,15 +216,60 @@ class CofenseToSentinelMapping:
                     "indicator_id": indicator_id,
                     "external_id": "{}-{}".format(indicator_externalId, source_cofence),
                     "report_link": report_link,
-                    "updated_at": updated_at
+                    "updated_at": updated_at,
                 }
                 report_data.append(reportdata)
                 data = {}
             return report_data, checkpoint
         except CofenseException:
+            raise CofenseException()
+        except Exception as error:
             applogger.error(
-                "{}(method={}) : {} : error occurred while mapping data of indicators.".format(
-                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL
+                "{}(method={}) : {} : error occurred while mapping data of indicators : {}.".format(
+                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error
+                )
+            )
+            raise CofenseException()
+
+    def process_indicator_data(self, indicator_data):
+        """Process indicator data."""
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            report_data, checkpoint = self.data_mapping(indicator_data)
+            if report_data:
+                self.microsoft_obj.post_data(json.dumps(report_data), self.log_type)
+            else:
+                applogger.warning(
+                    "{}(method={}) : {} : no new report data found in the indicator.".format(
+                        LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL
+                    )
+                )
+            if checkpoint:
+                datetime_checkpoint = datetime.datetime.strptime(checkpoint, DATETIMEFORMAT) + datetime.timedelta(
+                    milliseconds=1
+                )
+                latest_checkpoint = datetime.datetime.strftime(datetime_checkpoint, DATETIMEFORMAT)
+                self.state_obj.post(latest_checkpoint)
+                applogger.info(
+                    "{}(method={}) : {} : checkpoint saved {}".format(
+                        LOGS_STARTS_WITH,
+                        __method_name,
+                        COFENSE_TO_SENTINEL,
+                        latest_checkpoint,
+                    )
+                )
+            else:
+                applogger.info(
+                    "{}(method={}) : {} : no new checkpoint found.".format(
+                        LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL
+                    )
+                )
+        except CofenseException:
+            raise CofenseException()
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : error occurred while processing indicators : {}.".format(
+                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error
                 )
             )
             raise CofenseException()
@@ -233,12 +302,8 @@ class CofenseToSentinelMapping:
                     )
                 )
             else:
-                today_datetime = datetime.datetime.utcnow() - datetime.timedelta(
-                    days=15
-                )
-                from_datetime = datetime.datetime.strftime(
-                    today_datetime, DATETIMEFORMAT
-                )
+                today_datetime = datetime.datetime.utcnow() - datetime.timedelta(days=15)
+                from_datetime = datetime.datetime.strftime(today_datetime, DATETIMEFORMAT)
                 params["filter[updated_at_gteq]"] = from_datetime
                 applogger.info(
                     "{}(method={}) : {} : getting indicators data of last 15 days from {}".format(
@@ -258,46 +323,12 @@ class CofenseToSentinelMapping:
                 )
             )
             while True:
-                indicators_data_json = (
-                    self.cofense_triage_obj.get_indicators_from_cofense(
-                        url=list_indicator_url, params=params
-                    )
+                indicators_data_json = self.cofense_triage_obj.get_indicators_from_cofense(
+                    url=list_indicator_url, params=params
                 )
                 indicator_data = indicators_data_json.get("data", [])
                 if indicator_data:
-                    report_data, checkpoint = self.data_mapping(indicator_data)
-                    if report_data:
-                        self.microsoft_obj.post_data(
-                            json.dumps(report_data), self.log_type
-                        )
-                    else:
-                        applogger.warning(
-                            "{}(method={}) : {} : no new report data found in the indicator.".format(
-                                LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL
-                            )
-                        )
-                    if checkpoint:
-                        datetime_checkpoint = datetime.datetime.strptime(
-                            checkpoint, DATETIMEFORMAT
-                        ) + datetime.timedelta(milliseconds=1)
-                        latest_checkpoint = datetime.datetime.strftime(
-                            datetime_checkpoint, DATETIMEFORMAT
-                        )
-                        self.state_obj.post(latest_checkpoint)
-                        applogger.info(
-                            "{}(method={}) : {} : checkpoint saved {}".format(
-                                LOGS_STARTS_WITH,
-                                __method_name,
-                                COFENSE_TO_SENTINEL,
-                                latest_checkpoint,
-                            )
-                        )
-                    else:
-                        applogger.info(
-                            "{}(method={}) : {} : no new checkpoint found.".format(
-                                LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL
-                            )
-                        )
+                    self.process_indicator_data(indicator_data)
                     if not indicators_data_json.get("links", {}).get("next", ""):
                         applogger.info(
                             "{}(method={}) : {} : all indicators created successfully.".format(
@@ -315,3 +346,10 @@ class CofenseToSentinelMapping:
                     break
         except CofenseException:
             raise CofenseException()
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : error occurred while creating sentinel indicators : {}.".format(
+                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error
+                )
+            )
+            raise CofenseException()

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/CofenseBasedIndicatorCreator/sentinel.py

@@ -1,4 +1,5 @@
 """This file contains methods for creating microsoft indicator and custom log table."""
+
 import base64
 import time
 import requests
@@ -33,7 +34,7 @@ class MicrosoftSentinel:
         """Initialize instance variable for class."""
         self.bearer_token = auth_sentinel(COFENSE_TO_SENTINEL)
 
-    def create_indicator(self, indicator_data):
+    def create_indicator(self, indicator_data, indicator_id):
         """To create indicator into Microsoft Sentinel."""
         __method_name = inspect.currentframe().f_code.co_name
         try:
@@ -69,6 +70,17 @@ class MicrosoftSentinel:
                         )
                     )
                     return response_json
+                elif response.status_code == 400:
+                    applogger.warning(
+                        "{}(method={}) : {} :  error 400 for indicatorId : {}.".format(
+                            LOGS_STARTS_WITH,
+                            __method_name,
+                            COFENSE_TO_SENTINEL,
+                            indicator_id,
+                        )
+                    )
+                    return None
+
                 elif response.status_code == 429:
                     applogger.error(
                         "{}(method={}) : {} : trying again error 429.".format(
@@ -88,8 +100,8 @@ class MicrosoftSentinel:
                     retry_count_401 += 1
                 else:
                     applogger.error(
-                        "{}(method={}) : {} : url: {}, Status Code : {}: Error while creating microsoft"
-                        " sentinel access_token. Error Reason: {}".format(
+                        "{}(method={}) : {} : url: {}, Status Code : {}: Error while creating indicator, "
+                        "Error Reason: {}".format(
                             LOGS_STARTS_WITH,
                             __method_name,
                             COFENSE_TO_SENTINEL,
@@ -105,9 +117,11 @@ class MicrosoftSentinel:
                 )
             )
             raise CofenseException()
-        except CofenseException as error:
+        except CofenseException:
+            raise CofenseException()
+        except Exception as error:
             applogger.error(
-                "{}(method={}) : {} : Error generated while getting sentinel access token :{}".format(
+                "{}(method={}) : {} : Error generated while creating indicator : {}".format(
                     LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error
                 )
             )
@@ -123,17 +137,7 @@ class MicrosoftSentinel:
     ):
         """To build signature which is required in header."""
         x_headers = "x-ms-date:" + date
-        string_to_hash = (
-            method
-            + "\n"
-            + str(content_length)
-            + "\n"
-            + content_type
-            + "\n"
-            + x_headers
-            + "\n"
-            + resource
-        )
+        string_to_hash = method + "\n" + str(content_length) + "\n" + content_type + "\n" + x_headers + "\n" + resource
         bytes_to_hash = bytes(string_to_hash, encoding="utf-8")
         decoded_key = base64.b64decode(WORKSPACE_KEY)
         encoded_hash = base64.b64encode(
@@ -173,16 +177,8 @@ class MicrosoftSentinel:
                     LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, err
                 )
             )
-            raise CofenseException(
-                "Error while generating signature for posting data into log analytics."
-            )
-        uri = (
-            "https://"
-            + WORKSPACE_ID
-            + ".ods.opinsights.azure.com"
-            + resource
-            + "?api-version=2016-04-01"
-        )
+            raise CofenseException("Error while generating signature for posting data into log analytics.")
+        uri = "https://" + WORKSPACE_ID + ".ods.opinsights.azure.com" + resource + "?api-version=2016-04-01"
 
         headers = {
             "content-type": content_type,
@@ -194,7 +190,8 @@ class MicrosoftSentinel:
             response = requests.post(uri, data=body, headers=headers)
             if response.status_code >= 200 and response.status_code <= 299:
                 applogger.debug(
-                    "{}(method={}) : {} : Status_code: {} Accepted: Data Posted Successfully to microsoft sentinel.".format(
+                    "{}(method={}) : {} : Status_code: {} Accepted: Data Posted\
+                          Successfully to microsoft sentinel.".format(
                         LOGS_STARTS_WITH,
                         __method_name,
                         COFENSE_TO_SENTINEL,
@@ -210,17 +207,11 @@ class MicrosoftSentinel:
                 )
         except CofenseException as error:
             applogger.error(
-                "{}(method={}) : {} : Error:{}".format(
-                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error
-                )
-            )
-            raise CofenseException(
-                "CofenseException: Error while posting data to sentinel."
+                "{}(method={}) : {} : Error:{}".format(LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error)
             )
+            raise CofenseException("CofenseException: Error while posting data to sentinel.")
         except Exception as error:
             applogger.error(
-                "{}(method={}) : {} : Error:{}".format(
-                    LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error
-                )
+                "{}(method={}) : {} : Error:{}".format(LOGS_STARTS_WITH, __method_name, COFENSE_TO_SENTINEL, error)
             )
             raise CofenseException("Exception: Error while posting data to sentinel.")

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/CofenseTriage_API_FunctionApp.json

@@ -2,7 +2,7 @@
     "id": "CofenseTriage",
     "title": "Cofense Triage Threat Indicators Ingestion",
     "publisher": "Cofense",
-    "descriptionMarkdown": "The [Cofense-Triage](https://cofense.com/product-services/cofense-triage/) data connector provides the following capabilities: \n 1. CofenseBasedIndicatorCreator : \n >* Get Threat Indicators from the Cofense Triage platform and create Threat Intelligence Indicators in Microsoft Sentinel. \n > * Ingest Cofense Indicator ID and report links into custom logs table. \n 2. NonCofenseBasedIndicatorCreatorToCofense : \n >* Get Non-Cofense Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Cofense Triage platform. \n 3. IndicatorCreatorToDefender : \n >* Get Cofense Triage Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Microsoft Defender for Endpoints. \n\n\n For more details of REST APIs refer to the below two documentations: \n 1. Cofense API documentation: \n> https://`<your-cofense-instance-name>`/docs/api/v2/index.html \n 2. Microsoft Threat Intelligence Indicator documentation: \n> https://learn.microsoft.com/rest/api/securityinsights/preview/threat-intelligence-indicator \n 3. Microsoft Defender for Endpoints Indicator documentation: \n> https://learn.microsoft.com/microsoft-365/security/defender-endpoint/ti-indicator?view=o365-worldwide",
+    "descriptionMarkdown": "The [Cofense-Triage](https://cofense.com/product-services/cofense-triage/) data connector provides the following capabilities: \n 1. CofenseBasedIndicatorCreator : \n >* Get Threat Indicators from the Cofense Triage platform and create Threat Intelligence Indicators in Microsoft Sentinel. \n > * Ingest Cofense Indicator ID and report links into custom logs table. \n 2. NonCofenseBasedIndicatorCreatorToCofense : \n >* Get Non-Cofense Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Cofense Triage platform. \n 3. IndicatorCreatorToDefender : \n >* Get Cofense Triage Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Microsoft Defender for Endpoints. \n 4. RetryFailedIndicators : \n >* Get failed indicators from failed indicators files and retry creating/updating Threat Intelligence indicators in Microsoft Sentinel. \n\n\n For more details of REST APIs refer to the below two documentations: \n 1. Cofense API documentation: \n> https://`<your-cofense-instance-name>`/docs/api/v2/index.html \n 2. Microsoft Threat Intelligence Indicator documentation: \n> https://learn.microsoft.com/rest/api/securityinsights/preview/threat-intelligence-indicator \n 3. Microsoft Defender for Endpoints Indicator documentation: \n> https://learn.microsoft.com/microsoft-365/security/defender-endpoint/ti-indicator?view=o365-worldwide",
     "graphQueries": [
         {
             "metricName": "Cofense Triage Threat Indicators data received",
@@ -18,6 +18,11 @@
             "metricName": "Non-Cofense Triage Threat Indicators data received",
             "legend": "ThreatIntelligenceIndicator | where SourceSystem !startswith 'Cofense : '",
             "baseQuery": "ThreatIntelligenceIndicator | where SourceSystem !startswith 'Cofense : '"
+        },
+        {
+            "metricName": "Total cofense triage failed to create indicators",
+            "legend": "Cofense_Triage_failed_indicators_CL",
+            "baseQuery": "Cofense_Triage_failed_indicators_CL"
         }
     ],
     "sampleQueries": [
@@ -32,6 +37,10 @@
         {
             "description": "Non-Cofense Based Indicators Events - All Non-Cofense indicators in Microsoft Sentinel Threat Intelligence.",
             "query": "ThreatIntelligenceIndicator\n | where SourceSystem !startswith 'Cofense : '\n | sort by TimeGenerated desc"
+        },
+        {
+            "description": "Cofense Triage Failed Indicators - All Cofense triage failed indicators data.",
+            "query": "Cofense_Triage_failed_indicators_CL\n | sort by TimeGenerated desc"
         }
     ],
     "dataTypes": [
@@ -42,6 +51,10 @@
         {
             "name": "Report_links_data_CL",
             "lastDataReceivedQuery": "Report_links_data_CL\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
+        },
+        {
+            "name": "Cofense_Triage_failed_indicators_CL",
+            "lastDataReceivedQuery": "Cofense_Triage_failed_indicators_CL\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
         }
     ],
     "connectivityCriterias": [
@@ -62,6 +75,12 @@
             "value": [
                 "ThreatIntelligenceIndicator\n             | where SourceSystem !startswith 'Cofense : '\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(30d)"
             ]
+        },
+        {
+            "type": "IsConnectedQuery",
+            "value": [
+                "Cofense_Triage_failed_indicators_CL\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(30d)"
+            ]
         }
     ],
     "availability": {

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/RetryFailedIndicators/__init__.py

@@ -0,0 +1,18 @@
+"""init file for Retrying Failed Indicators."""
+import datetime
+import logging
+import azure.functions as func
+from .retry_failed_indicators import RetryFailedIndicators
+
+
+def main(mytimer: func.TimerRequest) -> None:
+    """Driver method for RetryFailedIndicators."""
+    utc_timestamp = (
+        datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat()
+    )
+    retry_obj = RetryFailedIndicators()
+    retry_obj.get_failed_indicators_and_retry()
+    if mytimer.past_due:
+        logging.info("The timer is past due!")
+
+    logging.info("Python timer trigger function ran at %s", utc_timestamp)

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/RetryFailedIndicators/function.json

@@ -0,0 +1,11 @@
+{
+  "scriptFile": "__init__.py",
+  "bindings": [
+    {
+      "name": "mytimer",
+      "type": "timerTrigger",
+      "direction": "in",
+      "schedule": "%Schedule%"
+    }
+  ]
+}

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/RetryFailedIndicators/retry_failed_indicators.py

@@ -0,0 +1,318 @@
+"""Retry Failed Indicators."""
+
+import inspect
+import json
+from ..SharedCode.cofense_exception import CofenseException
+from ..SharedCode.consts import (
+    LOGS_STARTS_WITH,
+    CONNECTION_STRING,
+    RETRY_FAILED_INDICATORS,
+    FAILED_INDICATORS_TABLE_NAME,
+    REPORTS_TABLE_NAME,
+    FAILED_INDICATORS_FILE_SHARE,
+)
+from ..SharedCode.state_manager import StateManager
+from ..SharedCode.logger import applogger
+from .sentinel import MicrosoftSentinel
+from azure.storage.fileshare import ShareDirectoryClient
+from azure.core.exceptions import ResourceNotFoundError
+from datetime import datetime, timezone, timedelta
+
+
+class RetryFailedIndicators:
+    """Retry Failed Indicators."""
+
+    def __init__(self):
+        """Initialize instance variable for class."""
+        self.microsoft_obj = MicrosoftSentinel()
+        self.log_type = REPORTS_TABLE_NAME
+
+    def return_file_names_to_query_in_the_current_execution(self, file_names_list):
+        """Return the file names to query in the current execution.
+
+        Args:
+            file_names_list (list): list of file names
+
+        Returns:
+            list: list of file names.
+        """
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            files_to_query_in_current_execution = []
+            now = datetime.now(timezone.utc)
+            one_hour_ago = now - timedelta(hours=1)
+            for file in file_names_list:
+                if file.isnumeric() and int(file) < int(one_hour_ago.timestamp()):
+                    files_to_query_in_current_execution.append(file)
+            applogger.info(
+                "{}(method={}) : {} : Found {} failed Indicator files.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    len(files_to_query_in_current_execution),
+                )
+            )
+            return files_to_query_in_current_execution
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : Error in returning which files to process, Error : {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    error,
+                )
+            )
+            raise CofenseException()
+
+    def get_failed_indicator_files(self, parent_dir):
+        """Get failed indicator's file names.
+
+        Args:
+            parent_dir (ShareDirectory.from_connection_string): Object of ShareDirectory to perform operations
+            on file share.
+
+        Returns:
+            list: list of file names of failed indicators
+        """
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            files_list = list(parent_dir.list_directories_and_files())
+            file_names = []
+            if (len(files_list)) > 0:
+                for file in files_list:
+                    file_names.append(file["name"])
+                return file_names
+            else:
+                return None
+        except ResourceNotFoundError:
+            applogger.info(
+                "{}(method={}) : {} : No Failed Indicators File Found.".format(
+                    LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS
+                )
+            )
+            return None
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : error while getting list of files, Error : {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    error,
+                )
+            )
+            raise CofenseException()
+
+    def delete_file_from_file_share(self, file_name, parent_dir):
+        """Delete the file from azure file share.
+
+        Args:
+            file_name (str): name of the file to delete
+            parent_dir (ShareDirectory.from_connection_string): Object of ShareDirectory to perform operations
+            on file share.
+        """
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            parent_dir.delete_file(file_name)
+        except ResourceNotFoundError:
+            applogger.info(
+                "{}(method={}) : {} : No Failed Indicators File Found, filename-{}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    file_name,
+                )
+            )
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : error while deleting file, Error : {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    error,
+                )
+            )
+            raise CofenseException()
+
+    def get_failed_indicators(self, file_name):
+        """Get Failed indicators list from files."""
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            state_manager_obj = StateManager(
+                CONNECTION_STRING, file_path=file_name, share_name=FAILED_INDICATORS_FILE_SHARE
+            )
+            failed_indicators = state_manager_obj.get(RETRY_FAILED_INDICATORS)
+            if failed_indicators is None or failed_indicators == "":
+                return None
+            else:
+                json_data = json.loads(failed_indicators)
+                return json_data
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : error while getting data for failed indicators, Error : {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    error,
+                )
+            )
+            raise CofenseException()
+
+    def post_failed_indicators(self, indicators_data):
+        """Post indicators to Microsoft Sentinel Threat Intelligence.
+
+        Args:
+            indicators_data (dict): Failed Indicators Data
+
+        Returns:
+            dict: Dictionary containing the success_count and failure_count
+        """
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            success_count = 0
+            failed_count = 0
+            for indicator in indicators_data:
+                failed_indicator = indicator.get("indicator", "")
+                report_link = indicator.get("report_link", "")
+                indicator_id = indicator.get("id", "")
+                indicator_response = self.microsoft_obj.create_indicator(failed_indicator, indicator_id)
+                if indicator_response is None:
+                    failed_indicator['report_link'] = report_link
+                    self.microsoft_obj.post_data(
+                        body=json.dumps([failed_indicator]),
+                        log_type=FAILED_INDICATORS_TABLE_NAME,
+                    )
+                    applogger.info(
+                        "{}(method={}) : {} : Posted failed indicator to log analytics, indicatorId- {}".format(
+                            LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS, indicator_id
+                        )
+                    )
+                    failed_count += 1
+                else:
+                    indicator_externalId = indicator_response.get("properties", {}).get("externalId", "")
+                    applogger.debug(
+                        "{}(method={}) : {}: indicator created successfully, indicatorId- {}.".format(
+                            LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS, indicator_id
+                        )
+                    )
+                    updated_at = failed_indicator.get("properties", {}).get("externalLastUpdatedTimeUtc", "")
+                    source_cofence = failed_indicator.get("properties", {}).get("source", "")
+                    report_data = {
+                        "indicator_id": indicator_id,
+                        "external_id": "{}-{}".format(indicator_externalId, source_cofence),
+                        "report_link": report_link,
+                        "updated_at": updated_at,
+                    }
+                    self.microsoft_obj.post_data(json.dumps(report_data), self.log_type)
+                    success_count += 1
+            applogger.info(
+                "{}(method={}) : {} : Total_Invocations: {}, Successful Indicators Posting: {},\
+                    Failed Indicators Posting: {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    len(indicators_data),
+                    success_count,
+                    failed_count,
+                )
+            )
+            return {"success_count": success_count, "failure_count": failed_count}
+        except CofenseException:
+            applogger.error(
+                "{}(method={}) : {} : Indicator Creation Failed while retrying.".format(
+                    LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS
+                )
+            )
+            raise CofenseException()
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : Indicator Creation Failed while retrying, Error : {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    error,
+                )
+            )
+            raise CofenseException()
+
+    def get_failed_indicators_and_retry(self):
+        """Get failed indicators data from files and try creating them again."""
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            total_retry_indicators = 0
+            retry_success = 0
+            retry_failure = 0
+            parent_dir = ShareDirectoryClient.from_connection_string(
+                conn_str=CONNECTION_STRING,
+                share_name=FAILED_INDICATORS_FILE_SHARE,
+                directory_path="",
+            )
+            failed_indicator_files = self.get_failed_indicator_files(parent_dir)
+            if not failed_indicator_files:
+                applogger.info(
+                    "{}(method={}) : {} : No failed indicators found.".format(
+                        LOGS_STARTS_WITH,
+                        __method_name,
+                        RETRY_FAILED_INDICATORS,
+                    )
+                )
+                return
+            file_names_to_query = self.return_file_names_to_query_in_the_current_execution(failed_indicator_files)
+            if not file_names_to_query:
+                applogger.info(
+                    "{}(method={}) : {} : No previously failed indicators found.".format(
+                        LOGS_STARTS_WITH,
+                        __method_name,
+                        RETRY_FAILED_INDICATORS,
+                    )
+                )
+                return
+            for file in file_names_to_query:
+                failed_indicators = self.get_failed_indicators(file)
+                if not failed_indicators:
+                    applogger.info(
+                        "{}(method={}) : {} : No Failed indicators found in the file-{}.".format(
+                            LOGS_STARTS_WITH,
+                            __method_name,
+                            RETRY_FAILED_INDICATORS,
+                            file,
+                        )
+                    )
+                    continue
+
+                result = self.post_failed_indicators(failed_indicators)
+                total_retry_indicators += len(failed_indicators)
+                retry_success += result["success_count"]
+                retry_failure += result["failure_count"]
+                self.delete_file_from_file_share(file, parent_dir)
+                applogger.info(
+                    "{}(method={}) : {} : Successfully deleted the file, filename : {}.".format(
+                        LOGS_STARTS_WITH,
+                        __method_name,
+                        RETRY_FAILED_INDICATORS,
+                        file,
+                    )
+                )
+            applogger.info(
+                "{}(method={}) : {} : Total collected Failed Indicators : {}, "
+                "posted indicators into sentinel: {}, "
+                "failed indicators while posting : {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    total_retry_indicators,
+                    retry_success,
+                    retry_failure,
+                )
+            )
+        except CofenseException:
+            raise CofenseException()
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : Retrying Failed Indicators incurred an error, Error: {}.".format(
+                    LOGS_STARTS_WITH,
+                    __method_name,
+                    RETRY_FAILED_INDICATORS,
+                    error,
+                )
+            )
+            raise CofenseException()

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/RetryFailedIndicators/sentinel.py

@@ -0,0 +1,216 @@
+"""This file contains methods for creating microsoft indicator and custom log table."""
+
+import base64
+import time
+import requests
+import json
+import hashlib
+import hmac
+import inspect
+import datetime
+from ..SharedCode.consts import (
+    AZURE_SUBSCRIPTION_ID,
+    AZURE_RESOURCE_GROUP,
+    AZURE_WORKSPACE_NAME,
+    CREATE_SENTINEL_INDICATORS_URL,
+    LOGS_STARTS_WITH,
+    SENTINEL_429_SLEEP,
+    SENTINEL_400_SLEEP,
+    WORKSPACE_KEY,
+    WORKSPACE_ID,
+    RETRY_FAILED_INDICATORS,
+)
+from ..SharedCode.logger import applogger
+from ..SharedCode.cofense_exception import CofenseException
+from ..SharedCode.utils import (
+    auth_sentinel,
+    make_rest_call,
+)
+
+
+class MicrosoftSentinel:
+    """This class contains methods to create indicator into Microsoft Sentinel."""
+
+    def __init__(self):
+        """Initialize instance variable for class."""
+        self.bearer_token = auth_sentinel(RETRY_FAILED_INDICATORS)
+
+    def create_indicator(self, indicator_data, indicator_id):
+        """To create indicator into Microsoft Sentinel."""
+        __method_name = inspect.currentframe().f_code.co_name
+        try:
+            retry_count_429 = 0
+            retry_count_401 = 0
+            retry_count_400 = 0
+            while retry_count_429 <= 3 and retry_count_401 <= 1 and retry_count_400 <= 1:
+                create_indicator_url = CREATE_SENTINEL_INDICATORS_URL.format(
+                    subscriptionId=AZURE_SUBSCRIPTION_ID,
+                    resourceGroupName=AZURE_RESOURCE_GROUP,
+                    workspaceName=AZURE_WORKSPACE_NAME,
+                )
+                headers = {
+                    "Content-Type": "application/json",
+                    "Authorization": "Bearer {}".format(self.bearer_token),
+                }
+                response = make_rest_call(
+                    url=create_indicator_url,
+                    method="POST",
+                    azure_function_name=RETRY_FAILED_INDICATORS,
+                    payload=json.dumps(indicator_data),
+                    headers=headers,
+                )
+                if response.status_code >= 200 and response.status_code <= 299:
+                    response_json = response.json()
+                    applogger.debug(
+                        "{}(method={}) : {} : Created the indicator into the sentinel with status code {}"
+                        " and got the response {}".format(
+                            LOGS_STARTS_WITH,
+                            __method_name,
+                            RETRY_FAILED_INDICATORS,
+                            response.status_code,
+                            response_json,
+                        )
+                    )
+                    return response_json
+                elif response.status_code == 400:
+                    applogger.warning(
+                        "{}(method={}) : {} : trying again error 400 for indicatorId : {}.".format(
+                            LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS, indicator_id
+                        )
+                    )
+                    retry_count_400 += 1
+                    time.sleep(SENTINEL_400_SLEEP)
+                elif response.status_code == 429:
+                    applogger.error(
+                        "{}(method={}) : {} : trying again error 429.".format(
+                            LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS
+                        )
+                    )
+                    retry_count_429 += 1
+                    time.sleep(SENTINEL_429_SLEEP)
+                elif response.status_code == 401:
+                    applogger.error(
+                        "{}(method={}) : {} : trying again error 401.".format(
+                            LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS
+                        )
+                    )
+                    self.bearer_token = auth_sentinel(RETRY_FAILED_INDICATORS)
+                    headers["Authorization"] = ("Bearer {}".format(self.bearer_token),)
+                    retry_count_401 += 1
+                else:
+                    applogger.error(
+                        "{}(method={}) : {} : url: {}, Status Code : {}: Error while creating microsoft"
+                        " sentinel access_token. Error Reason: {}".format(
+                            LOGS_STARTS_WITH,
+                            __method_name,
+                            RETRY_FAILED_INDICATORS,
+                            create_indicator_url,
+                            response.status_code,
+                            response.reason,
+                        )
+                    )
+                    raise CofenseException()
+            applogger.error(
+                "{}(method={}) : {} : Max retries exceeded for microsoft sentinel.".format(
+                    LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS
+                )
+            )
+            return None
+        except CofenseException:
+            raise CofenseException()
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : Error generated while creating indicator :{}".format(
+                    LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS, error
+                )
+            )
+            raise CofenseException()
+
+    def build_signature(
+        self,
+        date,
+        content_length,
+        method,
+        content_type,
+        resource,
+    ):
+        """To build signature which is required in header."""
+        x_headers = "x-ms-date:" + date
+        string_to_hash = method + "\n" + str(content_length) + "\n" + content_type + "\n" + x_headers + "\n" + resource
+        bytes_to_hash = bytes(string_to_hash, encoding="utf-8")
+        decoded_key = base64.b64decode(WORKSPACE_KEY)
+        encoded_hash = base64.b64encode(
+            hmac.new(decoded_key, bytes_to_hash, digestmod=hashlib.sha256).digest()
+        ).decode()
+        authorization = "SharedKey {}:{}".format(WORKSPACE_ID, encoded_hash)
+        return authorization
+
+    # Build and send a request to the POST API
+    def post_data(self, body, log_type):
+        """Build and send a request to the POST API.
+
+        Args:
+            body (str): Data to post into Sentinel log analytics workspace
+            log_type (str): Custom log table name in which data wil be added.
+
+        Returns:
+            status_code: Returns the response status code got while posting data to sentinel.
+        """
+        __method_name = inspect.currentframe().f_code.co_name
+        method = "POST"
+        content_type = "application/json"
+        resource = "/api/logs"
+        rfc1123date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
+        content_length = len(body)
+        try:
+            signature = self.build_signature(
+                rfc1123date,
+                content_length,
+                method,
+                content_type,
+                resource,
+            )
+        except Exception as err:
+            applogger.error(
+                "{}(method={}) : {} : Error occurred: {}".format(
+                    LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS, err
+                )
+            )
+            raise CofenseException("Error while generating signature for posting data into log analytics.")
+        uri = "https://" + WORKSPACE_ID + ".ods.opinsights.azure.com" + resource + "?api-version=2016-04-01"
+
+        headers = {
+            "content-type": content_type,
+            "Authorization": signature,
+            "Log-Type": log_type,
+            "x-ms-date": rfc1123date,
+        }
+        try:
+            response = requests.post(uri, data=body, headers=headers)
+            if response.status_code >= 200 and response.status_code <= 299:
+                applogger.debug(
+                    "{}(method={}) : {} : Status_code: {} Accepted: Data Posted\
+                          Successfully to microsoft sentinel.".format(
+                        LOGS_STARTS_WITH,
+                        __method_name,
+                        RETRY_FAILED_INDICATORS,
+                        response.status_code,
+                    )
+                )
+                return response.status_code
+            else:
+                raise CofenseException(
+                    "Response code: {} from posting data to log analytics.\nError: {}".format(
+                        response.status_code, response.content
+                    )
+                )
+        except CofenseException as error:
+            applogger.error(
+                "{}(method={}) : {} : Error:{}".format(LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS, error)
+            )
+            raise CofenseException("CofenseException: Error while posting data to sentinel.")
+        except Exception as error:
+            applogger.error(
+                "{}(method={}) : {} : Error:{}".format(LOGS_STARTS_WITH, __method_name, RETRY_FAILED_INDICATORS, error)
+            )
+            raise CofenseException("Exception: Error while posting data to sentinel.")

Solutions/CofenseTriage/Data Connectors/CofenseTriageDataConnector/SharedCode/consts.py

@@ -1,4 +1,5 @@
 """This file contains all constants."""
+
 import os
 
 # To do: in every file, just import consts.py file, rather than importing all constants.
@@ -19,7 +20,7 @@ AZURE_AUTHENTICATION_URL = "https://login.microsoftonline.com/{}/oauth2/token"
 CREATE_SENTINEL_INDICATORS_URL = (
     "https://management.azure.com/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers"
     "/Microsoft.OperationalInsights/workspaces/{workspaceName}/providers/Microsoft.SecurityInsights/threatIntelligence"
-    "/main/createIndicator?api-version=2022-12-01-preview"
+    "/main/createIndicator?api-version=2024-03-01"
 )
 AZURE_RESOURCE_GROUP = os.environ.get("Azure_Resource_Group_Name", "")
 AZURE_WORKSPACE_NAME = os.environ.get("Azure_Workspace_Name", "")
@@ -42,6 +43,7 @@ COFENSE_UPDATE_INDICATOR_URL = COFENSE_BASE_URL + "/api/public/v2/threat_indicat
 SENTINEL_TO_COFENSE = "SentinelToCofense"
 COFENSE_429_SLEEP = 300
 SENTINEL_429_SLEEP = 60
+SENTINEL_400_SLEEP = 60
 SENTINEL_SOURCE_PREFIX = "Sentinel : "
 COFENSE_SOURCE_PREFIX = "Cofense : "
 COFENSE_PAGE_SIZE = 100
@@ -54,9 +56,7 @@ THREAT_LEVEL_INTERMEDIATE = 33
 THREAT_LEVEL_SUSPICIOUS = 75
 THREAT_LEVEL_MALICIOUS = 100
 SCHEDULE = os.environ.get("Schedule", "")
-NON_COFENSE_THROTTLE_LIMIT = os.environ.get(
-    "Throttle_Limit_For_Non_Cofense_Indicators", ""
-)
+NON_COFENSE_THROTTLE_LIMIT = os.environ.get("Throttle_Limit_For_Non_Cofense_Indicators", "")
 DEFENDER_POST_INDICATOR_URL = "https://api.securitycenter.microsoft.com/api/indicators"
 SENTINEL_TO_DEFENDER = "SentinelToDefender"
 DEFENDER_429_SLEEP = 60
@@ -65,3 +65,6 @@ THREAT_LEVEL = os.environ.get("Threat_Level", "")
 THREAT_LEVEL_FILTER_BENIGN = "Benign"
 THREAT_LEVEL_FILTER_SUSPICIOUS = "Suspicious"
 THREAT_LEVEL_FILTER_MALICIOUS = "Malicious"
+RETRY_FAILED_INDICATORS = "Retry Failed Indicators"
+FAILED_INDICATORS_TABLE_NAME = "Cofense_Triage_failed_indicators"
+FAILED_INDICATORS_FILE_SHARE = "failed-indicators"

Solutions/CofenseTriage/Data/Solution_CofenseTriage.json

@@ -10,7 +10,7 @@
         "Data Connectors/CofenseTriageDataConnector/CofenseTriage_API_FunctionApp.json"
     ],
     "BasePath": "C:\\Azure-Sentinel\\Solutions\\CofenseTriage",
-    "Version": "2.0.1",
+    "Version": "3.0.0",
     "Metadata": "SolutionMetadata.json",
     "TemplateSpec": true,
     "Is1PConnector": false

Solutions/CofenseTriage/Package/createUiDefinition.json

@@ -6,7 +6,7 @@
     "config": {
       "isWizard": false,
       "basics": {
-        "description": "<img src=\"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Logos/CofenseTriage.svg\" width=\"75px\" height=\"75px\">\n\n**Note:** _There may be [known issues](https://aka.ms/sentinelsolutionsknownissues) pertaining to this Solution, please refer to them before installing._\n\nCofense Triage for Microsoft Sentinel.\n\n**Data Connectors:** 1, **Workbooks:** 1\n\n[Learn more about Microsoft Sentinel](https://aka.ms/azuresentinel) | [Learn more about Solutions](https://aka.ms/azuresentinelsolutionsdoc)",
+        "description": "<img src=\"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Logos/CofenseTriage.svg\" width=\"75px\" height=\"75px\">\n\n**Note:** Please refer to the following before installing the solution: \n\n• Review the solution [Release Notes](https://github.com/Azure/Azure-Sentinel/tree/master/Solutions/CofenseTriage/ReleaseNotes.md)\n\n • There may be [known issues](https://aka.ms/sentinelsolutionsknownissues) pertaining to this Solution, please refer to them before installing.\n\nCofense Triage for Microsoft Sentinel.\n\n**Data Connectors:** 1, **Workbooks:** 1\n\n[Learn more about Microsoft Sentinel](https://aka.ms/azuresentinel) | [Learn more about Solutions](https://aka.ms/azuresentinelsolutionsdoc)",
         "subscription": {
           "resourceProviders": [
             "Microsoft.OperationsManagement/solutions",
@@ -60,7 +60,7 @@
             "name": "dataconnectors1-text",
             "type": "Microsoft.Common.TextBlock",
             "options": {
-              "text": "This Solution installs the data connector for CofenseTriage. You can get CofenseTriage custom log data in your Microsoft Sentinel workspace. Configure and enable this data connector in the Data Connector gallery after this Solution deploys. This data connector creates custom log table(s) ThreatIntelligenceIndicator Report_links_data_CL in your Microsoft Sentinel / Azure Log Analytics workspace."
+              "text": "This Solution installs the data connector for CofenseTriage. You can get CofenseTriage custom log data in your Microsoft Sentinel workspace. After installing the solution, configure and enable this data connector by following guidance in Manage solution view."
             }
           },
           {
@@ -88,7 +88,7 @@
             "name": "workbooks-text",
             "type": "Microsoft.Common.TextBlock",
             "options": {
-              "text": "This Microsoft Sentinel Solution installs workbooks. Workbooks provide a flexible canvas for data monitoring, analysis, and the creation of rich visual reports within the Azure portal. They allow you to tap into one or many data sources from Microsoft Sentinel and combine them into unified interactive experiences."
+              "text": "This solution installs workbook(s) to help you gain insights into the telemetry collected in Microsoft Sentinel. After installing the solution, start using the workbook in Manage solution view."
             }
           },
           {
@@ -100,6 +100,20 @@
                 "uri": "https://docs.microsoft.com/azure/sentinel/tutorial-monitor-your-data"
               }
             }
+          },
+          {
+            "name": "workbook1",
+            "type": "Microsoft.Common.Section",
+            "label": "CofenseTriageThreatIndicators",
+            "elements": [
+              {
+                "name": "workbook1-text",
+                "type": "Microsoft.Common.TextBlock",
+                "options": {
+                  "text": "This workbook provides visualization of Cofense Triage threat indicators which are ingested in the Microsoft Sentinel Threat intelligence."
+                }
+              }
+            ]
           }
         ]
       }

Solutions/CofenseTriage/Package/mainTemplate.json

@@ -38,54 +38,41 @@
     }
   },
   "variables": {
-    "solutionId": "cofense.cofense-triage-sentinel",
-    "_solutionId": "[variables('solutionId')]",
     "email": "support@cofense.com",
     "_email": "[variables('email')]",
-    "workbookVersion1": "1.0.0",
+    "_solutionName": "CofenseTriage",
+    "_solutionVersion": "3.0.0",
+    "solutionId": "cofense.cofense-triage-sentinel",
+    "_solutionId": "[variables('solutionId')]",
+    "workbookVersion1": "1.0",
     "workbookContentId1": "CofenseTriageThreatIndicators",
     "workbookId1": "[resourceId('Microsoft.Insights/workbooks', variables('workbookContentId1'))]",
-    "workbookTemplateSpecName1": "[concat(parameters('workspace'),'-wb-',uniquestring(variables('_workbookContentId1')))]",
+    "workbookTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-wb-',uniquestring(variables('_workbookContentId1'))))]",
     "_workbookContentId1": "[variables('workbookContentId1')]",
     "workspaceResourceId": "[resourceId('microsoft.OperationalInsights/Workspaces', parameters('workspace'))]",
+    "_workbookcontentProductId1": "[concat(take(variables('_solutionId'),50),'-','wb','-', uniqueString(concat(variables('_solutionId'),'-','Workbook','-',variables('_workbookContentId1'),'-', variables('workbookVersion1'))))]",
     "uiConfigId1": "CofenseTriage",
     "_uiConfigId1": "[variables('uiConfigId1')]",
     "dataConnectorContentId1": "CofenseTriage",
     "_dataConnectorContentId1": "[variables('dataConnectorContentId1')]",
     "dataConnectorId1": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId1'))]",
     "_dataConnectorId1": "[variables('dataConnectorId1')]",
-    "dataConnectorTemplateSpecName1": "[concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId1')))]",
-    "dataConnectorVersion1": "1.0.0"
+    "dataConnectorTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId1'))))]",
+    "dataConnectorVersion1": "1.0.0",
+    "_dataConnectorcontentProductId1": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId1'),'-', variables('dataConnectorVersion1'))))]",
+    "_solutioncontentProductId": "[concat(take(variables('_solutionId'),50),'-','sl','-', uniqueString(concat(variables('_solutionId'),'-','Solution','-',variables('_solutionId'),'-', variables('_solutionVersion'))))]"
   },
   "resources": [
     {
-      "type": "Microsoft.Resources/templateSpecs",
-      "apiVersion": "2022-02-01",
+      "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates",
+      "apiVersion": "2023-04-01-preview",
       "name": "[variables('workbookTemplateSpecName1')]",
       "location": "[parameters('workspace-location')]",
-      "tags": {
-        "hidden-sentinelWorkspaceId": "[variables('workspaceResourceId')]",
-        "hidden-sentinelContentType": "Workbook"
-      },
-      "properties": {
-        "description": "CofenseTriage Workbook with template",
-        "displayName": "CofenseTriage workbook template"
-      }
-    },
-    {
-      "type": "Microsoft.Resources/templateSpecs/versions",
-      "apiVersion": "2022-02-01",
-      "name": "[concat(variables('workbookTemplateSpecName1'),'/',variables('workbookVersion1'))]",
-      "location": "[parameters('workspace-location')]",
-      "tags": {
-        "hidden-sentinelWorkspaceId": "[variables('workspaceResourceId')]",
-        "hidden-sentinelContentType": "Workbook"
-      },
       "dependsOn": [
-        "[resourceId('Microsoft.Resources/templateSpecs', variables('workbookTemplateSpecName1'))]"
+        "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "CofenseTriageThreatIndicatorsWorkbook Workbook with template version 2.0.1",
+        "description": "CofenseTriageThreatIndicators Workbook with template version 3.0.0",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('workbookVersion1')]",
@@ -99,7 +86,7 @@
               "kind": "shared",
               "apiVersion": "2021-08-01",
               "metadata": {
-                "description": ""
+                "description": "This workbook provides visualization of Cofense Triage threat indicators which are ingested in the Microsoft Sentinel Threat intelligence."
               },
               "properties": {
                 "displayName": "[parameters('workbook1-name')]",
@@ -114,7 +101,7 @@
               "apiVersion": "2022-01-01-preview",
               "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Workbook-', last(split(variables('workbookId1'),'/'))))]",
               "properties": {
-                "description": ".description",
+                "description": "@{workbookKey=CofenseTriageThreatIndicators; logoFileName=CofenseTriage.svg; description=This workbook provides visualization of Cofense Triage threat indicators which are ingested in the Microsoft Sentinel Threat intelligence.; dataTypesDependencies=System.Object[]; dataConnectorsDependencies=System.Object[]; previewImagesFileNames=System.Object[]; version=1.0; title=CofenseTriageThreatIndicators; templateRelativePath=CofenseTriageThreatIndicators.json; subtitle=; provider=Cofense}.description",
                 "parentId": "[variables('workbookId1')]",
                 "contentId": "[variables('_workbookContentId1')]",
                 "kind": "Workbook",
@@ -133,41 +120,51 @@
                   "email": "support@cofense.com",
                   "tier": "Partner",
                   "link": "https://cofense.com/contact-support/"
+                },
+                "dependencies": {
+                  "operator": "AND",
+                  "criteria": [
+                    {
+                      "contentId": "ThreatIntelligenceIndicator",
+                      "kind": "DataType"
+                    },
+                    {
+                      "contentId": "Report_links_data_CL",
+                      "kind": "DataType"
+                    },
+                    {
+                      "contentId": "CofenseTriageDataConnector",
+                      "kind": "DataConnector"
+                    }
+                  ]
                 }
               }
             }
           ]
-        }
+        },
+        "packageKind": "Solution",
+        "packageVersion": "[variables('_solutionVersion')]",
+        "packageName": "[variables('_solutionName')]",
+        "packageId": "[variables('_solutionId')]",
+        "contentSchemaVersion": "3.0.0",
+        "contentId": "[variables('_workbookContentId1')]",
+        "contentKind": "Workbook",
+        "displayName": "[parameters('workbook1-name')]",
+        "contentProductId": "[variables('_workbookcontentProductId1')]",
+        "id": "[variables('_workbookcontentProductId1')]",
+        "version": "[variables('workbookVersion1')]"
       }
     },
     {
-      "type": "Microsoft.Resources/templateSpecs",
-      "apiVersion": "2022-02-01",
+      "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates",
+      "apiVersion": "2023-04-01-preview",
       "name": "[variables('dataConnectorTemplateSpecName1')]",
       "location": "[parameters('workspace-location')]",
-      "tags": {
-        "hidden-sentinelWorkspaceId": "[variables('workspaceResourceId')]",
-        "hidden-sentinelContentType": "DataConnector"
-      },
-      "properties": {
-        "description": "CofenseTriage data connector with template",
-        "displayName": "CofenseTriage template"
-      }
-    },
-    {
-      "type": "Microsoft.Resources/templateSpecs/versions",
-      "apiVersion": "2022-02-01",
-      "name": "[concat(variables('dataConnectorTemplateSpecName1'),'/',variables('dataConnectorVersion1'))]",
-      "location": "[parameters('workspace-location')]",
-      "tags": {
-        "hidden-sentinelWorkspaceId": "[variables('workspaceResourceId')]",
-        "hidden-sentinelContentType": "DataConnector"
-      },
       "dependsOn": [
-        "[resourceId('Microsoft.Resources/templateSpecs', variables('dataConnectorTemplateSpecName1'))]"
+        "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "CofenseTriage data connector with template version 2.0.1",
+        "description": "CofenseTriage data connector with template version 3.0.0",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('dataConnectorVersion1')]",
@@ -183,9 +180,9 @@
               "properties": {
                 "connectorUiConfig": {
                   "id": "[variables('_uiConfigId1')]",
-                  "title": "Cofense Triage Threat Indicators Ingestion (using Azure Function)",
+                  "title": "Cofense Triage Threat Indicators Ingestion (using Azure Functions)",
                   "publisher": "Cofense",
-                  "descriptionMarkdown": "The [Cofense-Triage](https://cofense.com/product-services/cofense-triage/) data connector provides the following capabilities: \n 1. CofenseBasedIndicatorCreator : \n >* Get Threat Indicators from the Cofense Triage platform and create Threat Intelligence Indicators in Microsoft Sentinel. \n > * Ingest Cofense Indicator ID and report links into custom logs table. \n 2. NonCofenseBasedIndicatorCreatorToCofense : \n >* Get Non-Cofense Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Cofense Triage platform. \n 3. IndicatorCreatorToDefender : \n >* Get Cofense Triage Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Microsoft Defender for Endpoints. \n\n\n For more details of REST APIs refer to the below two documentations: \n 1. Cofense API documentation: \n> https://`<your-cofense-instance-name>`/docs/api/v2/index.html \n 2. Microsoft Threat Intelligence Indicator documentation: \n> https://learn.microsoft.com/rest/api/securityinsights/preview/threat-intelligence-indicator \n 3. Microsoft Defender for Endpoints Indicator documentation: \n> https://learn.microsoft.com/microsoft-365/security/defender-endpoint/ti-indicator?view=o365-worldwide",
+                  "descriptionMarkdown": "The [Cofense-Triage](https://cofense.com/product-services/cofense-triage/) data connector provides the following capabilities: \n 1. CofenseBasedIndicatorCreator : \n >* Get Threat Indicators from the Cofense Triage platform and create Threat Intelligence Indicators in Microsoft Sentinel. \n > * Ingest Cofense Indicator ID and report links into custom logs table. \n 2. NonCofenseBasedIndicatorCreatorToCofense : \n >* Get Non-Cofense Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Cofense Triage platform. \n 3. IndicatorCreatorToDefender : \n >* Get Cofense Triage Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Microsoft Defender for Endpoints. \n 4. RetryFailedIndicators : \n >* Get failed indicators from failed indicators files and retry creating/updating Threat Intelligence indicators in Microsoft Sentinel. \n\n\n For more details of REST APIs refer to the below two documentations: \n 1. Cofense API documentation: \n> https://`<your-cofense-instance-name>`/docs/api/v2/index.html \n 2. Microsoft Threat Intelligence Indicator documentation: \n> https://learn.microsoft.com/rest/api/securityinsights/preview/threat-intelligence-indicator \n 3. Microsoft Defender for Endpoints Indicator documentation: \n> https://learn.microsoft.com/microsoft-365/security/defender-endpoint/ti-indicator?view=o365-worldwide",
                   "graphQueries": [
                     {
                       "metricName": "Cofense Triage Threat Indicators data received",
@@ -201,6 +198,11 @@
                       "metricName": "Non-Cofense Triage Threat Indicators data received",
                       "legend": "ThreatIntelligenceIndicator | where SourceSystem !startswith 'Cofense : '",
                       "baseQuery": "ThreatIntelligenceIndicator | where SourceSystem !startswith 'Cofense : '"
+                    },
+                    {
+                      "metricName": "Total cofense triage failed to create indicators",
+                      "legend": "Cofense_Triage_failed_indicators_CL",
+                      "baseQuery": "Cofense_Triage_failed_indicators_CL"
                     }
                   ],
                   "sampleQueries": [
@@ -215,6 +217,10 @@
                     {
                       "description": "Non-Cofense Based Indicators Events - All Non-Cofense indicators in Microsoft Sentinel Threat Intelligence.",
                       "query": "ThreatIntelligenceIndicator\n | where SourceSystem !startswith 'Cofense : '\n | sort by TimeGenerated desc"
+                    },
+                    {
+                      "description": "Cofense Triage Failed Indicators - All Cofense triage failed indicators data.",
+                      "query": "Cofense_Triage_failed_indicators_CL\n | sort by TimeGenerated desc"
                     }
                   ],
                   "dataTypes": [
@@ -225,6 +231,10 @@
                     {
                       "name": "Report_links_data_CL",
                       "lastDataReceivedQuery": "Report_links_data_CL\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
+                    },
+                    {
+                      "name": "Cofense_Triage_failed_indicators_CL",
+                      "lastDataReceivedQuery": "Cofense_Triage_failed_indicators_CL\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
                     }
                   ],
                   "connectivityCriterias": [
@@ -245,6 +255,12 @@
                       "value": [
                         "ThreatIntelligenceIndicator\n             | where SourceSystem !startswith 'Cofense : '\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(30d)"
                       ]
+                    },
+                    {
+                      "type": "IsConnectedQuery",
+                      "value": [
+                        "Cofense_Triage_failed_indicators_CL\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(30d)"
+                      ]
                     }
                   ],
                   "availability": {
@@ -358,7 +374,7 @@
             },
             {
               "type": "Microsoft.OperationalInsights/workspaces/providers/metadata",
-              "apiVersion": "2022-01-01-preview",
+              "apiVersion": "2023-04-01-preview",
               "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId1'),'/'))))]",
               "properties": {
                 "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId1'))]",
@@ -383,12 +399,23 @@
               }
             }
           ]
-        }
+        },
+        "packageKind": "Solution",
+        "packageVersion": "[variables('_solutionVersion')]",
+        "packageName": "[variables('_solutionName')]",
+        "packageId": "[variables('_solutionId')]",
+        "contentSchemaVersion": "3.0.0",
+        "contentId": "[variables('_dataConnectorContentId1')]",
+        "contentKind": "DataConnector",
+        "displayName": "Cofense Triage Threat Indicators Ingestion (using Azure Functions)",
+        "contentProductId": "[variables('_dataConnectorcontentProductId1')]",
+        "id": "[variables('_dataConnectorcontentProductId1')]",
+        "version": "[variables('dataConnectorVersion1')]"
       }
     },
     {
       "type": "Microsoft.OperationalInsights/workspaces/providers/metadata",
-      "apiVersion": "2022-01-01-preview",
+      "apiVersion": "2023-04-01-preview",
       "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId1'),'/'))))]",
       "dependsOn": [
         "[variables('_dataConnectorId1')]"
@@ -424,9 +451,9 @@
       "kind": "GenericUI",
       "properties": {
         "connectorUiConfig": {
-          "title": "Cofense Triage Threat Indicators Ingestion (using Azure Function)",
+          "title": "Cofense Triage Threat Indicators Ingestion (using Azure Functions)",
           "publisher": "Cofense",
-          "descriptionMarkdown": "The [Cofense-Triage](https://cofense.com/product-services/cofense-triage/) data connector provides the following capabilities: \n 1. CofenseBasedIndicatorCreator : \n >* Get Threat Indicators from the Cofense Triage platform and create Threat Intelligence Indicators in Microsoft Sentinel. \n > * Ingest Cofense Indicator ID and report links into custom logs table. \n 2. NonCofenseBasedIndicatorCreatorToCofense : \n >* Get Non-Cofense Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Cofense Triage platform. \n 3. IndicatorCreatorToDefender : \n >* Get Cofense Triage Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Microsoft Defender for Endpoints. \n\n\n For more details of REST APIs refer to the below two documentations: \n 1. Cofense API documentation: \n> https://`<your-cofense-instance-name>`/docs/api/v2/index.html \n 2. Microsoft Threat Intelligence Indicator documentation: \n> https://learn.microsoft.com/rest/api/securityinsights/preview/threat-intelligence-indicator \n 3. Microsoft Defender for Endpoints Indicator documentation: \n> https://learn.microsoft.com/microsoft-365/security/defender-endpoint/ti-indicator?view=o365-worldwide",
+          "descriptionMarkdown": "The [Cofense-Triage](https://cofense.com/product-services/cofense-triage/) data connector provides the following capabilities: \n 1. CofenseBasedIndicatorCreator : \n >* Get Threat Indicators from the Cofense Triage platform and create Threat Intelligence Indicators in Microsoft Sentinel. \n > * Ingest Cofense Indicator ID and report links into custom logs table. \n 2. NonCofenseBasedIndicatorCreatorToCofense : \n >* Get Non-Cofense Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Cofense Triage platform. \n 3. IndicatorCreatorToDefender : \n >* Get Cofense Triage Threat Intelligence Indicators from Microsoft Sentinel Threat Intelligence and create/update Indicators in Microsoft Defender for Endpoints. \n 4. RetryFailedIndicators : \n >* Get failed indicators from failed indicators files and retry creating/updating Threat Intelligence indicators in Microsoft Sentinel. \n\n\n For more details of REST APIs refer to the below two documentations: \n 1. Cofense API documentation: \n> https://`<your-cofense-instance-name>`/docs/api/v2/index.html \n 2. Microsoft Threat Intelligence Indicator documentation: \n> https://learn.microsoft.com/rest/api/securityinsights/preview/threat-intelligence-indicator \n 3. Microsoft Defender for Endpoints Indicator documentation: \n> https://learn.microsoft.com/microsoft-365/security/defender-endpoint/ti-indicator?view=o365-worldwide",
           "graphQueries": [
             {
               "metricName": "Cofense Triage Threat Indicators data received",
@@ -442,6 +469,11 @@
               "metricName": "Non-Cofense Triage Threat Indicators data received",
               "legend": "ThreatIntelligenceIndicator | where SourceSystem !startswith 'Cofense : '",
               "baseQuery": "ThreatIntelligenceIndicator | where SourceSystem !startswith 'Cofense : '"
+            },
+            {
+              "metricName": "Total cofense triage failed to create indicators",
+              "legend": "Cofense_Triage_failed_indicators_CL",
+              "baseQuery": "Cofense_Triage_failed_indicators_CL"
             }
           ],
           "dataTypes": [
@@ -452,6 +484,10 @@
             {
               "name": "Report_links_data_CL",
               "lastDataReceivedQuery": "Report_links_data_CL\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
+            },
+            {
+              "name": "Cofense_Triage_failed_indicators_CL",
+              "lastDataReceivedQuery": "Cofense_Triage_failed_indicators_CL\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
             }
           ],
           "connectivityCriterias": [
@@ -472,6 +508,12 @@
               "value": [
                 "ThreatIntelligenceIndicator\n             | where SourceSystem !startswith 'Cofense : '\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(30d)"
               ]
+            },
+            {
+              "type": "IsConnectedQuery",
+              "value": [
+                "Cofense_Triage_failed_indicators_CL\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(30d)"
+              ]
             }
           ],
           "sampleQueries": [
@@ -486,6 +528,10 @@
             {
               "description": "Non-Cofense Based Indicators Events - All Non-Cofense indicators in Microsoft Sentinel Threat Intelligence.",
               "query": "ThreatIntelligenceIndicator\n | where SourceSystem !startswith 'Cofense : '\n | sort by TimeGenerated desc"
+            },
+            {
+              "description": "Cofense Triage Failed Indicators - All Cofense triage failed indicators data.",
+              "query": "Cofense_Triage_failed_indicators_CL\n | sort by TimeGenerated desc"
             }
           ],
           "availability": {
@@ -599,13 +645,20 @@
       }
     },
     {
-      "type": "Microsoft.OperationalInsights/workspaces/providers/metadata",
-      "apiVersion": "2022-01-01-preview",
+      "type": "Microsoft.OperationalInsights/workspaces/providers/contentPackages",
+      "apiVersion": "2023-04-01-preview",
       "location": "[parameters('workspace-location')]",
       "properties": {
-        "version": "2.0.1",
+        "version": "3.0.0",
         "kind": "Solution",
-        "contentSchemaVersion": "2.0.0",
+        "contentSchemaVersion": "3.0.0",
+        "displayName": "CofenseTriage",
+        "publisherDisplayName": "Cofense Support",
+        "descriptionHtml": "<p><strong>Note:</strong> Please refer to the following before installing the solution:</p>\n<p>• Review the solution <a href=\"https://github.com/Azure/Azure-Sentinel/tree/master/Solutions/CofenseTriage/ReleaseNotes.md\">Release Notes</a></p>\n<p>• There may be <a href=\"https://aka.ms/sentinelsolutionsknownissues\">known issues</a> pertaining to this Solution, please refer to them before installing.</p>\n<p>Cofense Triage for Microsoft Sentinel.</p>\n<p><strong>Data Connectors:</strong> 1, <strong>Workbooks:</strong> 1</p>\n<p><a href=\"https://aka.ms/azuresentinel\">Learn more about Microsoft Sentinel</a> | <a href=\"https://aka.ms/azuresentinelsolutionsdoc\">Learn more about Solutions</a></p>\n",
+        "contentKind": "Solution",
+        "contentProductId": "[variables('_solutioncontentProductId')]",
+        "id": "[variables('_solutioncontentProductId')]",
+        "icon": "<img src=\"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Logos/CofenseTriage.svg\" width=\"75px\" height=\"75px\">",
         "contentId": "[variables('_solutionId')]",
         "parentId": "[variables('_solutionId')]",
         "source": {

Solutions/CofenseTriage/Package/testParameters.json

@@ -0,0 +1,32 @@
+{
+  "location": {
+    "type": "string",
+    "minLength": 1,
+    "defaultValue": "[resourceGroup().location]",
+    "metadata": {
+      "description": "Not used, but needed to pass arm-ttk test `Location-Should-Not-Be-Hardcoded`.  We instead use the `workspace-location` which is derived from the LA workspace"
+    }
+  },
+  "workspace-location": {
+    "type": "string",
+    "defaultValue": "",
+    "metadata": {
+      "description": "[concat('Region to deploy solution resources -- separate from location selection',parameters('location'))]"
+    }
+  },
+  "workspace": {
+    "defaultValue": "",
+    "type": "string",
+    "metadata": {
+      "description": "Workspace name for Log Analytics where Microsoft Sentinel is setup"
+    }
+  },
+  "workbook1-name": {
+    "type": "string",
+    "defaultValue": "CofenseTriageThreatIndicators",
+    "minLength": 1,
+    "metadata": {
+      "description": "Name for the workbook"
+    }
+  }
+}

Solutions/CofenseTriage/ReleaseNotes.md

@@ -0,0 +1,3 @@
+| **Version** | **Date Modified (DD-MM-YYYY)** | **Change History**                          |
+|-------------|--------------------------------|---------------------------------------------|
+| 3.0.0       | 20-06-2024                     | Updated Data Connector to retry failed indicators.                    |