diff --git a/NwNsgProject/Stage3QueueTrigger.cs b/NwNsgProject/Stage3QueueTrigger.cs index b6ad51a..740c5d0 100644 --- a/NwNsgProject/Stage3QueueTrigger.cs +++ b/NwNsgProject/Stage3QueueTrigger.cs @@ -9,6 +9,8 @@ using Newtonsoft.Json; using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Security.Cryptography.X509Certificates; +using System.Net.Security; namespace NwNsgProject { @@ -73,6 +75,16 @@ namespace NwNsgProject public static async Task SendMessagesDownstream(string myMessages, TraceWriter log) { + // + // myMessages looks like this: + // { + // "records":[ + // {...}, + // {...} + // ... + // ] + // } + // string outputBinding = Util.GetEnvironmentVariable("outputBinding"); if (outputBinding.Length == 0) { @@ -88,6 +100,9 @@ namespace NwNsgProject case "arcsight": await obArcsight(myMessages, log); break; + case "splunk": + await obSplunk(myMessages, log); + break; } } @@ -254,8 +269,20 @@ namespace NwNsgProject log.Error($"Exception logging record: {ex.Message}"); } } + static async Task obArcsight(string newClientContent, TraceWriter log) { + // + // newClientContent looks like this: + // + // { + // "records":[ + // {...}, + // {...} + // ... + // ] + // } + // string arcsightAddress = Util.GetEnvironmentVariable("arcsightAddress"); string arcsightPort = Util.GetEnvironmentVariable("arcsightPort"); @@ -304,6 +331,187 @@ namespace NwNsgProject await stream.FlushAsync(); } + public static bool ValidateMyCert(object sender, X509Certificate cert, X509Chain chain, SslPolicyErrors sslErr) + { + var splunkCertThumbprint = Util.GetEnvironmentVariable("splunkCertThumbprint"); + + // if user has not configured a cert, anything goes + if (splunkCertThumbprint == "") + return true; + + // if user has configured a cert, must match + var thumbprint = cert.GetCertHashString(); + if (thumbprint == splunkCertThumbprint) + return true; + + return false; + } + + static async Task obSplunk(string newClientContent, TraceWriter log) + { + // + // newClientContent looks like this: + // + // { + // "records":[ + // {...}, + // {...} + // ... + // ] + // } + // + + string splunkAddress = Util.GetEnvironmentVariable("splunkAddress"); + string splunkToken = Util.GetEnvironmentVariable("splunkToken"); + + if (splunkAddress.Length == 0 || splunkToken.Length == 0) + { + log.Error("Values for splunkAddress and splunkToken are required."); + return; + } + + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + ServicePointManager.ServerCertificateValidationCallback += new RemoteCertificateValidationCallback(ValidateMyCert); + + var transmission = new StringBuilder(); + foreach (var message in convertToSplunk(newClientContent, null, log)) + { + // + // message looks like this: + // + // { + // "time": "xxx", + // "category": "xxx", + // "operationName": "xxx", + // "version": "xxx", + // "deviceExtId": "xxx", + // "flowOrder": "xxx", + // "nsgRuleName": "xxx", + // "dmac|smac": "xxx", + // "rt": "xxx", + // "src": "xxx", + // "dst": "xxx", + // "spt": "xxx", + // "dpt": "xxx", + // "proto": "xxx", + // "deviceDirection": "xxx", + // "act": "xxx" + // } + transmission.Append(GetSplunkEventFromMessage(message)); + } + + var client = new SingleHttpClientInstance(); + try + { + HttpRequestMessage req = new HttpRequestMessage(HttpMethod.Post, splunkAddress); + req.Headers.Accept.Clear(); + req.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + req.Headers.Add("Authorization", "Splunk " + splunkToken); + req.Content = new StringContent(transmission.ToString(), Encoding.UTF8, "application/json"); + HttpResponseMessage response = await SingleHttpClientInstance.SendToSplunk(req); + if (response.StatusCode != HttpStatusCode.OK) + { + throw new System.Net.Http.HttpRequestException($"StatusCode from Splunk: {response.StatusCode}, and reason: {response.ReasonPhrase}"); + } + } + catch (System.Net.Http.HttpRequestException e) + { + throw new System.Net.Http.HttpRequestException("Sending to Splunk. Is Splunk service running?", e); + } + catch (Exception f) + { + throw new System.Exception("Sending to Splunk. Unplanned exception.", f); + } + + } + + static System.Collections.Generic.IEnumerable convertToSplunk(string newClientContent, Binder errorRecordBinder, TraceWriter log) + { + // + // newClientContent looks like this: + // + // { + // "records":[ + // {...}, + // {...} + // ... + // ] + // } + // + + NSGFlowLogRecords logs = JsonConvert.DeserializeObject(newClientContent); + + string logIncomingJSON = Util.GetEnvironmentVariable("logIncomingJSON"); + Boolean flag; + if (Boolean.TryParse(logIncomingJSON, out flag)) + { + if (flag) + { + logErrorRecord(newClientContent, errorRecordBinder, log).Wait(); + } + } + + var sbBase = new StringBuilder(); + foreach (var record in logs.records) + { + sbBase.Clear(); + sbBase.Append("{"); + sbBase.Append("\"time\":\"").Append(record.time).Append("\""); + sbBase.Append(",\"category\":\"").Append(record.category).Append("\""); + sbBase.Append(",\"operationName\":\"").Append(record.operationName).Append("\""); + sbBase.Append(",\"version\":\"").Append(record.properties.Version.ToString("0.0")).Append("\""); + sbBase.Append(",\"deviceExtId\":\"").Append(record.MakeDeviceExternalID()).Append("\""); + + int count = 1; + var sbOuterFlowRecord = new StringBuilder(); + foreach (var outerFlows in record.properties.flows) + { + sbOuterFlowRecord.Clear(); + sbOuterFlowRecord.Append(sbBase.ToString()); + sbOuterFlowRecord.Append(",\"flowOrder\":\"").Append(count).Append("\""); + sbOuterFlowRecord.Append(",\"nsgRuleName\":\"").Append(outerFlows.rule).Append("\""); + + var sbInnerFlowRecord = new StringBuilder(); + foreach (var innerFlows in outerFlows.flows) + { + sbInnerFlowRecord.Clear(); + sbInnerFlowRecord.Append(sbOuterFlowRecord.ToString()); + + var firstFlowTupleEncountered = true; + foreach (var flowTuple in innerFlows.flowTuples) + { + var tuple = new NSGFlowLogTuple(flowTuple); + + if (firstFlowTupleEncountered) + { + sbInnerFlowRecord.Append((tuple.GetDirection == "I" ? ",\"dmac\":\"" : ",\"smac\":\"")).Append(innerFlows.MakeMAC()).Append("\""); + firstFlowTupleEncountered = false; + } + + yield return sbInnerFlowRecord.Append(tuple.JsonSubString()).ToString() + "}"; + } + + } + } + } + } + + static StringBuilder sb = new StringBuilder(); + static string GetSplunkEventFromMessage(string message) + { + string json = Newtonsoft.Json.JsonConvert.SerializeObject(message); + + sb.Clear(); + sb.Append("{"); + sb.Append("\"sourcetype\": \"").Append("nsgFlowLog").Append("\","); + sb.Append("\"event\": ").Append(json); + sb.Append("}"); + + return sb.ToString(); + + } + static Byte[] AppendToTransmission(Byte[] existingMessages, string appendMessage) { Byte[] appendMessageBytes = Encoding.ASCII.GetBytes(appendMessage); @@ -354,6 +562,13 @@ namespace NwNsgProject } return response; } + + public static async Task SendToSplunk(HttpRequestMessage req) + { + HttpResponseMessage response = await HttpClient.SendAsync(req); + return response; + } + } static async Task obLogstash(string newClientContent, TraceWriter log) diff --git a/NwNsgProject/classes.cs b/NwNsgProject/classes.cs index 091a213..0cced1f 100644 --- a/NwNsgProject/classes.cs +++ b/NwNsgProject/classes.cs @@ -97,6 +97,21 @@ class NSGFlowLogTuple return temp.ToString(); } + + public string JsonSubString() + { + var sb = new StringBuilder(); + sb.Append(",\"rt\":\"").Append((Convert.ToUInt64(startTime) * 1000).ToString()).Append("\""); + sb.Append(",\"src\":\"").Append(sourceAddress).Append("\""); + sb.Append(",\"dst\":\"").Append(destinationAddress).Append("\""); + sb.Append(",\"spt\":\"").Append(sourcePort).Append("\""); + sb.Append(",\"dpt\":\"").Append(destinationPort).Append("\""); + sb.Append(",\"proto\":\"").Append((transportProtocol == "U" ? "UDP" : "TCP")).Append("\""); + sb.Append(",\"deviceDirection\":\"").Append((deviceDirection == "I" ? "0" : "1")).Append("\""); + sb.Append(",\"act\":\"").Append(deviceAction).Append("\""); + + return sb.ToString(); + } } class NSGFlowLogsInnerFlows diff --git a/NwNsgProject/sampleOutputOfConvertToSplunk.json b/NwNsgProject/sampleOutputOfConvertToSplunk.json new file mode 100644 index 0000000..d043f76 --- /dev/null +++ b/NwNsgProject/sampleOutputOfConvertToSplunk.json @@ -0,0 +1,18 @@ +{ + "time": "2018-08-13T12:56:57.7450115Z", + "category": "NetworkSecurityGroupFlowEvent", + "operationName": "NetworkSecurityGroupFlowEvents", + "version": "1.0", + "deviceExtId": "743F6ED6-83A8-46F0-822D-EA93B953952D/ESPLUNKRG/STANDALONE-NSG", + "flowOrder": "1", + "nsgRuleName": "DefaultRule_DenyAllInBound", + "dmac": "00:22:48:01:03:62:", + "rt": "1534164982000", + "src": "185.92.73.26", + "dst": "172.17.0.4", + "spt": "65533", + "dpt": "3389", + "proto": "TCP", + "deviceDirection": "0", + "act": "D" +} \ No newline at end of file diff --git a/NwNsgProject/sampleStage3Input.txt b/NwNsgProject/sampleStage3Input.txt new file mode 100644 index 0000000..e88890e --- /dev/null +++ b/NwNsgProject/sampleStage3Input.txt @@ -0,0 +1 @@ +,{"time":"2018-08-13T12:56:57.7450115Z","systemId":"7b900133-9802-46f5-a0cd-8c01a74d9106","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/743F6ED6-83A8-46F0-822D-EA93B953952D/RESOURCEGROUPS/ESPLUNKRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/STANDALONE-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":1,"flows":[{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"002248010362","flowTuples":["1534164982,185.92.73.26,172.17.0.4,65533,3389,T,I,D","1534164987,142.93.192.28,172.17.0.4,51414,22,T,I,D","1534164988,79.124.56.142,172.17.0.4,51591,778,T,I,D","1534164990,201.210.235.23,172.17.0.4,65019,445,T,I,D"]}]},{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"002248010362","flowTuples":["1534164955,172.17.0.4,51.140.168.94,36908,443,T,O,A","1534164955,172.17.0.4,51.140.168.142,52738,443,T,O,A","1534164955,172.17.0.4,51.141.128.36,42696,443,T,O,A","1534164955,172.17.0.4,51.141.128.36,42698,443,T,O,A","1534164958,172.17.0.4,51.141.128.36,42702,443,T,O,A","1534164958,172.17.0.4,51.140.168.142,52748,443,T,O,A","1534164958,172.17.0.4,51.140.168.142,52750,443,T,O,A","1534164958,172.17.0.4,51.141.128.36,42708,443,T,O,A","1534164961,172.17.0.4,40.121.36.196,52560,443,T,O,A","1534164961,172.17.0.4,51.141.128.36,42714,443,T,O,A","1534164961,172.17.0.4,51.140.168.142,52760,443,T,O,A","1534164961,172.17.0.4,51.141.128.36,42718,443,T,O,A","1534164962,172.17.0.4,51.141.128.36,42720,443,T,O,A","1534164965,172.17.0.4,51.140.168.94,36940,443,T,O,A","1534164965,172.17.0.4,51.140.168.142,52770,443,T,O,A","1534164965,172.17.0.4,51.140.168.142,52772,443,T,O,A","1534164965,172.17.0.4,51.141.128.36,42730,443,T,O,A","1534164968,172.17.0.4,51.140.168.94,36950,443,T,O,A","1534164968,172.17.0.4,51.140.168.142,52780,443,T,O,A","1534164968,172.17.0.4,51.140.168.142,52782,443,T,O,A","1534164968,172.17.0.4,51.141.128.36,42740,443,T,O,A","1534164971,172.17.0.4,51.140.168.94,36960,443,T,O,A","1534164971,172.17.0.4,51.140.168.142,52790,443,T,O,A","1534164971,172.17.0.4,51.141.128.36,42748,443,T,O,A","1534164971,172.17.0.4,51.141.128.36,42750,443,T,O,A","1534164974,172.17.0.4,51.141.128.36,42754,443,T,O,A","1534164974,172.17.0.4,51.140.168.142,52800,443,T,O,A","1534164974,172.17.0.4,51.140.168.142,52802,443,T,O,A","1534164974,172.17.0.4,51.141.128.36,42760,443,T,O,A","1534164977,172.17.0.4,51.140.168.94,36980,443,T,O,A","1534164977,172.17.0.4,51.140.168.142,52810,443,T,O,A","1534164977,172.17.0.4,51.141.128.36,42768,443,T,O,A","1534164977,172.17.0.4,51.141.128.36,42770,443,T,O,A","1534164980,172.17.0.4,51.141.128.36,42774,443,T,O,A","1534164980,172.17.0.4,51.140.168.142,52820,443,T,O,A","1534164980,172.17.0.4,51.140.168.142,52822,443,T,O,A","1534164980,172.17.0.4,51.141.128.36,42780,443,T,O,A","1534164983,172.17.0.4,51.140.168.94,37000,443,T,O,A","1534164983,172.17.0.4,51.140.168.142,52830,443,T,O,A","1534164983,172.17.0.4,51.140.168.142,52832,443,T,O,A","1534164983,172.17.0.4,51.141.128.36,42790,443,T,O,A","1534164986,172.17.0.4,51.140.168.94,37010,443,T,O,A","1534164986,172.17.0.4,51.140.168.142,52840,443,T,O,A","1534164986,172.17.0.4,51.140.168.142,52842,443,T,O,A","1534164986,172.17.0.4,51.141.128.36,42800,443,T,O,A","1534164989,172.17.0.4,51.141.128.36,42804,443,T,O,A","1534164989,172.17.0.4,51.140.168.142,52850,443,T,O,A","1534164989,172.17.0.4,51.140.168.142,52852,443,T,O,A","1534164989,172.17.0.4,51.141.128.36,42810,443,T,O,A","1534164992,172.17.0.4,51.141.128.36,42814,443,T,O,A","1534164992,172.17.0.4,51.140.168.142,52860,443,T,O,A","1534164992,172.17.0.4,51.140.168.142,52862,443,T,O,A","1534164992,172.17.0.4,51.141.128.36,42820,443,T,O,A","1534164995,172.17.0.4,51.140.168.94,37040,443,T,O,A","1534164995,172.17.0.4,51.140.168.142,52870,443,T,O,A","1534164995,172.17.0.4,51.141.128.36,42828,443,T,O,A","1534164995,172.17.0.4,51.141.128.36,42830,443,T,O,A","1534164998,172.17.0.4,51.141.128.36,42834,443,T,O,A","1534164998,172.17.0.4,51.140.168.142,52880,443,T,O,A","1534164998,172.17.0.4,51.141.128.36,42838,443,T,O,A","1534164998,172.17.0.4,51.141.128.36,42840,443,T,O,A","1534165001,172.17.0.4,51.141.128.36,42844,443,T,O,A","1534165001,172.17.0.4,51.140.168.142,52890,443,T,O,A","1534165001,172.17.0.4,51.140.168.142,52892,443,T,O,A","1534165001,172.17.0.4,51.141.128.36,42850,443,T,O,A","1534165004,172.17.0.4,51.141.128.36,42854,443,T,O,A","1534165004,172.17.0.4,51.140.168.142,52900,443,T,O,A","1534165004,172.17.0.4,51.140.168.142,52902,443,T,O,A","1534165004,172.17.0.4,51.141.128.36,42860,443,T,O,A","1534165007,172.17.0.4,51.141.128.36,42864,443,T,O,A","1534165008,172.17.0.4,51.140.168.142,52910,443,T,O,A","1534165008,172.17.0.4,51.140.168.142,52912,443,T,O,A","1534165008,172.17.0.4,51.141.128.36,42870,443,T,O,A","1534165011,172.17.0.4,51.141.128.36,42874,443,T,O,A","1534165011,172.17.0.4,51.140.168.142,52920,443,T,O,A","1534165011,172.17.0.4,51.140.168.142,52922,443,T,O,A","1534165011,172.17.0.4,51.141.128.36,42880,443,T,O,A","1534165014,172.17.0.4,51.141.128.36,42884,443,T,O,A","1534165014,172.17.0.4,51.140.168.142,52930,443,T,O,A","1534165014,172.17.0.4,51.140.168.142,52932,443,T,O,A","1534165014,172.17.0.4,51.141.128.36,42890,443,T,O,A"]}]}]}} \ No newline at end of file