V1 AWS CloudTrail Ingestion using AWS Lambda

DataConnectors/AWS-CloudTrail-Ingestion-Lambda/IngestCloudTrailEventsToSentinel.ps1

@@ -0,0 +1,307 @@
+# PowerShell script file to be executed as a AWS Lambda function.
+#
+# When executing in Lambda the following variables will be predefined.
+#   $LambdaInput - A PSObject that contains the Lambda function input data.
+#   $LambdaContext - An Amazon.Lambda.Core.ILambdaContext object that contains information about the currently running Lambda environment.
+#
+# The last item in the PowerShell pipeline will be returned as the result of the Lambda function.
+#
+# To include PowerShell modules with your Lambda function, like the AWS.Tools.S3 module, add a "#Requires" statement
+# indicating the module and version. If using an AWS.Tools.* module the AWS.Tools.Common module is also required.
+#
+# The following link contains documentation describing the structure of the S3 event object.
+# https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
+#
+# This example demonstrates how to process an S3 Event that follows the process:
+# S3 Event -> SNS Topic -> Lambda Function
+
+#Requires -Modules @{ModuleName='AWS.Tools.Common';ModuleVersion='4.1.5.0'}
+#Requires -Modules @{ModuleName='AWS.Tools.S3';ModuleVersion='4.1.5.0'}
+#Requires -Modules @{ModuleName='AWS.Tools.SecretsManager';ModuleVersion='4.1.5.0'}
+
+# Uncomment to send the input event to CloudWatch Logs
+#Write-Host (ConvertTo-Json -InputObject $LambdaInput -Compress -Depth 5)
+#$PSVersionTable
+
+# Get the current universal time in the default string format.
+$currentUTCtime = (Get-Date).ToUniversalTime()
+
+# Code to retrieve credentials from AWS Secrets Manager
+$secretName = $env:SecretName
+$secretValue = ConvertFrom-Json (Get-SECSecretValue -SecretId $secretName -ErrorAction Stop -Verbose).SecretString -ErrorAction Stop
+
+$workspaceId = $secretValue.LAWID
+$workspaceKey = $secretValue.LAWKEY
+$LATableName = $env:LogAnalyticsTableName
+$ResourceID = ''  
+
+#The $eventobjectlist is the Json Parameter field names that form the core of the Json message that we want in the ALL Table in Log Ananlytics
+$eventobjectlist = @('eventTime', 'eventVersion', 'userIdentity', 'eventSource', 'eventName', 'awsRegion', 'sourceIPAddress', 'userAgent', 'errorCode', 'errorMessage', 'requestID', 'eventID', 'eventType', 'apiVersion', 'managementEvent', 'readOnly', 'resources', 'recipientAccountId', 'serviceEventDetails', 'sharedEventID', 'vpcEndpointId', 'eventCategory', 'additionalEventData')
+
+
+Function Expand-GZipFile {
+    Param(
+        $infile,
+        $outfile       
+    )
+	Write-Host "Processing Expand-GZipFile for: infile = $infile, outfile = $outfile"
+    $inputfile = New-Object System.IO.FileStream $infile, ([IO.FileMode]::Open), ([IO.FileAccess]::Read), ([IO.FileShare]::Read)	
+    $output = New-Object System.IO.FileStream $outfile, ([IO.FileMode]::Create), ([IO.FileAccess]::Write), ([IO.FileShare]::None)	
+    $gzipStream = New-Object System.IO.Compression.GzipStream $inputfile, ([IO.Compression.CompressionMode]::Decompress)	
+	
+    $buffer = New-Object byte[](1024)	
+    while ($true) {
+        $read = $gzipstream.Read($buffer, 0, 1024)		
+        if ($read -le 0) { break }		
+		$output.Write($buffer, 0, $read)		
+	}
+	
+    $gzipStream.Close()
+    $output.Close()
+    $inputfile.Close()
+}
+
+#function to create HTTP Header signature required to authenticate post
+Function New-BuildSignature {
+    param(
+        $customerId, 
+        $sharedKey, 
+        $date, 
+        $contentLength, 
+        $method, 
+        $contentType, 
+        $resource )
+    
+    $xHeaders = "x-ms-date:" + $date
+    $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource
+    $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash)
+    $keyBytes = [Convert]::FromBase64String($sharedKey)
+    $sha256 = New-Object System.Security.Cryptography.HMACSHA256
+    $sha256.Key = $keyBytes
+    $calculatedHash = $sha256.ComputeHash($bytesToHash)
+    $encodedHash = [Convert]::ToBase64String($calculatedHash)
+    $authorization = 'SharedKey {0}:{1}' -f $customerId, $encodedHash
+    return $authorization
+}
+        
+# Function to create and post the request
+Function Invoke-LogAnalyticsData {
+    Param( 
+        $CustomerId, 
+        $SharedKey, 
+        $Body, 
+        $LogTable, 
+        $TimeStampField,
+        $resourceId)
+
+    $method = "POST"
+    $contentType = "application/json"
+    $resource = "/api/logs"
+    $rfc1123date = [DateTime]::UtcNow.ToString("r")
+    $contentLength = $Body.Length
+    $signature = New-BuildSignature `
+        -customerId $CustomerId `
+        -sharedKey $SharedKey `
+        -date $rfc1123date `
+        -contentLength $contentLength `
+        -method $method `
+        -contentType $contentType `
+        -resource $resource
+    $uri = "https://" + $CustomerId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01"
+    $headers1 = @{
+        "Authorization"        = $signature;
+        "Log-Type"             = $LogTable;
+        "x-ms-date"            = $rfc1123date;
+        "x-ms-AzureResourceId" = $resourceId;
+        "time-generated-field" = $TimeStampField;
+    }  
+    $status = $false
+    do {
+        $response = Invoke-WebRequest -Uri $uri -Method $method -ContentType $contentType -Headers $headers1 -Body $Body
+        If ($reponse.StatusCode -eq 429) {
+            $rand = get-random -minimum 10 -Maximum 80
+            start-sleep -seconds $rand 
+        }
+        else { $status = $true }
+    }until($status) 
+    Remove-variable -name Body
+    return $response.StatusCode
+    
+}
+
+
+foreach ($snsRecord in $LambdaInput.Records)
+{
+    $snsMessage = ConvertFrom-Json -InputObject $snsRecord.Sns.Message
+    foreach ($s3Event in $snsMessage.Records)
+    {
+        $s3BucketName = $s3Event.s3.bucket.name
+        $s3BucketKey = $s3Event.s3.object.key
+
+        Write-Host "Processing event for: bucket = $s3BucketName, key = $s3BucketKey"
+	
+		IF ($Null -ne $s3BucketName -and $Null -ne $s3BucketKey) {
+			$s3KeyPath = $s3BucketKey -Replace ('%3A', ':')			
+			$fileNameSplit = $s3KeyPath.split('/')			
+			$fileSplits = $fileNameSplit.Length - 1			
+			$fileName = $filenameSplit[$fileSplits].replace(':', '_')			
+			$downloadedFile = Read-S3Object -BucketName $s3BucketName -Key $s3BucketKey -File "/tmp/$filename"			
+			Write-Host "Object $s3BucketKey is $($downloadedFile.Size) bytes; Extension is $($downloadedFile.Extension)"
+			
+			if ($downloadedFile.Extension -eq '.gz' ) {
+				$infile = "/tmp/$filename"				
+				$outfile = "/tmp/" + $filename -replace ($downloadedFile.Extension, '')					
+				Expand-GZipFile $infile.Trim() $outfile.Trim()
+				$null = Remove-Item -Path $infile -Force -Recurse -ErrorAction Ignore
+				$filename = $filename -replace ($downloadedFile.Extension, '')
+				$filename = $filename.Trim()
+			}
+    
+			$logEvents = Get-Content -Raw -LiteralPath ("/tmp/$filename" ) 
+			$logEvents = $LogEvents.Substring(0, ($LogEvents.length) - 1)
+			$LogEvents = $LogEvents -Replace ('{"Records":', '')
+			$loglength = $logEvents.Length    
+			$logevents = Convertfrom-json $LogEvents -AsHashTable
+			$groupevents = @{}
+			$coreEvents = @()
+			$eventSources = @()
+			Foreach ($log in $logevents) {
+				$Logdetails = @{}
+				$Logdetails1 = @{}
+				$b = ((($log.eventSource).split('.'))[0]) -replace ('-', '')
+				IF ($b -eq 'ec2') {
+					foreach ($col in $eventobjectlist) {
+						$logdetails1 += @{$col = $log.$col }
+					}
+					$ec2Header = $b + '_Header'
+					IF ($null -eq $groupevents[$ec2Header]) {
+						Add-Member -inputobject $groupevents -Name $b -MemberType NoteProperty -value @() -Force
+						$groupevents[$ec2Header] = @()
+						$eventSources += $ec2Header
+					}
+					$groupevents[$ec2Header] += $Logdetails1
+					$Ec2Request = $b + '_Request'
+					IF ($null -eq $groupevents[$Ec2Request]) {
+						Add-Member -inputobject $groupevents -Name $Ec2Request -MemberType NoteProperty -value @() -Force
+						$groupevents[$Ec2Request] = @()
+						$eventSources += $Ec2Request
+					}
+					$ec2Events = @{} 
+					$ec2Events += @{'eventID' = $log.eventID }
+					$ec2Events += @{'awsRegion' = $log.awsRegion }
+					$ec2Events += @{'requestID' = $log.requestID }
+					$ec2Events += @{'eventTime' = $log.eventTime }
+					$ec2Events += @{'requestParameters' = $log.requestParameters }
+					$groupevents[$Ec2Request] += $ec2Events
+					$Ec2Response = $b + '_Response'
+					IF ($null -eq $groupevents[$Ec2Response]) {
+						Add-Member -inputobject $groupevents -Name $Ec2Response -MemberType NoteProperty -value @() -Force
+						$groupevents[$Ec2Response] = @()
+						$eventSources += $Ec2Response
+					}
+					$ec2Events = @{} 
+					$ec2Events += @{'eventID' = $log.eventID }
+					$ec2Events += @{'awsRegion' = $log.awsRegion }
+					$ec2Events += @{'requestID' = $log.requestID }
+					$ec2Events += @{'eventTime' = $log.eventTime }
+					$ec2Events += @{'responseElements' = $log.responseElements }
+					$groupevents[$Ec2Response] += $ec2Events
+				}
+				Else {
+					IF ($null -eq $groupevents[$b]) {
+						Add-Member -inputobject $groupevents -Name $b -MemberType NoteProperty -value @() -Force
+						$groupevents[$b] = @()
+						$eventSources += $b
+					}
+					$groupevents[$b] += $log
+				}
+				foreach ($col in $eventobjectlist) {
+					$logdetails += @{$col = $log.$col }
+				}
+				$coreEvents += $Logdetails
+			
+			}
+
+			$coreJson = convertto-json $coreevents -depth 5 -Compress    
+			$Table = "$LATableName" + "_All"
+			IF (($corejson.Length) -gt 28MB) {
+				Write-Host "Log length is greater than 28 MB, splitting and sending to Log Analytics"
+				$bits = [math]::Round(($corejson.length) / 20MB) + 1
+				$TotalRecords = $coreEvents.Count
+				$RecSetSize = [math]::Round($TotalRecords / $bits) + 1
+				$start = 0
+				For ($x = 0; $x -lt $bits; $X++) {
+					IF ( ($start + $recsetsize) -gt $TotalRecords) {
+						$finish = $totalRecords
+					}
+					ELSE {
+						$finish = $start + $RecSetSize
+					}
+					$body = Convertto-Json ($coreEvents[$start..$finish]) -Depth 5 -Compress
+					$result = Invoke-LogAnalyticsData -CustomerId $workspaceId -SharedKey $workspaceKey -Body $body -LogTable $Table -TimeStampField 'eventTime' -ResourceId $ResourceID          
+					if ($result -eq 200)
+					{
+						Write-Host "CloudTrail Logs successfully ingested to LogAnalytics Workspace under Custom Logs --> Table: $Table"
+					}
+					$start = $Finish + 1
+				}
+				$null = Remove-variable -name body        
+
+			}
+			Else {
+				#$logEvents = Convertto-Json $events -depth 20 -compress
+				$result = Invoke-LogAnalyticsData -CustomerId $workspaceId -SharedKey $workspaceKey -Body $coreJson -LogTable $Table -TimeStampField 'eventTime' -ResourceId $ResourceID
+				if ($result -eq 200)
+				{
+					Write-Host "CloudTrail Logs successfully ingested to LogAnalytics Workspace under Custom Logs --> Table: $Table"
+				}
+			}
+    
+			$null = remove-variable -name coreEvents
+			$null = remove-variable -name coreJson
+			
+			$RecCount = 0
+			foreach ($d in $eventSources) { 
+				#$events = $groupevents[$d]
+				$eventsJson = ConvertTo-Json $groupevents[$d] -depth 5 -Compress
+				$Table = $LATableName + '_' + $d
+				$TotalRecords = $groupevents[$d].Count
+				$recCount += $TotalRecords
+				IF (($eventsjson.Length) -gt 28MB) {
+					#$events = Convertfrom-json $corejson
+					$bits = [math]::Round(($eventsjson.length) / 20MB) + 1
+					$TotalRecords = $groupevents[$d].Count
+					$RecSetSize = [math]::Round($TotalRecords / $bits) + 1
+					$start = 0
+					For ($x = 0; $x -lt $bits; $X++) {
+						IF ( ($start + $recsetsize) -gt $TotalRecords) {
+							$finish = $totalRecords
+						}
+						ELSE {
+							$finish = $start + $RecSetSize
+						}
+						$body = Convertto-Json ($groupevents[$d][$start..$finish]) -Depth 5 -Compress
+						$result = Invoke-LogAnalyticsData -CustomerId $workspaceId -SharedKey $workspaceKey -Body $body -LogTable $Table -TimeStampField 'eventTime' -ResourceId $ResourceID                
+						if ($result -eq 200)
+						{
+							Write-Host "CloudTrail Logs successfully ingested to LogAnalytics Workspace under Custom Logs --> Table: $Table"
+						}
+						$start = $Finish + 1
+					}
+					$null = Remove-variable -name body        
+				}
+				Else {
+					#$logEvents = Convertto-Json $events -depth 20 -compress
+					$result = Invoke-LogAnalyticsData -CustomerId $workspaceId -SharedKey $workspaceKey -Body $eventsJson -LogTable $Table -TimeStampField 'eventTime' -ResourceId $ResourceID
+					if ($result -eq 200)
+					{
+						Write-Host "CloudTrail Logs successfully ingested to LogAnalytics Workspace under Custom Logs --> Table: $Table"
+					}
+				}
+			}
+    
+			$null = Remove-Variable -Name groupevents
+			$null = Remove-Variable -Name LogEvents			
+		}
+	}
+}

DataConnectors/AWS-CloudTrail-Ingestion-Lambda/README.md

@@ -0,0 +1,104 @@
+# AWS Lambda Function to import CloudTrail Logs to Azure Sentinel
+This Lambda function is designed to ingest AWS CloudTrail Events and send them to Azure Log Analytics workspace using the Log Analytics API.
+
+AWS CloudTrail logs are audit type events from all/any AWS resources in a tenancy. Each AWS resource has a unique set of Request and Response Parameters. Azure Log Analytics has a column per table limit of 500, (plus some system columns) the aggregate of AWS parameter fields will exceed this quickly leading to potential loss of event records
+
+Code does the following things with the logs it processes. 
+1.	Takes the core fields of the record. i.e. all fields except for the Request and Response associated fields and puts them in a Table_ALL. providing a single table with all records with core event information. 
+2.	Looks at each event and puts it into a table with an extension <AWSREsourceType> i.e. AwsCloudTrail_s3 
+3.	Exception to 2 above is for EC2 events. the volume of fields for EC2 Request and Response parameters exceeds 500 columns. EC2 data is split into 3 tables, Header, Request & Response. 
+4.	In future if other AWS datatypes exceed 500 columns a similar split may be required for them as well. 
+5.	The processing of Data as described in 3 will lead to some data being ingested into 2 or more different tables and increase the log ingestion metrics\billing. The customer can decide they don't want the _ALL table and this would remove the duplicate data storage volume
+
+Special thanks to [Chris Abberley](https://github.com/cabberley) for the above logic
+
+## **Function Flow process**
+CloudTrail Logs --> AWS S3 --> AWS SNS Topic --> AWS Lambda --> Azure Log Analytics
+![Picture9](./Graphics/Picture9.png)
+
+## Installation / Setup Guide
+
+## **Pre-requisites**
+
+This function requires AWS Secrets Manager to store Azure Log Analytics WorkspaceId and WorkspaceKey
+
+![Picture10](./Graphics/Picture10.png)
+### **Option 1**
+
+### Machine Setup
+To deploy this, you will need a machine prepared with the following:
+ - PowerShell Core – I recommend PowerShell 7 [found here](https://github.com/PowerShell/PowerShell/releases)
+ - .Net Core 3.1 SDK [found here](https://dotnet.microsoft.com/download) 
+ - AWSLambdaPSCore module – You can install this either from the [PowerShell Gallery](https://www.powershellgallery.com/packages?q=AWSLambdaPSCore), or you can install it by using the following PowerShell Core shell command:  
+```powershell
+Install-Module AWSLambdaPSCore -Scope CurrentUser
+```
+See the documentation here https://docs.aws.amazon.com/lambda/latest/dg/powershell-devenv.html 
+
+I recommend you review https://docs.aws.amazon.com/lambda/latest/dg/powershell-package.html to review the cmdlets that are part of AWSLambdaPSCore.
+
+Note: If the environment uses a proxy, you may need to add the following to VSCode profile
+```powershell
+Added to VS Code profile:
+$webclient=New-Object System.Net.WebClient
+$webclient.Proxy.Credentials = [System.Net.CredentialCache]::DefaultNetworkCredentials
+```
+
+### Create the Lambda Function
+To deploy the PowerShell script, you can create a Package (zip file) to upload to the AWS console or you can use the Publish-AWSPowerShell cmdlet.
+```powershell
+Publish-AWSPowerShellLambda -Name YourLambdaNameHere -ScriptPath <path>/IngestCloudTrailEventsToSentinel.ps1 -Region <region> -IAMRoleArn <arn of role created earlier> -ProfileName <profile>
+```
+You might need –ProfileName if your configuration of .aws/credentials file doesn't contain a default.  See this document for information on setting up your AWS credentials. 
+
+### **Option 2**
+1.	Create a new AWS Lambda and select "Author from scratch"
+2.	Give Function Name and select Runtime ".NET Core 2.1 (C#/PowerShell)" and click Create function
+3.	After successful creation, now you can change its code and configuration 
+4.	Under Function code, click on Actions --> Upload a .zip file (/aws-data-connector-az-sentinel/blob/main/IngestCloudTrailEventsToSentinel.zip)
+5.	Follow the steps in "### Lambda Configuration" from step 2
+
+### **Note: Either you choose Option 1/Option 2, the following configuration steps are mandatory.**
+
+### **Lambda Configuration**
+1. Once created, login to the AWS console. In Find services, search for Lambda. Click on Lambda.
+![Picture1](./Graphics/Picture1.png)
+
+2. Click on the lambda function name you used with the cmdlet.  Click Environment Variables and add the following
+```
+SecretName
+LogAnalyticsTableName
+```
+![Picture4](./Graphics/Picture4.png)
+3. Click on the lambda function name you used with the cmdlet.Click Add Trigger 
+![Picture2](./Graphics/Picture2.png)
+4. Select SNS.  Select the SNS Name. Click Add. 
+![Picture3](./Graphics/Picture3.png)
+
+5. Create AWS Role : The Lambda function will need an execution role defined that grants access to the S3 bucket and CloudWatch logs.  To create an execution role: 
+	
+	1. Open the [roles](https://console.aws.amazon.com/iam/home#/roles) page in the IAM console. 
+	2. Choose Create role. 
+	3. Create a role with the following properties. 
+		 - Trusted entity – AWS Lambda. 		 
+		 - Role name – AWSSNStoAzureSentinel. 
+		 - Permissions – AWSLambdaBasicExecutionRole &  AmazonS3ReadOnlyAccess & secretsmanager:GetSecretValue & kms:Decrypt - required only if you use a customer-managed AWS KMS key to encrypt the secret. You do not need this permission to use the account's default AWS managed CMK for Secrets Manager
+
+	The AWSLambdaExecute policy has the permissions that the function needs to manage objects in Amazon S3 and write logs to CloudWatch Logs. Copy the arn of the role created as you will need it for the next step. 
+
+6. Your lambda function is ready to send data to Log Analytics.
+
+### **Test the function**
+1. To test your function, Perform some actions like Start EC2, Stop EC2, Login into EC2, etc.,. 
+2. To see the logs, go the Lambda function.  Click Monitoring tab. Click view logs in CloudWatch. 
+![Pciture5](./Graphics/Picture5.png)
+3. In CloudWatch, you will see each log stream from the runs. Select the latest.
+![Picture6](./Graphics/Picture6.png)
+
+4. Here you can see anything from the script from the Write-Host cmdlet. 
+
+![Picture7](./Graphics/Picture7.png)
+
+5. Go to portal.azure.com and verify your data is in the custom log. 
+
+![Picture8](./Graphics/Picture8.png)