Merge pull request #916 from gene1wood/add-es-service-link-role

Add ElasticSearch service linked IAM role

.gitignore

@@ -13,4 +13,5 @@ alerts/generic_alerts
 /.project
 /data
 .vscode
-cloudy_mozdef/aws_parameters.json
+cloudy_mozdef/aws_parameters.json
+cloudy_mozdef/aws_parameters.sh

cloudy_mozdef/Makefile

@@ -8,7 +8,8 @@ STACK_PARAMS	:= file://aws_parameters.json
 S3_BUCKET_NAME  := mozdef.infosec.allizom.org
 S3_BUCKET_PATH	:= cf
 S3_BUCKET_URI	:= s3://$(S3_BUCKET_NAME)/$(S3_BUCKET_PATH)
-S3_STACK_URI	:= https://s3-$(AWS_REGION).amazonaws.com/$(S3_BUCKET_NAME)/$(S3_BUCKET_PATH)/mozdef-parent.yml
+S3_STACK_URI	:= https://s3-$(AWS_REGION).amazonaws.com/$(S3_BUCKET_NAME)/$(S3_BUCKET_PATH)/
+# OIDC_CLIENT_SECRET is set in an environment variable by running "source aws_parameters.sh"
 
 all:
 	@echo 'Available make targets:'
@@ -23,10 +24,12 @@ packer-build: ## Build the base AMI with packer
 .PHONY: create-stack
 create-stack: test ## Create everything you need for a fresh new stack!
 	@export AWS_REGION=$(AWS_REGION)
-	@echo "Make sure you have a param file ($(STACK_PARAMS)) with OIDCClientSecret set."
-	aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI) \
+	@echo "Make sure you have an environment variable OIDC_CLIENT_SECRET set."
+	aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI)mozdef-parent.yml \
 	  --capabilities CAPABILITY_IAM \
 	  --parameters $(STACK_PARAMS) \
+	  --parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_STACK_URI) \
+	               ParameterKey=OIDCClientSecret,ParameterValue=$(OIDC_CLIENT_SECRET) \
 	  --output text
 
 .PHONY: create-s3-bucket
@@ -37,9 +40,10 @@ create-s3-bucket:
 .PHONY: updated-nested-stack
 update-stack: test ## Updates the nested stack on AWS
 	@export AWS_REGION=$(AWS_REGION)
-	aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI) \
+	aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI)mozdef-parent.yml \
 	  --capabilities CAPABILITY_IAM \
-	  --parameters $(STACK_PARAMS) \
+	  --parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_STACK_URI) \
+	               ParameterKey=OIDCClientSecret,ParameterValue=$(OIDC_CLIENT_SECRET) \
 	  --output text
 
 # --ignore-checks=E2502 : https://github.com/awslabs/cfn-python-lint/issues/408

cloudy_mozdef/aws_parameters.example.sh

@@ -0,0 +1 @@
+export OIDC_CLIENT_SECRET=secretgoeshere

cloudy_mozdef/cloudformation/base-iam.yml

@@ -14,9 +14,15 @@ Parameters:
   MozDefSQSQueueArn:
     Type: String
     Description: The ARN of the SQS queue that receives events destined for MozDef
+  ESServiceLinkedRoleExists:
+    Type: String
+    Description: Does the ES Service Linked Role already exist. true or false
+    Default: 'true'
 Conditions:
   IAMRoleSet:
     !Not [!Equals [!Ref CloudTrailS3BucketIAMRoleArn, '']]
+  CreateESServiceLinkedRole:
+    !Not [!Equals [!Ref ESServiceLinkedRoleExists, 'true']]  # Match either True or 'true' but not 'True'
 Resources:
   MozDefCloudTrailPolicy:
     Type: AWS::IAM::ManagedPolicy
@@ -116,6 +122,12 @@ Resources:
     Properties:
       Roles:
         - Ref: MozDefIAMRole
+  ESServiceLinkedIAMRole:
+    Type: "AWS::IAM::ServiceLinkedRole"
+    Condition: CreateESServiceLinkedRole
+    DeletionPolicy: Retain
+    Properties:
+      AWSServiceName: es.amazonaws.com
 Outputs:
   InstanceProfileArn:
     Description: The arn of the Instance Profile

cloudy_mozdef/cloudformation/mozdef-parent.yml

@@ -72,6 +72,7 @@ Resources:
         CloudTrailSQSQueueArn: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueArn
         MozDefSQSQueueArn: !GetAtt MozDefSQS.Outputs.SQSQueueArn
         # CloudTrailS3BucketIAMRoleArn we leave empty as we will consume CloudTrail logs from our own account
+        ESServiceLinkedRoleExists: !GetAtt ESServiceLinkedRoleExists.RoleExists
       Tags:
         - Key: application
           Value: mozdef
@@ -105,6 +106,7 @@ Resources:
       TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-instance.yml ] ]
   MozDefES:
     Type: AWS::CloudFormation::Stack
+    DependsOn: MozDefIAMRoleAndInstanceProfile
     Properties:
       Parameters:
         SubnetIds: !Join [ ',', !Ref PublicSubnetIds ]
@@ -172,6 +174,7 @@ Resources:
                 Effect: Allow
                 Action:
                   - logs:*
+                  - iam:ListRoles
                 Resource: '*'
   GetArrayLengthLambdaFunction:
     Type: AWS::Lambda::Function
@@ -204,3 +207,42 @@ Resources:
     Properties:
       Array: !Ref PublicSubnetIds
       ServiceToken: !GetAtt GetArrayLengthLambdaFunction.Arn
+  DoesRoleExistLambdaFunction:
+    Type: AWS::Lambda::Function
+    DependsOn: CloudFormationLambdaIAMRole
+    # This DependsOn shouldn't be needed because the "Role" value is set to
+    # "!GetAtt CloudFormationLambdaIAMRole.Arn" but without DependsOn the error
+    # "Template error: IAM role mozdef-aws-nested-CloudFormationLambdaIAMRole-108UCUPESC6WG doesn't exist"
+    # occurs on stack creation for this Lambda Function resource. The DependsOn
+    # prevents the error.
+    Properties:
+      Code:
+        ZipFile: |
+          import cfnresponse
+          import boto3, secrets, string
+          def handler(event, context):
+              paginator = boto3.client('iam').get_paginator('list_roles')
+              args = {'PathPrefix': event['ResourceProperties']['PathPrefix']} if 'PathPrefix' in event['ResourceProperties'] else {}
+              iterator = paginator.paginate(**args).search(
+                  "Roles[?RoleName == '%s'][]" % event['ResourceProperties']['RoleName'])
+              response = {'RoleExists': len([x for x in iterator]) > 0}
+              physical_id = ''.join(
+                  secrets.choice(string.ascii_uppercase + string.digits) for i in
+                  range(13))
+              cfnresponse.send(event, context, cfnresponse.SUCCESS, response,
+                               "DoesRoleExist-%s" % physical_id)
+      Handler: index.handler
+      Runtime: python3.6
+      Role: !GetAtt CloudFormationLambdaIAMRole.Arn
+      Tags:
+        - Key: application
+          Value: mozdef
+        - Key: stack
+          Value: !Ref AWS::StackName
+      Timeout: 20
+  ESServiceLinkedRoleExists:
+    Type: AWS::CloudFormation::CustomResource
+    Properties:
+      RoleName: AWSServiceRoleForAmazonElasticsearchService
+      PathPrefix: '/aws-service-role/es.amazonaws.com/'
+      ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn

cloudy_mozdef/dmake

@@ -37,4 +37,5 @@ exec docker run --rm --name ${CONTAINER_NAME} \
   -e "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}" \
   -e "AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}" \
   -e "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" \
+  -e "OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET}"
   ${HUB}/${IMG_NAME}:latest make $@