Merge pull request #1049 from gene1wood/add-http-redirect

Redirect all HTTP calls to HTTPS URLs

cloudy_mozdef/Makefile

@@ -2,20 +2,21 @@ ROOT_DIR	:= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 PARENTDIR       := $(realpath ../)
 AWS_REGION	:= us-west-2
 STACK_NAME	:= mozdef-aws-nested
-STACK_PARAMS_FILENAME	:= aws_parameters.json
+DEV_STACK_PARAMS_FILENAME	:= aws_parameters.dev.json
 # For more information on the rationale behind the code in STACK_PARAMS see https://github.com/aws/aws-cli/issues/2429#issuecomment-441133480
-STACK_PARAMS	:= $(shell test -e $(STACK_PARAMS_FILENAME) && python -c 'import json,sys;f=open(sys.argv[1]);print(" ".join([",".join(["%s=\\\"%s\\\""%(k,v) for k,v in x.items()]) for x in json.load(f)]));f.close()' $(STACK_PARAMS_FILENAME))
+DEV_STACK_PARAMS	:= $(shell test -e $(DEV_STACK_PARAMS_FILENAME) && python -c 'import json,sys;f=open(sys.argv[1]);print(" ".join([",".join(["%s=\\\"%s\\\""%(k,v) for k,v in x.items()]) for x in json.load(f)]));f.close()' $(DEV_STACK_PARAMS_FILENAME))
 # MozDef uses a nested CF stack, the mozdef-parent.yml will tie all child stacks together and load them from S3
 # See also mozdef.infosec.mozilla.org bucket
-S3_BUCKET_NAME	:= mozdef.infosec.allizom.org
-S3_BUCKET_PATH	:= cf
-S3_BUCKET_URI	:= s3://$(S3_BUCKET_NAME)/$(S3_BUCKET_PATH)
+S3_DEV_BUCKET_NAME	:= mozdef.infosec.allizom.org
+S3_DEV_BUCKET_PATH	:= cf
+S3_DEV_BUCKET_URI	:= s3://$(S3_DEV_BUCKET_NAME)/$(S3_DEV_BUCKET_PATH)
+S3_DEV_STACK_URI	:= https://s3-$(AWS_REGION).amazonaws.com/$(S3_DEV_BUCKET_NAME)/$(S3_DEV_BUCKET_PATH)/
 # Location to publish templates for public consumption
-S3_PUBLISHED_BUCKET_NAME	:= public.us-west-2.infosec.mozilla.org
-S3_PUBLISHED_BUCKET_PATH	:= mozdef/cf
-S3_PUBLISHED_BUCKET_URI	:= s3://$(S3_PUBLISHED_BUCKET_NAME)/$(S3_PUBLISHED_BUCKET_PATH)
+S3_PROD_BUCKET_NAME	:= public.us-west-2.infosec.mozilla.org
+S3_PROD_BUCKET_PATH	:= mozdef/cf
+S3_PROD_BUCKET_URI	:= s3://$(S3_PROD_BUCKET_NAME)/$(S3_PROD_BUCKET_PATH)
+S3_PROD_STACK_URI	:= https://s3-$(AWS_REGION).amazonaws.com/$(S3_PROD_BUCKET_NAME)/$(S3_PROD_BUCKET_PATH)/
 
-S3_STACK_URI	:= https://s3-$(AWS_REGION).amazonaws.com/$(S3_BUCKET_NAME)/$(S3_BUCKET_PATH)/
 # OIDC_CLIENT_SECRET is set in an environment variable by running "source aws_parameters.sh"
 OIDC_CLIENT_SECRET_PARAM_ARG := $(shell test -n "$(OIDC_CLIENT_SECRET)" && echo "ParameterKey=OIDCClientSecret,ParameterValue=$(OIDC_CLIENT_SECRET)")
 
@@ -29,48 +30,56 @@ all:
 packer-build: ## Build the base AMI with packer
 	cd packer && packer build packer.json
 
-.PHONY: create-stack
-create-stack: test ## Create everything you need for a fresh new stack!
+.PHONY: create-prod-stack
+create-dev-stack: test ## Create everything you need for a fresh new stack!
 	@export AWS_REGION=$(AWS_REGION)
 	@echo "Make sure you have an environment variable OIDC_CLIENT_SECRET set."
-	aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI)mozdef-parent.yml \
+	aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
 	  --capabilities CAPABILITY_IAM \
-	  --parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_STACK_URI) \
+	  --parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_DEV_STACK_URI) \
 	               $(OIDC_CLIENT_SECRET_PARAM_ARG) \
-	               $(STACK_PARAMS) \
+	               $(DEV_STACK_PARAMS) \
 	  --output text
 
-.PHONY: create-s3-bucket
-create-s3-bucket:
+.PHONY: create-dev-s3-bucket
+create-dev-s3-bucket:
 	@export AWS_REGION=$(AWS_REGION)
-	aws s3api create-bucket --bucket $(S3_BUCKET_NAME) --acl public-read --create-bucket-configuration LocationConstraint=$(AWS_REGION)
+	aws s3api create-bucket --bucket $(S3_DEV_BUCKET_NAME) --acl public-read --create-bucket-configuration LocationConstraint=$(AWS_REGION)
 
-.PHONY: updated-nested-stack
-update-stack: test ## Updates the nested stack on AWS
+.PHONY: updated-dev-stack
+update-dev-stack: test ## Updates the nested stack on AWS
 	@export AWS_REGION=$(AWS_REGION)
-	aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI)mozdef-parent.yml \
+	aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
 	  --capabilities CAPABILITY_IAM \
-	  --parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_STACK_URI) \
+	  --parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_DEV_STACK_URI) \
 	               $(OIDC_CLIENT_SECRET_PARAM_ARG) \
+	               $(DEV_STACK_PARAMS) \
 	  --output text
 
-# --ignore-checks=E2502 : https://github.com/awslabs/cfn-python-lint/issues/408
-.PHONY: cflint test
-test: cflint
-cflint: ## Verify the CloudFormation template pass linting tests
-	-cfn-lint --ignore-checks=E2502 cloudformation/*.yml
+.PHONY: cfn-lint test
+test: cfn-lint
+cfn-lint: ## Verify the CloudFormation template pass linting tests
+	-cfn-lint cloudformation/*.yml
 
 .PHONY: stack-status
 stack-status: ## Output current CloudFormation stack status
 	@export AWS_REGION=$(AWS_REGION)
 	watch -g aws cloudformation describe-stacks --stack-name $(STACK_NAME)
 
-.PHONY: upload-templates
-upload-templates:
+.PHONY: publish-dev-templates
+publish-dev-templates:
 	@export AWS_REGION=$(AWS_REGION)
-	aws s3 sync cloudformation/ $(S3_BUCKET_URI) --acl public-read --exclude="*" --include="*.yml"
+	aws s3 sync cloudformation/ $(S3_DEV_BUCKET_URI) --acl public-read --exclude="*" --include="*.yml"
 
-.PHONY: publish-templates
-publish-templates:
+.PHONY: publish-prod-templates
+publish-prod-templates:
 	@export AWS_REGION=$(AWS_REGION)
-	aws s3 sync cloudformation/ $(S3_PUBLISHED_BUCKET_URI) --exclude="*" --include="*.yml"
+	aws s3 sync cloudformation/ $(S3_PROD_BUCKET_URI) --exclude="*" --include="*.yml"
+
+.PHONY: diff-dev-templates
+diff-dev-templates:
+	tempdir=`mktemp --directory`; aws s3 sync $(S3_DEV_BUCKET_URI) "$$tempdir" --exclude="*" --include="*.yml"; diff --recursive --unified "$$tempdir" cloudformation; rm -rf "$$tempdir"
+
+.PHONY: diff-prod-templates
+diff-prod-templates:
+	tempdir=`mktemp --directory`; aws s3 sync $(S3_PROD_BUCKET_URI) "$$tempdir" --exclude="*" --include="*.yml"; diff --recursive --unified "$$tempdir" cloudformation; rm -rf "$$tempdir"

cloudy_mozdef/cloudformation/mozdef-instance.yml

@@ -313,9 +313,14 @@ Resources:
     Type : AWS::ElasticLoadBalancingV2::Listener
     Properties:
       DefaultActions:
-      - Type: forward
-        TargetGroupArn:
-          Ref: MozDefElasticLoadBalancingV2TargetGroup
+      - Type: redirect
+        RedirectConfig:
+          Host: '#{host}'
+          Path: '/#{path}'
+          Protocol: HTTPS
+          Query: '#{query}'
+          Port: '443'
+          StatusCode: HTTP_301
       LoadBalancerArn:
         Ref: MozDefElasticLoadBalancingV2LoadBalancer
       Port: 80

cloudy_mozdef/cloudformation/mozdef-security-group.yml

@@ -46,6 +46,11 @@ Resources:
           FromPort: 443
           ToPort: 443
           CidrIp: 0.0.0.0/0
+        - Description: Allow 80 inbound from everywhere for redirection
+          IpProtocol: tcp
+          FromPort: 80
+          ToPort: 80
+          CidrIp: 0.0.0.0/0
         - Description: Allow 9090 inbound from everywhere
           IpProtocol: tcp
           FromPort: 9090

docs/source/cloud_deployment.rst

@@ -4,7 +4,7 @@ MozDef for AWS
 **What is MozDef for AWS**
 
 Cloud based MozDef is an opinionated deployment of the MozDef services created in 2018 to help AWS users
-ingest cloudtrail, guardduty, and provide security services.
+ingest CloudTrail, GuardDuty, and provide security services.
 
 .. image:: images/cloudformation-launch-stack.png
    :target: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/public.us-west-2.infosec.mozilla.org/mozdef/cf/mozdef-parent.yml
@@ -27,19 +27,30 @@ Dependencies
 
 MozDef requires the following:
 
-- A DNS name ( cloudymozdef.security.allizom.org )
+- A DNS name ( e.g. cloudymozdef.security.allizom.org ) which you will need to point
+  at the IP address of the Application Load Balancer
 - An OIDC Provider with ClientID, ClientSecret, and Discovery URL
-  - Mozilla Uses Auth0 but you can use any OIDC provider you like: Shibboleth, KeyCloak, AWS Cognito, Okta, Ping (etc)
+
+  - Mozilla Uses Auth0 but you can use any OIDC provider you like: Shibboleth,
+    KeyCloak, AWS Cognito, Okta, Ping (etc)
+  - You will need to configure the redirect URI of `/redirect_uri` as allowed in
+    your OIDC provider
 - An ACM Certificate in the deployment region for your DNS name
 - A VPC with three public subnets available.
+
   - It is advised that this VPC be dedicated to MozDef or used solely for security automation.
-- An SQS queue recieving GuardDuty events.  At the time of writing this is not required but may be required in future.
+  - The three public subnets must all be in different `availability zones <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#using-regions-availability-zones-describe>`_
+    and have a large enough number of IP addresses to accommodate the infrastructure
+  - The VPC must have an `internet gateway <https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html>`_
+    enabled on it so that MozDef can reach the internet
+- An SQS queue receiving GuardDuty events.  At the time of writing this is not required but may be required in future.
 
 
 Supported Regions
 ------------------
 
-MozDef for AWS is currently only supported in us-west-2 but will onboard additional regions over time.
+MozDef for AWS is currently only supported in us-west-2 but will onboard
+additional regions over time.
 
 
 Architecture
@@ -52,9 +63,14 @@ Deployment Process
 -------------------
 
 1. Launch the one click stack and provide the requisite values.
-2. Wait for the stack to complete.  You'll see several nested stacks in the Cloudformation console. *Note: This may take a while*
-3. Navigate to the URL you set up for MozDef.  It should redirect you to the single sign on provider.  If successful you'll see the MozDef UI.
-4. Try navigating to ElasticSearch https://your_base_url:9090
+2. Wait for the stack to complete.  You'll see several nested stacks in the
+   CloudFormation console. Once the EC2 instance is running there are still
+   provisioning steps taking place on the instance. *Note: This may take a while*
+3. Configure your DNS name to point to the application load balancer
+4. Navigate to the URL you set up for MozDef.  It should redirect you to the
+   single sign on provider.  If successful you'll see the MozDef UI.
+5. Try navigating to ElasticSearch https://your_base_url:9090
+
 You should see the following:
 ::
 
@@ -74,6 +90,21 @@ You should see the following:
 
 5. Test out Kibana at https://your_base_url:9090/_plugin/kibana/app/kibana#/discover?_g=()
 
+Troubleshooting
+---------------
+
+To view logs on the ec2 instance
+
+1. Determine the name/IP of the autoscaled EC2 instance via the command line or web console
+2. SSH into that EC2 instance as the `ec2-user` user using the SSH keypair that you
+   set as the `KeyName` parameter in CloudFormation
+3. List out all the containers with `sudo docker container ls`
+4. Tail logs from the container you'd like to examine with `sudo docker logs --follow NAME_OF_CONTAINER`
+   where `NAME_OF_CONTAINER` is the container name or ID that you found in the
+   step above
+5. To enter the environment for that container run `sudo docker exec --interactive --tty NAME_OF_CONTAINER /bin/bash`
+6. To view the environment variables being made available to the containers view
+   the file `/opt/mozdef/docker/compose/cloudy_mozdef.env`
 
 Using MozDef
 -------------