Merge branch 'infosec_workweek' of https://github.com/mozilla/mozdef into config_system

2018-10-19 13:08:46 -07:00 · 2018-10-19 13:08:46 -07:00 · 11fd235672
--- a/5
+++ b/5
@ -26,13 +26,16 @@ run: build ## Run all MozDef containers
 run-only:
 	docker-compose -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose.yml -p $(NAME) up -d

-.PHONY: run-cloudy-mozdef
+.PHONY: run-cloudy-mozdef restart-cloudy-mozdef
 run-cloudy-mozdef: ## Run the MozDef containers necessary to run in AWS (`cloudy-mozdef`). This is used by the CloudFormation-initiated setup.
 	$(shell test -f docker/compose/cloudy_mozdef.env || touch docker/compose/cloudy_mozdef.env)
 	$(shell test -f docker/compose/cloudy_mozdef_kibana.env || touch docker/compose/cloudy_mozdef_kibana.env)
 	docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) pull
 	docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) up -d

+restart-cloudy-mozdef:
+	docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) restart
+
 # TODO? add custom test targets for individual tests (what used to be `multiple-tests` for example
 # The docker files are still in docker/compose/docker*test*
 .PHONY: test tests run-tests
--- a/cloudy_mozdef/Makefile
+++ b/cloudy_mozdef/Makefile
@ -1,51 +1,56 @@
 ROOT_DIR	:= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 PARENTDIR       := $(realpath ../)
-IMAGE_NAME	:= mozdef-deployment
-S3_INFOSEC_URI	:= s3://mozdef.infosec.mozilla.org/cf
-S3_STACK_URI	:= https://s3-us-west-2.amazonaws.com/cf/nested-stack.yml
-DOCKER_BASH_RUN	:= docker run -v ~/.aws:/root/.aws -v `pwd`:/opt/mozdef $(IMAGE_NAME):latest /bin/bash -c
+AWS_REGION	:= us-west-2
+STACK_NAME	:= mozdef-aws-nested
+STACK_PARAMS	:= file://aws_parameters.json
+# MozDef uses a nested CF stack, the mozdef-parent.yml will tie all child stacks together and load them from S3
+# See also mozdef.infosec.mozilla.org bucket
+S3_BUCKET_NAME  := mozdef.infosec.allizom.org
+S3_BUCKET_PATH	:= cf
+S3_BUCKET_URI	:= s3://$(S3_BUCKET_NAME)/$(S3_BUCKET_PATH)
+S3_STACK_URI	:= https://s3-$(AWS_REGION).amazonaws.com/$(S3_BUCKET_NAME)/$(S3_BUCKET_PATH)/mozdef-parent.yml

 all:
 	@echo 'Available make targets:'
 	@grep '^[^#[:space:]\.PHONY.*].*:' Makefile
-
-.PHONY: build docker-build
-build: docker-build packer-build
-docker-build: ## Build the docker image that is used for deployment of CloudFormation templates
-	docker build -t $(IMAGE_NAME):latest .
-
-.PHONY: docker-shell
-deploy-shell: ## Spawn a shell for hacking into the docker image
-	docker run -ti -v ~/.aws:/root/.aws -v `pwd`:/opt/mozdef -v $(PARENTDIR):/opt/gitrepo $(IMAGE_NAME):latest /bin/bash
+	@echo 'Run ./dmake <target> in order to run the Makefile targets in Docker'

 # Note: This requires AWS access
 .PHONY: packer-build
-packer-build: docker-build ## Build the base AMI with packer
-	$(DOCKER_BASH_RUN) "cd packer && packer build packer.json"
+packer-build: ## Build the base AMI with packer
+	cd packer && packer build packer.json

-.PHONY: deploy-nested-cloudformation
-deploy-nested-cloudformation: cflint ## Deploy our nested ClouFormation stack
-	$(DOCKER_BASH_RUN) "ansible-playbook -c local ansible/update-ami-metadata.yml"
-	$(DOCKER_BASH_RUN) "aws s3 sync /opt/mozdef/ansible/files/stacks/ $(S3_INFOSEC_URI) --acl public-read"
+.PHONY: create-stack
+create-stack: test ## Create everything you need for a fresh new stack!
+	@export AWS_REGION=$(AWS_REGION)
+	@echo "Make sure you have a param file ($(STACK_PARAMS)) with OIDCClientSecret set."
+	aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI) \
+	  --capabilities CAPABILITY_IAM \
+	  --parameters $(STACK_PARAMS)

-.PHONY: test
-test: cflint test-nested-stack
+.PHONY: create-s3-bucket
+create-s3-bucket:
+	@export AWS_REGION=$(AWS_REGION)
+	aws s3api create-bucket --bucket $(S3_BUCKET_NAME) --acl public-read --create-bucket-configuration LocationConstraint=$(AWS_REGION)

-.PHONY: test-nested-stack cflint
-test-nested-stack:
-	$(DOCKER_BASH_RUN) "ansible-playbook -c local ansible/update-ami-metadata.yml"
-	$(DOCKER_BASH_RUN) "aws s3 sync /opt/mozdef/ansible/files/stacks/ $(S3_INFOSEC_URI) --acl public-read"
-	$(DOCKER_BASH_RUN) "aws s3 cp /opt/mozdef/cloudformation/nested-stack.yml $(S3_INFOSEC_URI)/nested-stack.yml --acl public-read"
-	$(DOCKER_BASH_RUN) "aws cloudformation update-stack --stack-name mozdef-nested --template-url $(S3_STACK_URI)"
+.PHONY: updated-nested-stack
+update-stack: test ## Updates the nested stack on AWS
+	@export AWS_REGION=$(AWS_REGION)
+	aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI) \
+	  --capabilities CAPABILITY_IAM \
+	  --parameters $(STACK_PARAMS)

-.PHONY: cflint
+.PHONY: cflint test
+test: cflint
 cflint: ## Verify the CloudFormation template pass linting tests
-	$(DOCKER_BASH_RUN) "cfn-lint cloudformation/*.yml"
+	-cfn-lint /opt/mozdef/cloudformation/*.yml

 .PHONY: stack-status
 stack-status: ## Output current CloudFormation stack status
-	$(DOCKER_BASH_RUN) "aws cloudformation describe-stacks --stack-name mozdef-nested"
+	@export AWS_REGION=$(AWS_REGION)
+	watch -g aws cloudformation describe-stacks --stack-name $(STACK_NAME)

 .PHONY: upload-templates
 upload-templates:
-	aws s3 sync cloudformation/ $(S3_INFOSEC_URI) --acl public-read
+	@export AWS_REGION=$(AWS_REGION)
+	aws s3 sync cloudformation/ $(S3_BUCKET_URI) --acl public-read
--- a/cloudy_mozdef/cloudformation/mozdef-instance.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml
@ -71,6 +71,9 @@ Resources:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      HealthCheckPath: "/health"
+      HealthCheckIntervalSeconds: 5
+      HealthCheckTimeoutSeconds: 2
+      HealthyThresholdCount: 2
      Port: 80
      Protocol: HTTP
      Tags:
@ -124,6 +127,7 @@ Resources:
                OPTIONS_METEOR_PORT=3000
                OPTIONS_METEOR_AUTHENTICATIONTYPE=oidc
                ES={"servers": [${ESURL}]}
+                cookiename=sesmeteor
              path: /opt/mozdef/docker/compose/cloudy_mozdef.env
            - content: |
                client_id=${OIDCClientId}
@ -132,6 +136,7 @@ Resources:
                backend=${KibanaDomainOnlyURL}
                redirect_uri_path=/redirect_uri
                httpsredir=no
+                cookiename=seskibana
              path: /opt/mozdef/docker/compose/cloudy_mozdef_kibana.env
          runcmd:
            - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef.env
--- a/cloudy_mozdef/cloudformation/mozdef-mq.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-mq.yml
@ -112,8 +112,8 @@ Resources:
          def handler(event, context):
            alphabet = string.ascii_letters + string.digits
            password = ''.join(secrets.choice(alphabet) for i in range(int(event['ResourceProperties']['Length'])))
-            response_data = {'Password': password}
-            cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data, "CustomResourcePhysicalID")
+            physical_id = ''.join(secrets.choice(string.ascii_uppercase + string.digits) for i in range(13))
+            cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Password': password}, "DefaultPasswordGenerator-%s" % physical_id)
      Handler: index.handler
      Runtime: python3.6
      Role: !GetAtt CloudFormationLambdaIAMRole.Arn
@ -129,13 +129,14 @@ Resources:
      Code:
        ZipFile: |
          import cfnresponse
-          import boto3
+          import boto3, secrets, string
          from urllib.parse import urlparse
          def handler(event, context):
            response = boto3.client('mq').describe_broker(BrokerId=event['ResourceProperties']['BrokerID'])
            url = urlparse(next(x for x in response['BrokerInstances'][0]['Endpoints'] if x.startswith('amqp+ssl://')))
            response = {'URL': url.geturl(), 'HostName': url.hostname, 'Scheme': url.scheme, 'Port': url.port}
-            cfnresponse.send(event, context, cfnresponse.SUCCESS, response, "CustomResourcePhysicalID")
+            physical_id = ''.join(secrets.choice(string.ascii_uppercase + string.digits) for i in range(13))
+            cfnresponse.send(event, context, cfnresponse.SUCCESS, response, "MQBrokerURLLookup-%s" % physical_id)
      Handler: index.handler
      Runtime: python3.6
      Role: !GetAtt CloudFormationLambdaIAMRole.Arn
--- a/cloudy_mozdef/cloudformation/mozdef-parent.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-parent.yml
@ -156,9 +156,11 @@ Resources:
      Code:
        ZipFile: |
          import cfnresponse
+          import secrets, string
          def handler(event, context):
            length = len(event['ResourceProperties']['Array'])
-            cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Length': length}, "CustomResourcePhysicalID")
+            physical_id = ''.join(secrets.choice(string.ascii_uppercase + string.digits) for i in range(13))
+            cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Length': length}, "GetArrayLength-%s" % physical_id)
      Handler: index.handler
      Runtime: python3.6
      Role: !GetAtt CloudFormationLambdaIAMRole.Arn
--- a/cloudy_mozdef/dmake
+++ b/cloudy_mozdef/dmake
@ -0,0 +1,40 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# @gdestuynder
+
+# Use this script to run the makefile within a docker container
+
+AWS_CREDS_DIR="$HOME/.aws"
+DOCKER_PROJECT_DIR="/opt/mozdef"
+IMG_NAME="mozdef_builder"
+HUB="mozdef"
+CONTAINER_NAME="$IMG_NAME-container"
+
+function usage() {
+  echo "Build make targets in a container (${IMG_NAME})"
+  echo "$0 make <make target>"
+  exit 127
+}
+
+function check_img() {
+  docker image ls ${IMG_NAME} 2>&1 > /dev/null && return 0
+  echo "Cannot find docker image ${IMG_NAME}."
+  echo "Please run \`make dkrbuild\` to build it, or \`docker pull ${HUB}/${IMG_NAME}\`".
+  return 1
+}
+
+[[ $# -eq 0 ]] && usage
+
+check_img || exit 127
+
+exec docker run --rm --name ${CONTAINER_NAME} \
+  -u $(id -u) \
+  -v ${AWS_CREDS_DIR}:/root/.aws \
+  -v $(pwd):${DOCKER_PROJECT_DIR} \
+  -e "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" \
+  -e "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}" \
+  -e "AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}" \
+  -e "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" \
+  ${HUB}/${IMG_NAME}:latest make $@
--- a/cloudy_mozdef/requirements.txt
+++ b/cloudy_mozdef/requirements.txt
@ -1,6 +0,0 @@
-ansible
-credstash
-faker
-awscli
-awsudo
-cfn-lint
--- a/cron/healthToMongo.py
+++ b/cron/healthToMongo.py
@ -106,8 +106,11 @@ def getEsNodesStats():

        load_average = jsonobj['nodes'][nodeid]['os']['cpu']['load_average']
        load_str = "{0},{1},{2}".format(load_average['1m'], load_average['5m'], load_average['15m'])
+        hostname = nodeid
+        if 'host' in jsonobj['nodes'][nodeid]:
+            hostname=jsonobj['nodes'][nodeid]['host']
        results.append({
-            'hostname': jsonobj['nodes'][nodeid]['host'],
+            'hostname': hostname,
            'disk_free': jsonobj['nodes'][nodeid]['fs']['total']['free_in_bytes'] / (1024 * 1024 * 1024),
            'disk_total': jsonobj['nodes'][nodeid]['fs']['total']['total_in_bytes'] / (1024 * 1024 * 1024),
            'mem_heap_per': jsonobj['nodes'][nodeid]['jvm']['mem']['heap_used_percent'],
--- a/docker/builder/Dockerfile
+++ b/docker/builder/Dockerfile
@ -1,6 +1,7 @@
 FROM amazonlinux:2

 # Base dependencies
+RUN yum update -y
 RUN yum install @development wget -y
 RUN yum install python python-dev python-pip -y
 ADD requirements.txt /tmp/
@ -19,6 +20,3 @@ RUN mkdir -p /opt/mozdef

 # Force this as the entrypoint
 WORKDIR /opt/mozdef
-
-# Add the deploytools for dev previews to the home
-ADD . /opt/mozdef/
--- a/docker/builder/Makefile
+++ b/docker/builder/Makefile
@ -0,0 +1,5 @@
+all:
+	docker build -t mozdef_builder:latest .
+	docker tag mozdef_builder mozdef/mozdef_builder:latest
+	docker login
+	docker push mozdef/mozdef_builder:latest
--- a/docker/builder/requirements.txt
+++ b/docker/builder/requirements.txt
@ -0,0 +1,4 @@
+awscli
+awsudo
+cfn-lint
+docker-compose
--- a/docker/compose/docker-compose-cloudy-mozdef.yml
+++ b/docker/compose/docker-compose-cloudy-mozdef.yml
@ -33,7 +33,7 @@ services:
    restart: always
    command: /usr/bin/mongod --smallfiles --config /etc/mongod.conf
    volumes:
-      - mongodb:/var/lib/mongo
+      - /var/lib/mongodb:/var/lib/mongo
    networks:
      - default
  bootstrap:
@ -170,7 +170,6 @@ services:
    volumes:
      - geolite_db:/opt/mozdef/envs/mozdef/data/
 volumes:
-  mongodb:
  cron:
  geolite_db:
  rabbitmq:
--- a/meteor/client/layout.js
+++ b/meteor/client/layout.js
@ -49,4 +49,40 @@ if (Meteor.isClient) {
            $('ul:first',$(e.target)).css('visibility', 'visible');
        }
    });
+
+    Template.layout.rendered=function(){
+    // Intercepts all XHRs and reload the main browser window on redirect or request error (such as CORS denying access)
+        // This is because, if you run MozDef behind an access-proxy, the requests maybe 302'd to an authentication
+        // provider, but Meteor does not know or handle this. Reloading the main browser window will send the user to the
+        // authentication provider correctly and follow the 302.
+        // Note that since they're 302's they will ALWAYS cause a CORS error, which we keep as this is the SAFE way to
+        // handle this situation.
+        (function(xhr) {
+            var authenticationType = getSetting('authenticationType').toLowerCase();
+            function intercept_xhr(xhrInstance) {
+                // Verify a user is actually logged in and Meteor is running
+                if ((Meteor.user() !== null) && (Meteor.status().connected)) {
+                    // Status 0 means the request failed (CORS denies access)
+                    if (xhrInstance.readyState == 4 && (xhrInstance.status == 302 || xhrInstance.status == 0)) {
+                            location.reload();
+                    }
+                }
+            }
+            var send = xhr.send;
+            xhr.send = function(data) {
+                var origFunc = this.onreadystatechange;
+                if (origFunc) {
+                    this.onreadystatechange = function() {
+                        // We only start hooking for oidc authentication, as this is the only method that is currently
+                        // REQUIRING an access proxy and thus likely to run into 302s
+                        if (authenticationType == 'oidc'){
+                            intercept_xhr(this);
+                        }
+                        return origFunc.apply(this, arguments);
+                    };
+                }
+                return send.apply(this, arguments);
+            };
+        })(XMLHttpRequest.prototype);
+    }
 }
--- a/meteor/client/main.js
+++ b/meteor/client/main.js
@ -5,8 +5,8 @@ import { Mongo } from 'meteor/mongo';
 import { Session } from 'meteor/session';
 import { _ } from 'meteor/underscore';
 import { Blaze } from 'meteor/blaze';
-import '/imports/collections.js';
 import '/imports/settings.js';
+import '/imports/collections.js';
 import '/imports/helpers.js';
 import '/imports/models.js';
 import '/client/about.html';
--- a/meteor/client/mozdef.js
+++ b/meteor/client/mozdef.js
@ -7,6 +7,7 @@ Copyright (c) 2014 Mozilla Corporation
 import { Meteor } from 'meteor/meteor'
 import { Template } from 'meteor/templating';
 import validator from 'validator';
+import '/imports/collections.js';
 import '/imports/settings.js';
 import '/imports/helpers.js';
 import '/client/js/jquery.highlight.js';
@ -430,37 +431,5 @@ if (Meteor.isClient) {
    Meteor.logoutViaAccounts = function(callback) {
        return Accounts.logout(callback);
    };
-    // Intercepts all XHRs and reload the main browser window on redirect or request error (such as CORS denying access)
-    // This is because, if you run MozDef behind an access-proxy, the requests maybe 302'd to an authentication
-    // provider, but Meteor does not know or handle this. Reloading the main browser window will send the user to the
-    // authentication provider correctly and follow the 302.
-    // Note that since they're 302's they will ALWAYS cause a CORS error, which we keep as this is the SAFE way to
-    // handle this situation.
-    (function(xhr) {
-        var authenticationType = getSetting('authenticationType').toLowerCase();
-        function intercept_xhr(xhrInstance) {
-            // Verify a user is actually logged in and Meteor is running
-            if ((Meteor.user() !== null) && (Meteor.status().connected)) {
-                // Status 0 means the request failed (CORS denies access)
-                if (xhrInstance.readyState == 4 && (xhrInstance.status == 302 || xhrInstance.status == 0)) {
-                        location.reload();
-                }
-            }
-        }
-        var send = xhr.send;
-        xhr.send = function(data) {
-            var origFunc = this.onreadystatechange;
-            if (origFunc) {
-                this.onreadystatechange = function() {
-                    // We only start hooking for oidc authentication, as this is the only method that is currently
-                    // REQUIRING an access proxy and thus likely to run into 302s
-                    if (authenticationType == 'oidc'){
-                        intercept_xhr(this);
-                    }
-                    return origFunc.apply(this, arguments);
-                };
-            }
-            return send.apply(this, arguments);
-        };
-    })(XMLHttpRequest.prototype);
+
 };
--- a/meteor/client/mozdefhealth.js
+++ b/meteor/client/mozdefhealth.js
@ -5,6 +5,9 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
 Copyright (c) 2014 Mozilla Corporation
 */
 import { Template } from 'meteor/templating';
+import '/imports/collections.js';
+import '/imports/settings.js';
+import '/imports/helpers.js';
 import '/client/router.js';
 import '/client/mozdef.js';
 import crossfilter from 'crossfilter2';
--- a/meteor/imports/collections.js
+++ b/meteor/imports/collections.js
@ -9,13 +9,13 @@ import uuid from "uuid";

 //collections shared by client/server
 Meteor.startup(() => {
+    mozdefsettings = new Meteor.Collection("mozdefsettings");
    events = new Meteor.Collection("events");
    alerts = new Meteor.Collection("alerts");
    investigations = new Meteor.Collection("investigations");
    incidents = new Meteor.Collection("incidents");
    veris = new Meteor.Collection("veris");
    kibanadashboards = new Meteor.Collection("kibanadashboards");
-    mozdefsettings = new Meteor.Collection("mozdefsettings");
    healthfrontend = new Meteor.Collection("healthfrontend");
    sqsstats = new Meteor.Collection("sqsstats");
    healthescluster = new Meteor.Collection("healthescluster");
@ -376,9 +376,10 @@ Meteor.startup(() => {
        options={
            _suppressSameNameError : true
        };
+        Meteor.subscribe("mozdefsettings");
        alertsCount = new Meteor.Collection("alerts-count",options);
        //client-side subscriptions to low volume collections
-        Meteor.subscribe("mozdefsettings");
+
        Meteor.subscribe("veris");
        Meteor.subscribe("kibanadashboards");
        Meteor.subscribe("userActivity");
--- a/meteor/imports/helpers.js
+++ b/meteor/imports/helpers.js
@ -5,13 +5,20 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
 Copyright (c) 2014 Mozilla Corporation

 */
+import { _ } from 'meteor/underscore';

 // helper functions
 getSetting=function (settingKey){
-	  //returns the value given a setting key
-	  //makes server-side settings easier to 
-	  //deploy than normal meteor --settings
-	  var settingvalue = mozdefsettings.findOne({ key : settingKey }).value;
-	  return settingvalue;
+
+    //prefer Meteor.settings.public
+    if ( _.has(Meteor.settings.public,settingKey) ){
+        return Meteor.settings.public.settingKey;
+    }else{
+        if ( mozdefsettings.findOne({ key : settingKey }) ){
+            return mozdefsettings.findOne({ key : settingKey }).value;
+        }else{
+            return '';
+        }
+    }
 	};

--- a/meteor/imports/settings.js
+++ b/meteor/imports/settings.js
@ -16,6 +16,8 @@ if (Meteor.isServer) {
    enableClientAccountCreation: process.env.OPTIONS_METEOR_ENABLECLIENTACCOUNTCREATION || true,
    authenticationType: process.env.OPTIONS_METEOR_AUTHENTICATIONTYPE || "meteor-password"
    }
+
+    Meteor.settings.public=mozdef;
 }

 if (Meteor.isClient) {
--- a/meteor/server/main.js
+++ b/meteor/server/main.js
@ -1,8 +1,9 @@
 import { Meteor } from 'meteor/meteor';
 import '/imports/settings.js';
 import '/imports/collections.js'
-import './methods.js';
 import './mozdef.js';
+import './methods.js';
+

 Meteor.startup(() => {
  // placeholder for any code to run on server at startup
--- a/meteor/server/mozdef.js
+++ b/meteor/server/mozdef.js
@ -46,7 +46,7 @@ if (Meteor.isServer) {
        mozdefsettings.insert({
            key: 'authenticationType',
            value: mozdef.authenticationType
-        })
+        });

        //allow local account creation?
        //http://docs.meteor.com/#/full/accounts_config