From a7f5ddfb61453e86d254722309af080c53727e58 Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Wed, 17 Oct 2018 14:37:47 -0700 Subject: [PATCH 1/7] typo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e2b8d6a9..8271a68f 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ run-only: .PHONY: run-cloudy-mozdef run-cloudy-mozdef: ## Run the MozDef containers necessary to run in AWS (`cloudy-mozdef`). This is used by the CloudFormation-initiated setup. - docker-compose -f -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) up -d + docker-compose -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) up -d # TODO? add custom test targets for individual tests (what used to be `multiple-tests` for example # The docker files are still in docker/compose/docker*test* From dd4b0ee4a019144f43b1ed6fa35db900f553b0cb Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Wed, 17 Oct 2018 14:38:56 -0700 Subject: [PATCH 2/7] prepare to use the hub images for make --- cloudy_mozdef/cloudformation/mozdef-instance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index e77b8583..bbc9b5db 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -100,7 +100,7 @@ Resources: path: /opt/mozdef/docker/compose/cloudy_mozdef.env runcmd: - chmod 600 /opt/mozdef/docker/compose/cloudy_mozdef.env - - make -C /opt/mozdef -f /opt/mozdef/Makefile # run-cloudy-mozdef + - make -C /opt/mozdef -f /opt/mozdef/Makefile # USE_DKR_IMAGES=docker/compose/docker-compose-norebuild.yml run-cloudy-mozdef MozDefAutoScaleGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: From 52d7446449957dbd06de982613caa960a56e1927 Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Wed, 17 Oct 2018 14:48:18 -0700 Subject: [PATCH 3/7] add docker-compose to the deps --- cloudy_mozdef/packer/packer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudy_mozdef/packer/packer.json b/cloudy_mozdef/packer/packer.json index 2f7d75ad..cdd90e7a 100644 --- a/cloudy_mozdef/packer/packer.json +++ b/cloudy_mozdef/packer/packer.json @@ -26,6 +26,7 @@ "sudo yum install -y git", "sudo yum install -y docker", "sudo pip install virtualenv ", + "sudo pip install docker-compose", "sudo systemctl enable docker", "sudo mkdir -p /opt/mozdef/", "sudo git clone https://github.com/mozilla/MozDef /opt/mozdef", From e2b32db60eab2a813192402ee1b89555dbf79443 Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Wed, 17 Oct 2018 14:57:22 -0700 Subject: [PATCH 4/7] update the repo before build (mainly for sprint iterations) --- cloudy_mozdef/cloudformation/mozdef-instance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index bbc9b5db..6da356a1 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -100,6 +100,7 @@ Resources: path: /opt/mozdef/docker/compose/cloudy_mozdef.env runcmd: - chmod 600 /opt/mozdef/docker/compose/cloudy_mozdef.env + - cd /opt/mozdef && git pull infosec_workweek - make -C /opt/mozdef -f /opt/mozdef/Makefile # USE_DKR_IMAGES=docker/compose/docker-compose-norebuild.yml run-cloudy-mozdef MozDefAutoScaleGroup: Type: AWS::AutoScaling::AutoScalingGroup From 5b71c82ea4179432c050c062f69d284d3cea7548 Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Wed, 17 Oct 2018 15:39:49 -0700 Subject: [PATCH 5/7] only use images for cloudy mozdef --- .../compose/docker-compose-cloudy-mozdef.yml | 44 +++++-------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/docker/compose/docker-compose-cloudy-mozdef.yml b/docker/compose/docker-compose-cloudy-mozdef.yml index 3f4c8040..a62e0b8e 100644 --- a/docker/compose/docker-compose-cloudy-mozdef.yml +++ b/docker/compose/docker-compose-cloudy-mozdef.yml @@ -2,9 +2,7 @@ version: '2.2' services: nginx: - build: - context: ../../ - dockerfile: docker/compose/nginx/Dockerfile + image: mozdef/mozdef_nginx restart: always command: /usr/sbin/nginx depends_on: @@ -17,9 +15,7 @@ services: networks: - default mongodb: - build: - context: ../../ - dockerfile: docker/compose/mongodb/Dockerfile + image: mozdef/mozdef_mongodb restart: always command: /usr/bin/mongod --smallfiles --config /etc/mongod.conf volumes: @@ -27,9 +23,7 @@ services: networks: - default bootstrap: - build: - context: ../../ - dockerfile: docker/compose/mozdef_bootstrap/Dockerfile + image: mozdef/mozdef_bootstrap env_file: - cloudy_mozdef.env command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python docker/conf/initial_setup.py ${ES_URL} cron/defaultMappingTemplate.json cron/backup.conf' @@ -39,18 +33,14 @@ services: - default # MozDef Specific Containers base: - build: - context: ../../ - dockerfile: docker/compose/mozdef_base/Dockerfile + image: mozdef/mozdef_base env_file: - cloudy_mozdef.env command: bash -c 'su - mozdef -c /opt/mozdef/envs/mozdef/cron/update_geolite_db.sh' volumes: - geolite_db:/opt/mozdef/envs/mozdef/data alertplugins: - build: - context: ../../ - dockerfile: docker/compose/mozdef_alertplugins/Dockerfile + image: mozdef/mozdef_alertplugins env_file: - cloudy_mozdef.env restart: always @@ -62,9 +52,7 @@ services: networks: - default alerts: - build: - context: ../../ - dockerfile: docker/compose/mozdef_alerts/Dockerfile + image: mozdef/mozdef_alerts env_file: - cloudy_mozdef.env restart: always @@ -75,9 +63,7 @@ services: networks: - default cron: - build: - context: ../../ - dockerfile: docker/compose/mozdef_cron/Dockerfile + image: mozdef/mozdef_cron env_file: - cloudy_mozdef.env restart: always @@ -91,9 +77,7 @@ services: networks: - default loginput: - build: - context: ../../ - dockerfile: docker/compose/mozdef_loginput/Dockerfile + image: mozdef/mozdef_loginput env_file: - cloudy_mozdef.env restart: always @@ -104,9 +88,7 @@ services: networks: - default meteor: - build: - context: ../../ - dockerfile: docker/compose/mozdef_meteor/Dockerfile + image: mozdef/mozdef_meteor restart: always command: bash -c 'node bundle/main.js' depends_on: @@ -116,9 +98,7 @@ services: networks: - default rest: - build: - context: ../../ - dockerfile: docker/compose/mozdef_rest/Dockerfile + image: mozdef/mozdef_rest env_file: - cloudy_mozdef.env restart: always @@ -130,9 +110,7 @@ services: networks: - default syslog: - build: - context: ../../ - dockerfile: docker/compose/mozdef_syslog/Dockerfile + image: mozdef/mozdef_syslog env_file: - cloudy_mozdef.env restart: always From dbedba303956b097cb65d2f7d9362a8b56f52310 Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Wed, 17 Oct 2018 15:40:43 -0700 Subject: [PATCH 6/7] do not use the norebuild system for cloudymozdef --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8271a68f..5c53ea80 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ run-only: .PHONY: run-cloudy-mozdef run-cloudy-mozdef: ## Run the MozDef containers necessary to run in AWS (`cloudy-mozdef`). This is used by the CloudFormation-initiated setup. - docker-compose -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) up -d + docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) up -d # TODO? add custom test targets for individual tests (what used to be `multiple-tests` for example # The docker files are still in docker/compose/docker*test* From 750531b5d5122d896a9126ef01da80f56377a198 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Wed, 17 Oct 2018 15:42:59 -0700 Subject: [PATCH 7/7] bump configlib over to 2.0.1 from andrewkrug --- docker/compose/mozdef_base/Dockerfile | 3 ++- requirements.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/compose/mozdef_base/Dockerfile b/docker/compose/mozdef_base/Dockerfile index f14d7919..670af273 100644 --- a/docker/compose/mozdef_base/Dockerfile +++ b/docker/compose/mozdef_base/Dockerfile @@ -39,7 +39,8 @@ USER mozdef RUN \ virtualenv /opt/mozdef/envs/python && \ source /opt/mozdef/envs/python/bin/activate && \ - pip install -r /opt/mozdef/envs/mozdef/requirements.txt + pip install -r /opt/mozdef/envs/mozdef/requirements.txt && \ + pip install git+https://github.com/andrewkrug/configlib # Local copy for dev COPY mozdef_util /opt/mozdef/envs/mozdef/mozdef_util diff --git a/requirements.txt b/requirements.txt index 526090e9..173f7753 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,4 +58,3 @@ uritemplate==0.6 urllib3==1.22 uwsgi==2.0.17.1 virtualenv==1.11.4 -configlib==2.0.0