port build system to docker compose 3.7

use dependency-based builds default to GITHASH=latest instead of the opposite rename mq_event-task to mq_worker where it wasn't fix dockerfiles to import from mozdef/* by default introduce `make BUILD_MODE=pull` to pull images (instead of build)
2018-10-30 13:01:47 -07:00 · 2018-10-30 13:01:47 -07:00 · 67c92b592e
--- a/39
+++ b/39
@ -8,23 +8,20 @@ ROOT_DIR	:= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 DKR_IMAGES	:= mozdef_alertplugins mozdef_alerts mozdef_base mozdef_bootstrap mozdef_meteor mozdef_rest \
 		   mozdef_mq_worker mozdef_loginput mozdef_cron mozdef_elasticsearch mozdef_mongodb \
 		   mozdef_syslog mozdef_nginx mozdef_tester mozdef_rabbitmq mozdef_kibana
-USE_DKR_IMAGES  := docker/compose/docker-compose-rebuild.yml  ## Pass docker/compose/docker-compose-norebuild.yml to use hub.docker.com images
+BUILD_MODE	:= build  ## Pass `pull` in order to pull images instead of building them
 NAME		:= mozdef
 VERSION		:= 0.1
 NO_CACHE	:= ## Pass `--no-cache` in order to disable Docker cache
-GITHASH		:= $(shell git rev-parse --short HEAD) ## Pass `latest` to tag docker hub images as latest instead
+GITHASH		:= latest  ## Pass `$(git rev-parse --short HEAD`) to tag docker hub images as latest git-hash instead

 .PHONY:all
 all:
 	@echo 'Available make targets:'
 	@grep '^[^#[:space:]^\.PHONY.*].*:' Makefile

-.PHONY: run run-only
+.PHONY: run
 run: build ## Run all MozDef containers
-	docker-compose -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose.yml -p $(NAME) up -d
-
-run-only:
-	docker-compose -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose.yml -p $(NAME) up -d
+	docker-compose -f docker/compose/docker-compose.yml -p $(NAME) up -d

 .PHONY: run-cloudy-mozdef restart-cloudy-mozdef
 run-cloudy-mozdef: ## Run the MozDef containers necessary to run in AWS (`cloudy-mozdef`). This is used by the CloudFormation-initiated setup.
@ -41,36 +38,33 @@ restart-cloudy-mozdef:
 .PHONY: test tests run-tests
 test: build-tests run-tests ## Running tests from locally-built images
 tests: build-tests run-tests
-
 run-tests:
-	docker-compose -f $(USE_DKR_IMAGES) -f tests/docker-compose.yml -p $(NAME) up -d
-	@echo "Waiting for the instance to come up..."
-	sleep 10
+	docker-compose -f docker/compose/docker-compose-tests.yml -p $(NAME) up -d
 	@echo "Running flake8.."
-	docker run -it mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && flake8 --config .flake8 ./"
+	docker run -it --rm mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && flake8 --config .flake8 ./"
 	@echo "Running py.test..."
-	docker run -it --network=mozdef_default mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && py.test --delete_indexes --delete_queues tests"
+	docker run -it --rm --network=mozdef_default mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && py.test --delete_indexes --delete_queues tests"
+	@echo "Shutting down test environment..."
+	docker-compose -f docker/compose/docker-compose-tests.yml -p $(NAME) stop

 .PHONY: build
 build:  ## Build local MozDef images (use make NO_CACHE=--no-cache build to disable caching)
-	docker-compose  -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose.yml -p $(NAME) $(NO_CACHE) build base
-	docker-compose  -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose.yml -p $(NAME) $(NO_CACHE) build
+	docker-compose -f docker/compose/docker-compose.yml -p $(NAME) $(NO_CACHE) $(BUILD_MODE)

-.PHONY: build-tests nobuild-tests
-build-tests:
-	docker-compose  -f $(USE_DKR_IMAGES) -f tests/docker-compose.yml -p $(NAME) $(NO_CACHE) build base
-	docker-compose  -f $(USE_DKR_IMAGES) -f tests/docker-compose.yml -p $(NAME) $(NO_CACHE) build
+.PHONY: build-tests fetch-tests
+build-tests:  ## Build end-to-end test environment only
+	docker-compose -f docker/compose/docker-compose-tests.yml -p $(NAME) $(NO_CACHE) $(BUILD_MODE)

 .PHONY: stop down
 stop: down
 down: ## Shutdown all services we started with docker-compose
-	docker-compose  -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose.yml -p $(NAME) stop
+	docker-compose -f docker/compose/docker-compose.yml -p $(NAME) stop

 .PHONY: docker-push docker-get hub hub-get
 docker-push: hub
 hub: ## Upload locally built MozDef images tagged as the current git head (hub.docker.com/mozdef).
 	docker login
-	@echo "Tagging current docker images with git HEAD shorthash..."
+	@echo "Tagging current docker images with $(GITHASH)..."
 	$(foreach var,$(DKR_IMAGES),docker tag $(var) mozdef/$(var):$(GITHASH);)
 	@echo "Uploading images to docker..."
 	$(foreach var,$(DKR_IMAGES),docker push mozdef/$(var):$(GITHASH);)
@ -81,7 +75,8 @@ hub-get: ## Download all pre-built images (hub.docker.com/mozdef)

 .PHONY: clean
 clean: ## Cleanup all docker volumes and shutdown all related services
-	-docker-compose  -f $(USE_DKR_IMAGES) -f docker/compose/docker-compose.yml -p $(NAME) down -v --remove-orphans
+	-docker-compose -f docker/compose/docker-compose.yml -p $(NAME) down -v --remove-orphans
+	-docker-compose -f docker/compose/docker-compose-tests.yml -p $(NAME) down -v --remove-orphans
 # Shorthands
 .PHONY: rebuild
 rebuild: clean build
--- a/docker/compose/docker-compose-norebuild.yml
+++ b/docker/compose/docker-compose-norebuild.yml
@ -1,35 +0,0 @@
---
-version: '2.2'
-services:
-  nginx:
-    image: mozdef/mozdef_nginx
-  kibana:
-    image: mozdef/mozdef_kibana
-  mongodb:
-    image: mozdef/mozdef_mongodb
-  elasticsearch:
-    image: mozdef/mozdef_elasticsearch
-  rabbitmq:
-    image: mozdef/mozdef_rabbitmq
-  base:
-    image: mozdef/mozdef_base
-  bootstrap:
-    image: mozdef/mozdef_bootstrap
-  alertplugins:
-    image: mozdef/mozdef_alertplugins
-  alerts:
-    image: mozdef/mozdef_alerts
-  cron:
-    image: mozdef/mozdef_cron
-  loginput:
-    image: mozdef/mozdef_loginput
-  mq_eventtask:
-    image: mozdef/mozdef_mq_worker
-  meteor:
-    image: mozdef/mozdef_meteor
-  rest:
-    image: mozdef/mozdef_rest
-  syslog:
-    image: mozdef/mozdef_syslog
-  tester:
-    image: mozdef/mozdef_tester
--- a/docker/compose/docker-compose-rebuild.yml
+++ b/docker/compose/docker-compose-rebuild.yml
@ -1,69 +0,0 @@
---
-version: '2.2'
-services:
-  base:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_base/Dockerfile
-  nginx:
-    build:
-      context: ../../
-      dockerfile: docker/compose/nginx/Dockerfile
-  kibana:
-    build:
-      context: ../../
-      dockerfile: docker/compose/kibana/Dockerfile
-  mongodb:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mongodb/Dockerfile
-  elasticsearch:
-    build:
-      context: ../../
-      dockerfile: docker/compose/elasticsearch/Dockerfile
-  rabbitmq:
-    build:
-      context: ../../
-      dockerfile: docker/compose/rabbitmq/Dockerfile
-  bootstrap:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_bootstrap/Dockerfile
-  alertplugins:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_alertplugins/Dockerfile
-  alerts:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_alerts/Dockerfile
-  cron:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_cron/Dockerfile
-  loginput:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_loginput/Dockerfile
-  # The generic worker container for all ingest processes
-  mq_worker:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_mq_worker/Dockerfile
-    restart: "no"
-  meteor:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_meteor/Dockerfile
-  rest:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_rest/Dockerfile
-  syslog:
-    build:
-      context: ../../
-      dockerfile: docker/compose/mozdef_syslog/Dockerfile
-  tester:
-    build:
-      context: ../../
-      dockerfile: docker/compose/tester/Dockerfile
--- a/docker/compose/docker-compose-tests.yml
+++ b/docker/compose/docker-compose-tests.yml
@ -1,26 +1,63 @@
 ---
-version: '3.0'
+version: '3.7'
 services:
  elasticsearch:
+    image: mozdef/mozdef_elasticsearch
    build:
      context: ../../
      dockerfile: docker/compose/elasticsearch/Dockerfile
+      cache_from:
+        - mozdef_elasticsearch:latest
+        - mozdef_base:latest
+        - mozdef/mozdef_elasticsearch
+        - mozdef/mozdef_base
+    depends_on:
+      - base
    restart: always
    command: bin/elasticsearch
-    ports:
-      - 9200:9200
+    # ports:
+    #   - 9200:9200
    networks:
      - default
  rabbitmq:
+    image: mozdef/mozdef_rabbitmq
    build:
      context: ../../
      dockerfile: docker/compose/rabbitmq/Dockerfile
+      cache_from:
+        - mozdef_rabbitmq:latest
+        - mozdef_base:latest
+        - mozdef/mozdef_rabbitmq
+        - mozdef/mozdef_base
+    depends_on:
+      - base
    restart: always
    command: rabbitmq-server
-    ports:
-      - 5672:5672
+    # ports:
+    #   - 5672:5672
+    networks:
+      - default
+  base:
+    image: mozdef/mozdef_base
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_base/Dockerfile
+      cache_from:
+        - mozdef_base:latest
+        - mozdef/mozdef_base
+  tester:
+    image: mozdef/mozdef_tester
+    build:
+      context: ../../
+      dockerfile: docker/compose/tester/Dockerfile
+      cache_from:
+        - mozdef_tester:latest
+        - mozdef_base:latest
+        - mozdef/mozdef_tester
+        - mozdef/mozdef_base
+    depends_on:
+      - base
    networks:
      - default
-
 networks:
  default:
--- a/docker/compose/docker-compose.yml
+++ b/docker/compose/docker-compose.yml
@ -1,7 +1,14 @@
 ---
-version: '2.2'
+version: '3.7'
 services:
  nginx:
+    image: mozdef/mozdef_nginx
+    build:
+      context: ../../
+      dockerfile: docker/compose/nginx/Dockerfile
+      cache_from:
+        - mozdef/mozdef_nginx
+        - mozdef_nginx:latest
    restart: always
    command: /usr/sbin/nginx
    depends_on:
@ -15,6 +22,13 @@ services:
    networks:
      - default
  mongodb:
+    image: mozdef/mozdef_mongodb
+    build:
+      context: ../../
+      dockerfile: docker/compose/mongodb/Dockerfile
+      cache_from:
+        - mozdef/mozdef_mongodb
+        - mozdef_mongodb:latest
    restart: always
    command: /usr/bin/mongod --smallfiles --config /etc/mongod.conf
    volumes:
@ -22,6 +36,13 @@ services:
    networks:
      - default
  kibana:
+    image: mozdef/mozdef_kibana
+    build:
+      context: ../../
+      dockerfile: docker/compose/kibana/Dockerfile
+      cache_from:
+        - mozdef/mozdef_kibana
+        - mozdef_kibana:latest
    restart: always
    command: bin/kibana --elasticsearch=http://elasticsearch:9200
    depends_on:
@ -29,6 +50,13 @@ services:
    networks:
      - default
  elasticsearch:
+    image: mozdef/mozdef_elasticsearch
+    build:
+      context: ../../
+      dockerfile: docker/compose/elasticsearch/Dockerfile
+      cache_from:
+        - mozdef/mozdef_elasticsearch
+        - mozdef_elasticsearch:latest
    command: bin/elasticsearch
    restart: always
    volumes:
@ -38,6 +66,13 @@ services:
    networks:
      - default
  rabbitmq:
+    image: mozdef/mozdef_rabbitmq
+    build:
+      context: ../../
+      dockerfile: docker/compose/rabbitmq/Dockerfile
+      cache_from:
+        - mozdef/mozdef_rabbitmq
+        - mozdef_rabbitmq:latest
    restart: always
    command: rabbitmq-server
    volumes:
@ -50,10 +85,24 @@ services:

  # MozDef Specific Containers
  base:
+    image: mozdef/mozdef_base
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_base/Dockerfile
+      cache_from:
+        - mozdef/mozdef_base
+        - mozdef_base:latest
    command: bash -c 'su - mozdef -c /opt/mozdef/envs/mozdef/cron/update_geolite_db.sh'
    volumes:
      - geolite_db:/opt/mozdef/envs/mozdef/data
  bootstrap:
+    image: mozdef/mozdef_bootstrap
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_bootstrap/Dockerfile
+      cache_from:
+        - mozdef/mozdef_bootstrap
+        - mozdef_bootstrap:latest
    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/backup.conf'
    depends_on:
      - base
@ -63,8 +112,15 @@ services:
    networks:
      - default
  alertplugins:
+    image: mozdef/mozdef_alertplugins
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_alertplugins/Dockerfile
+      cache_from:
+        - mozdef/mozdef_alertplugins
+        - mozdef_alertplugins:latest
    restart: always
-    command: bash -c 'sleep 90 && source /opt/mozdef/envs/python/bin/activate && python alert_worker.py -c alert_worker.conf'
+    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python alert_worker.py -c alert_worker.conf'
    depends_on:
      - base
      - elasticsearch
@ -74,8 +130,15 @@ services:
    networks:
      - default
  alerts:
+    image: mozdef/mozdef_alerts
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_alerts/Dockerfile
+      cache_from:
+        - mozdef/mozdef_alerts
+        - mozdef_alerts:latest
    restart: always
-    command: bash -c 'sleep 90 && source /opt/mozdef/envs/python/bin/activate && celery -A celeryconfig worker --loglevel=info --beat'
+    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && celery -A celeryconfig worker --loglevel=info --beat'
    depends_on:
      - base
      - elasticsearch
@ -96,8 +159,15 @@ services:
  #   volumes:
  #     - geolite_db:/opt/mozdef/envs/mozdef/data/
  cron:
+    image: mozdef/mozdef_cron
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_cron/Dockerfile
+      cache_from:
+        - mozdef/mozdef_cron
+        - mozdef_cron:latest
    restart: always
-    command: bash -c 'sleep 90 && crond -n'
+    command: bash -c 'crond -n'
    volumes:
      - cron:/opt/mozdef/envs/mozdef/cron
      - geolite_db:/opt/mozdef/envs/mozdef/data/
@ -110,8 +180,15 @@ services:
    networks:
      - default
  loginput:
+    image: mozdef/mozdef_loginput
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_loginput/Dockerfile
+      cache_from:
+        - mozdef/mozdef_loginput
+        - mozdef_loginput:latest
    restart: always
-    command: bash -c 'sleep 90 && source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
+    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
    depends_on:
      - base
      - elasticsearch
@ -119,11 +196,17 @@ services:
      - bootstrap
    networks:
      - default
-  mq_eventtask:
+  mq_worker:
+    image: mozdef/mozdef_mq_worker
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_mq_worker/Dockerfile
+      cache_from:
+        - mozdef/mozdef_mq_worker
+        - mozdef_mq_worker:latest
    image: mozdef_mq_worker
    restart: always
-    command: bash -c 'sleep 90 && source /opt/mozdef/envs/python/bin/activate && python esworker_eventtask.py -c esworker_eventtask.conf'
-    scale: 1
+    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_eventtask.py -c esworker_eventtask.conf'
    depends_on:
      - base
      - rabbitmq
@ -135,6 +218,13 @@ services:
    volumes:
      - geolite_db:/opt/mozdef/envs/mozdef/data/
  meteor:
+    image: mozdef/mozdef_meteor
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_meteor/Dockerfile
+      cache_from:
+        - mozdef/mozdef_meteor
+        - mozdef_meteor:latest
    restart: always
    command: bash -c 'node bundle/main.js'
    depends_on:
@ -143,6 +233,13 @@ services:
    networks:
      - default
  rest:
+    image: mozdef/mozdef_rest
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_rest/Dockerfile
+      cache_from:
+        - mozdef/mozdef_rest
+        - mozdef_rest:latest
    restart: always
    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
    depends_on:
@ -153,17 +250,31 @@ services:
    networks:
      - default
  syslog:
+    image: mozdef/mozdef_syslog
+    build:
+      context: ../../
+      dockerfile: docker/compose/mozdef_syslog/Dockerfile
+      cache_from:
+        - mozdef/mozdef_syslog
+        - mozdef_syslog:latest
    restart: always
-    command: bash -c 'sleep 95 && /usr/sbin/syslog-ng --no-caps -F'
+    command: bash -c '/usr/sbin/syslog-ng --no-caps -F'
    depends_on:
      - loginput
-      - mq_eventtask
+      - mq_worker
    ports:
      - 514:514/udp
      - 514:514
    networks:
      - default
  tester:
+    image: mozdef/mozdef_tester
+    build:
+      context: ../../
+      dockerfile: docker/compose/tester/Dockerfile
+      cache_from:
+        - mozdef/mozdef_tester
+        - mozdef_tester:latest
    networks:
      - default

--- a/docker/compose/mozdef_alertplugins/Dockerfile
+++ b/docker/compose/mozdef_alertplugins/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_alerts/Dockerfile
+++ b/docker/compose/mozdef_alerts/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_bootstrap/Dockerfile
+++ b/docker/compose/mozdef_bootstrap/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_bot/Dockerfile
+++ b/docker/compose/mozdef_bot/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_cron/Dockerfile
+++ b/docker/compose/mozdef_cron/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_loginput/Dockerfile
+++ b/docker/compose/mozdef_loginput/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_mq_worker/Dockerfile
+++ b/docker/compose/mozdef_mq_worker/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_rest/Dockerfile
+++ b/docker/compose/mozdef_rest/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

--- a/docker/compose/mozdef_sampledata/Dockerfile
+++ b/docker/compose/mozdef_sampledata/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 LABEL maintainer="mozdef@mozilla.com"

@ -11,4 +11,4 @@ RUN chmod u+rwx /opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh

 WORKDIR /opt/mozdef/envs/mozdef/examples/demo

-USER root
+USER root
--- a/docker/compose/tester/Dockerfile
+++ b/docker/compose/tester/Dockerfile
@ -1,4 +1,4 @@
-FROM mozdef_base:latest
+FROM mozdef/mozdef_base

 COPY tests /opt/mozdef/envs/mozdef/tests
 COPY alerts /opt/mozdef/envs/mozdef/alerts
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@ -1,25 +0,0 @@
---
-version: '2.2'
-services:
-  elasticsearch:
-    restart: always
-    command: bin/elasticsearch
-    # ports:
-    #   - 9200:9200
-    networks:
-      - default
-  rabbitmq:
-    restart: always
-    command: rabbitmq-server
-    # ports:
-    #   - 5672:5672
-    networks:
-      - default
-  base:
-  tester:
-    depends_on:
-      - base
-    networks:
-      - default
-networks:
-  default: