Merge pull request #1028 from mozilla/remove_hardcoded_pythonenv

Move virtualenv code into base container

docker/compose/docker-compose-cloudy-mozdef.yml

@@ -40,7 +40,7 @@ services:
     image: mozdef/mozdef_bootstrap
     env_file:
       - cloudy_mozdef.env
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/backup.conf'
+    command: bash -c 'python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/backup.conf'
     depends_on:
       - base
     networks:
@@ -58,7 +58,7 @@ services:
     env_file:
       - cloudy_mozdef.env
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python alert_worker.py -c alert_worker.conf'
+    command: bash -c 'python alert_worker.py -c alert_worker.conf'
     depends_on:
       - base
       - alerts
@@ -72,7 +72,7 @@ services:
     volumes:
       - /opt/mozdef/docker/compose/mozdef_alerts/files/config.py:/opt/mozdef/envs/mozdef/alerts/lib/config.py
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && celery -A celeryconfig worker --loglevel=info --beat'
+    command: bash -c 'celery -A celeryconfig worker --loglevel=info --beat'
     depends_on:
       - base
       - bootstrap
@@ -97,7 +97,7 @@ services:
     env_file:
       - cloudy_mozdef.env
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
+    command: bash -c 'python index.py -c index.conf'
     depends_on:
       - base
       - bootstrap
@@ -124,7 +124,7 @@ services:
     env_file:
       - cloudy_mozdef.env
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
+    command: bash -c 'python index.py -c index.conf'
     depends_on:
       - base
       - mongodb
@@ -160,7 +160,7 @@ services:
     env_file:
       - cloudy_mozdef.env
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_eventtask.py -c esworker_eventtask.conf'
+    command: bash -c 'python esworker_eventtask.py -c esworker_eventtask.conf'
     scale: 1
     depends_on:
       - base
@@ -177,7 +177,7 @@ services:
       - cloudy_mozdef.env
       - cloudy_mozdef_mq_cloudtrail.env
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_cloudtrail.py -c esworker_cloudtrail.conf'
+    command: bash -c 'python esworker_cloudtrail.py -c esworker_cloudtrail.conf'
     scale: 1
     depends_on:
       - base
@@ -194,7 +194,7 @@ services:
       - cloudy_mozdef.env
       - cloudy_mozdef_mq_sns_sqs.env
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_sns_sqs.py -c esworker_sns_sqs.conf'
+    command: bash -c 'python esworker_sns_sqs.py -c esworker_sns_sqs.conf'
     scale: 1
     depends_on:
       - base

docker/compose/docker-compose-rest.yml

@@ -6,7 +6,7 @@ services:
       context: ../../
       dockerfile: docker/compose/mozdef_rest/Dockerfile
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
+    command: bash -c 'python index.py -c index.conf'
     depends_on:
       - base
       - elasticsearch

docker/compose/docker-compose-sampledata.yml

@@ -6,7 +6,7 @@ services:
       context: ../../
       dockerfile: docker/compose/mozdef_sampledata/Dockerfile
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && /opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh'
+    command: bash -c '/opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh'
     links:
       - elasticsearch
     depends_on:

docker/compose/docker-compose.yml

@@ -103,7 +103,7 @@ services:
       cache_from:
         - mozdef/mozdef_bootstrap
         - mozdef_bootstrap:latest
-    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && source /opt/mozdef/envs/python/bin/activate && python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/backup.conf'
+    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/backup.conf'
     depends_on:
       - base
       - elasticsearch
@@ -120,7 +120,7 @@ services:
         - mozdef/mozdef_alertplugins
         - mozdef_alertplugins:latest
     restart: always
-    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && source /opt/mozdef/envs/python/bin/activate && python alert_worker.py -c alert_worker.conf'
+    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python alert_worker.py -c alert_worker.conf'
     depends_on:
       - base
       - elasticsearch
@@ -138,7 +138,7 @@ services:
         - mozdef/mozdef_alerts
         - mozdef_alerts:latest
     restart: always
-    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && source /opt/mozdef/envs/python/bin/activate && celery -A celeryconfig worker --loglevel=info --beat'
+    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && celery -A celeryconfig worker --loglevel=info --beat'
     depends_on:
       - base
       - elasticsearch
@@ -148,7 +148,7 @@ services:
       - default
   # bot:
   #   restart: always
-  #   command: bash -c 'sleep 90 && source /opt/mozdef/envs/python/bin/activate && python mozdefbot.py -c mozdefbot.conf'
+  #   command: bash -c 'sleep 90 && python mozdefbot.py -c mozdefbot.conf'
   #   depends_on:
   #     - base
   #     - rabbitmq
@@ -188,7 +188,7 @@ services:
         - mozdef/mozdef_loginput
         - mozdef_loginput:latest
     restart: always
-    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
+    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python index.py -c index.conf'
     depends_on:
       - base
       - elasticsearch
@@ -205,7 +205,7 @@ services:
         - mozdef/mozdef_mq_worker
         - mozdef_mq_worker:latest
     restart: always
-    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && source /opt/mozdef/envs/python/bin/activate && python esworker_eventtask.py -c esworker_eventtask.conf'
+    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python esworker_eventtask.py -c esworker_eventtask.conf'
     depends_on:
       - base
       - rabbitmq
@@ -240,7 +240,7 @@ services:
         - mozdef/mozdef_rest
         - mozdef_rest:latest
     restart: always
-    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && source /opt/mozdef/envs/python/bin/activate && python index.py -c index.conf'
+    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python index.py -c index.conf'
     depends_on:
       - base
       - elasticsearch

docker/compose/live-reload.override.yml

@@ -23,7 +23,7 @@ services:
       context: ../../
       dockerfile: docker/compose/mozdef_sampledata/Dockerfile
     restart: always
-    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && /opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh'
+    command: bash -c '/opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh'
     links:
       - elasticsearch
     depends_on:

docker/compose/mozdef_base/Dockerfile

@@ -34,23 +34,17 @@ COPY cron/update_geolite_db.py /opt/mozdef/envs/mozdef/cron/update_geolite_db.py
 COPY cron/update_geolite_db.conf /opt/mozdef/envs/mozdef/cron/update_geolite_db.conf
 COPY cron/update_geolite_db.sh /opt/mozdef/envs/mozdef/cron/update_geolite_db.sh
 
+COPY mozdef_util /opt/mozdef/envs/mozdef/mozdef_util
+
 RUN chown -R mozdef:mozdef /opt/mozdef/
+
 USER mozdef
 RUN \
   virtualenv /opt/mozdef/envs/python && \
   source /opt/mozdef/envs/python/bin/activate && \
-  pip install -r /opt/mozdef/envs/mozdef/requirements.txt
-
-# Local copy for dev
-COPY mozdef_util /opt/mozdef/envs/mozdef/mozdef_util
-USER root
-RUN \
-  source /opt/mozdef/envs/python/bin/activate && \
+  pip install -r /opt/mozdef/envs/mozdef/requirements.txt && \
   cd /opt/mozdef/envs/mozdef/mozdef_util && \
   pip install -e .
-RUN chown -R mozdef:mozdef /opt/mozdef/
-
-USER mozdef
 
 RUN mkdir /opt/mozdef/envs/mozdef/data
 
@@ -58,4 +52,7 @@ WORKDIR /opt/mozdef/envs/mozdef
 
 VOLUME /opt/mozdef/envs/mozdef/data
 
+# Automatically source into python virtual environment
+ENV PATH=/opt/mozdef/envs/python/bin:$PATH
+
 USER root

docker/compose/tester/Dockerfile

@@ -15,8 +15,4 @@ COPY docker/compose/tester/files/tests_config.conf /opt/mozdef/envs/mozdef/tests
 COPY docker/compose/tester/files/loginput_index.conf /opt/mozdef/envs/mozdef/tests/loginput/index.conf
 COPY docker/compose/tester/files/rest_index.conf /opt/mozdef/envs/mozdef/tests/rest/index.conf
 
-USER root
-
-RUN \
-  source /opt/mozdef/envs/python/bin/activate && \
-  pip install -r /opt/mozdef/envs/mozdef/tests/requirements_tests.txt
+RUN pip install -r /opt/mozdef/envs/mozdef/tests/requirements_tests.txt