From 8c53240bb9d165f86e1666f76a5b6d6e817016a1 Mon Sep 17 00:00:00 2001 From: Aaron Meihm Date: Fri, 29 Sep 2017 15:34:46 -0500 Subject: [PATCH] add docker-compose based demo environment The existing MIG docker image can be used as a standalone demo; this change extends the image to be used in a compose configuration, creating separate containers for each component. The compose environment is intended to potentially be used for deployments other than a demo, by selecting individual components intended for execution and configuring these components using the environment. --- Dockerfile | 13 +- tools/compose/README.rst | 199 ++++++++++++++++++ tools/compose/compose.env | 38 ++++ tools/compose/docker-compose.yml | 82 ++++++++ tools/compose/migagenthost1/Dockerfile | 11 + tools/compose/migagenthost1/mig-agent.cfg | 28 +++ tools/compose/migagenthost1/mig-agent.conf | 4 + tools/compose/migagenthost1/run.sh | 53 +++++ tools/compose/migagenthost2/Dockerfile | 17 ++ tools/compose/migagenthost2/demofiles/demokey | 27 +++ .../migagenthost2/demofiles/demokey.pub | 1 + .../migagenthost2/demofiles/samplefile1.txt | 1 + tools/compose/migagenthost2/mig-agent.cfg | 28 +++ tools/compose/migagenthost2/mig-agent.conf | 4 + tools/compose/migagenthost2/run.sh | 53 +++++ tools/compose/migagenthost3/Dockerfile | 14 ++ tools/compose/migagenthost3/audit.cfg | 4 + tools/compose/migagenthost3/audit.rules.json | 30 +++ tools/compose/migagenthost3/dispatch.cfg | 3 + tools/compose/migagenthost3/mig-agent.cfg | 28 +++ tools/compose/migagenthost3/mig-agent.conf | 4 + tools/compose/migagenthost3/run.sh | 53 +++++ tools/compose/migapi/Dockerfile | 11 + tools/compose/migapi/api.cfg | 27 +++ tools/compose/migapi/mig-api.conf | 3 + tools/compose/migapi/run.sh | 62 ++++++ tools/compose/migdb/Dockerfile | 9 + tools/compose/migdb/build.sh | 23 ++ tools/compose/migdb/run.sh | 17 ++ tools/compose/migrelay/Dockerfile | 9 + tools/compose/migrelay/build.sh | 30 +++ tools/compose/migrelay/run.sh | 28 +++ tools/compose/migscheduler/Dockerfile | 12 ++ tools/compose/migscheduler/build.sh | 30 +++ tools/compose/migscheduler/mig-scheduler.conf | 4 + tools/compose/migscheduler/run.sh | 9 + tools/compose/migscheduler/scheduler.cfg | 39 ++++ tools/compose/migworkstation/Dockerfile | 14 ++ tools/compose/migworkstation/mig-agent.cfg | 28 +++ tools/compose/migworkstation/mig-agent.conf | 4 + tools/compose/migworkstation/run.sh | 62 ++++++ tools/compose/migworkstation/simpleweb.conf | 4 + tools/compose/migworkstation/simpleweb.py | 30 +++ tools/docker_install.sh | 84 -------- tools/docker_start.sh | 135 +++++++++++- tools/mig-agent.cfg.demo | 2 +- tools/scheduler.cfg.demo | 4 +- 47 files changed, 1282 insertions(+), 93 deletions(-) create mode 100644 tools/compose/README.rst create mode 100644 tools/compose/compose.env create mode 100644 tools/compose/docker-compose.yml create mode 100644 tools/compose/migagenthost1/Dockerfile create mode 100644 tools/compose/migagenthost1/mig-agent.cfg create mode 100644 tools/compose/migagenthost1/mig-agent.conf create mode 100644 tools/compose/migagenthost1/run.sh create mode 100644 tools/compose/migagenthost2/Dockerfile create mode 100644 tools/compose/migagenthost2/demofiles/demokey create mode 100644 tools/compose/migagenthost2/demofiles/demokey.pub create mode 100644 tools/compose/migagenthost2/demofiles/samplefile1.txt create mode 100644 tools/compose/migagenthost2/mig-agent.cfg create mode 100644 tools/compose/migagenthost2/mig-agent.conf create mode 100644 tools/compose/migagenthost2/run.sh create mode 100644 tools/compose/migagenthost3/Dockerfile create mode 100644 tools/compose/migagenthost3/audit.cfg create mode 100644 tools/compose/migagenthost3/audit.rules.json create mode 100644 tools/compose/migagenthost3/dispatch.cfg create mode 100644 tools/compose/migagenthost3/mig-agent.cfg create mode 100644 tools/compose/migagenthost3/mig-agent.conf create mode 100644 tools/compose/migagenthost3/run.sh create mode 100644 tools/compose/migapi/Dockerfile create mode 100644 tools/compose/migapi/api.cfg create mode 100644 tools/compose/migapi/mig-api.conf create mode 100644 tools/compose/migapi/run.sh create mode 100644 tools/compose/migdb/Dockerfile create mode 100644 tools/compose/migdb/build.sh create mode 100644 tools/compose/migdb/run.sh create mode 100644 tools/compose/migrelay/Dockerfile create mode 100644 tools/compose/migrelay/build.sh create mode 100644 tools/compose/migrelay/run.sh create mode 100644 tools/compose/migscheduler/Dockerfile create mode 100644 tools/compose/migscheduler/build.sh create mode 100644 tools/compose/migscheduler/mig-scheduler.conf create mode 100644 tools/compose/migscheduler/run.sh create mode 100644 tools/compose/migscheduler/scheduler.cfg create mode 100644 tools/compose/migworkstation/Dockerfile create mode 100644 tools/compose/migworkstation/mig-agent.cfg create mode 100644 tools/compose/migworkstation/mig-agent.conf create mode 100644 tools/compose/migworkstation/run.sh create mode 100644 tools/compose/migworkstation/simpleweb.conf create mode 100755 tools/compose/migworkstation/simpleweb.py delete mode 100644 tools/docker_install.sh diff --git a/Dockerfile b/Dockerfile index b8178d2c..3d4d6ab3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,11 @@ FROM ubuntu:xenial MAINTAINER Mozilla +# Builds the MIG base image; this creates an image that has most of the MIG software +# compiled with it's default options. + +# See tools/docker_start.sh which is the default CMD entry point for this image. + RUN apt-get update && \ apt-get install -y sudo golang git make \ curl rng-tools tmux postgresql rabbitmq-server \ @@ -13,9 +18,15 @@ ADD . /go/src/mig.ninja/mig RUN chown -R mig /go USER mig + +# Build the various tools that are found in a typical MIG environment. RUN export GOPATH=/go && \ cd /go/src/mig.ninja/mig && \ - yes | bash ./tools/docker_install.sh && \ + go install mig.ninja/mig/mig-agent && \ + go install mig.ninja/mig/mig-api && \ + go install mig.ninja/mig/mig-scheduler && \ + go install mig.ninja/mig/client/mig-console && \ + go install mig.ninja/mig/client/mig && \ cp /go/src/mig.ninja/mig/tools/docker_start.sh /mig/docker_start.sh && \ chmod +x /mig/docker_start.sh diff --git a/tools/compose/README.rst b/tools/compose/README.rst new file mode 100644 index 00000000..53a574e7 --- /dev/null +++ b/tools/compose/README.rst @@ -0,0 +1,199 @@ +MIG docker-compose environment +============================== + +.. sectnum:: +.. contents:: Table of Contents + +The `docker-compose`_ environment for MIG is intended to be used as a demo environment, but could +also be modified for production based deployments of MIG components in containers. In this document +we describe how to configure the compose environment as a sandbox, and also run a few test queries +that demonstrate the various interactions between components in the docker setup. + +.. _`docker-compose`: https://docs.docker.com/compose/ + +Host configuration +------------------ + +Described in more detail later, but one container in the default compose configuration runs in privileged +mode in order to demonstrate memory scanning and audit trail integration. Make sure the host environment +is not running the standard Linux ``auditd`` if you want to play with this functionality. If it is, just +stop the audit daemon before creating the compose containers. + +Get the MIG docker image +------------------------ + +The compose environment makes use of the MIG base docker image for all of the various containers. +The easiest way to make sure you have the latest version is to build the docker image from the MIG +source repository. This example assumes you are checking the MIG respository out into your GOPATH +which would be a typical scenario, however this is not required. + +.. code:: bash + + $ GOPATH=$HOME/go + $ mkdir -p $GOPATH/src/mig.ninja + $ cd $GOPATH/src/mig.ninja + $ git clone https://github.com/mozilla/mig.git + $ cd mig + $ docker build -t mozilla/mig:latest . + +This should take a minute or so and you will now have the base docker image. The base docker image +itself can be used to run a demo environment standalone (e.g., a single container). We will now use +compose to deploy MIG into a multi-container configuration. + +Create and start the containers using docker-compose +---------------------------------------------------- + +To create and start the various MIG containers, use ``docker-compose``. The compose configuration will +create the following: + +* A workstation container, simulating an investigators workstation, also runs a MIG agent +* A database container hosting the MIG Postgres database +* A relay container hosting the MIG RabbitMQ relay (agents connect to this) +* A scheduler container, which runs the MIG scheduler +* An API container, which runs the MIG API (investigators connect to this) +* 3 agent hosts, which run the MIG agent + +See `docker-compose.yml` to get a detailed idea of the configuration. + +.. _`docker-compose.yml`: docker-compose.yml + +**Note**: In the default configuration, the third agent host (migagenthost3) runs in docker privileged +mode. Among other things this means that agent host has a degree of control over the host environment and +the host PID namespace. This container runs in privileged mode, as this is required to demonstrate some +capabilities of MIG such as memory analysis and kernel auditing integration. Generally if the agent is +deployed in a production scenario using docker, it is run in a privileged docker container. + +To disable this functionality, remove privileged related configuration from ``docker-compose.yml`` before +creating the containers, note however this will prevent certain functionality from being available. + +Start the compose containers: + +.. code:: bash + + $ cd tools/compose + $ docker-compose up -d + +This will spawn the containers in detached mode, after a few seconds in the output of ``docker ps`` you +should see a bunch of MIG related containers executing. + +You can stop the containers at any time by running ``docker-compose stop`` in the same directory as the +previous command. + +About the miginvestigator volume +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One volume is used in the compose setup, which is ``miginvestigator`` and it is mounted in a few of the +containers at ``/miginvestigator``. This volume when empty is populated with a PGP key used for a demonstration +investigator, and is subsequently configured in the database. If you delete the database container, be sure +to also remove this volume, this ensures fresh keys are created and added to the new database. This is only +required if you delete the database container without also deleting the miginvestigator volume. + +Sample queries to try in the compose environment +------------------------------------------------ + +To run some queries, enter the workstation container. You can get the container ID in the output of the +``docker ps`` command. + +.. code:: bash + + $ docker exec -t -i /bin/bash + +The command line query tool (``/go/bin/mig``) and console tool (``/go/bin/mig-console``) are available +here to experiment with. From here you can query all 4 of the sample agents that have been deployed. + +**Note**: The agents register with the scheduler by generating heartbeat messages periodically. Because +of the way the container environment comes up, it's possible the scheduler may miss the first few heartbeat +messages; it may take a minute or so for all agents to become available for query. + +Locate files on target systems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform a simple query, to locate a shadow file containing a root user. + +.. code:: bash + + $ /go/bin/mig file -path /etc -name '^shadow$' -content '^root:' + +Find a demonstration file containing specific contents, deployed to one of the agent hosts (migagenthost2). + +.. code:: bash + + $ /go/bin/mig file -path /bin -path /etc -path /sbin -path /lib -content DEMOCONTENT -size '<100k' -maxdepth 2 + +Find the same demo file using a hash. + +.. code:: bash + + $ /go/bin/mig file -path /etc -maxdepth 0 -size '<100k' -sha2 b70dd6990e416c3b1d9b2f45ef63a4e17badd15c87b4c8558605f964b4b14c5e + +Search for a given deployed package +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Find any installed versions of the python package. + +.. code:: bash + + $ /go/bin/mig pkg -name '^python$' + +Find contents in memory +~~~~~~~~~~~~~~~~~~~~~~~ + +Find any processes with an RSA private key present in the process memory address space. + +.. code:: bash + + $ /go/bin/mig memory -content 'BEGIN RSA PRIVATE' + +Note in this case, we only get a result back from one docker container, which is our privileged container. The +other containers are not executing with sufficient access to some operating system facilities, however because +the privileged container has this access, it is able to report on processes running both on the host operating +system and the other containers on the machine. + +Likewise, find any processes containing the string "OpenSSH" in memory: + +.. code:: bash + + $ /go/bin/mig memory -content 'OpenSSH' + +You'll probably see a number of the MIG components in this list, since they are actively processing the query +which itself contains this string. + +Find processes linked against a given library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Find processes linked against OpenSSL libcrypto: + +.. code:: bash + + $ /go/bin/mig memory -lib libcrypto.so.1.0.0 + +View kernel audit trail +~~~~~~~~~~~~~~~~~~~~~~~ + +The queries we have demonstrated thus far illustrate some of the query capabilities of MIG. These execute +modules in the agent, which return the results of the query. MIG also has the ability to persistently run modules; +these are referred to in the documentation as persistent modules and can be used to perform more active on-going +monitoring and alerting from the agent. + +In the demo environment, the privileged container is also running the ``audit`` and ``dispatch`` persistent +modules. These modules interact with the Linux kernel of Netlink to obtain the kernel audit trail, and dispatch +the events from the agent to SNS/HTTP endpoints respectively. If auditing is executing, you can take a look at the +output from the audit module by having a look at the supervisor log for a simple HTTP POST endpoint running in the +demo environment. + +.. code:: bash + + $ sudo su - + # cd /var/log/supervisor + # tail -f simpleweb-stdout* + +Here you can see the output of the dispatch module from the agent on migagenthost3, which should contain various +kernel audit messages. The policy the agent installs in the demo environment logs instances of the execve system +call, and any writes to the password or shadow file as an example. + +Additional samples +~~~~~~~~~~~~~~~~~~ + +For additional examples, see the MIG `cheatsheet`_. + +.. _`cheatsheet`: ../../doc/cheatsheet.rst diff --git a/tools/compose/compose.env b/tools/compose/compose.env new file mode 100644 index 00000000..0fd0e69b --- /dev/null +++ b/tools/compose/compose.env @@ -0,0 +1,38 @@ +# Hostname containers will use for database +MIGDBHOST=migdb + +# Hostname containers will use for relay +MIGRELAYHOST=migrelay + +# Hostname containers will use for API +MIGAPIHOST=migapi + +# Database passwords components will use when connecting to the database +MIGDBADMINPASSWORD=dbadmin +MIGDBAPIPASSWORD=dbapi +MIGDBSCHEDULERPASSWORD=dbscheduler + +# Relay passwords components will use when connecting to RabbitMQ +# +# Note the MIGRELAYAGENTS variable is different here, this is a space separated list +# of accounts that will be added for agent connections, in the format username:password +MIGRELAYADMINPASSWORD=relayadmin +MIGRELAYSCHEDULERPASSWORD=relayscheduler +MIGRELAYWORKERPASSWORD=relayworker +MIGRELAYAGENTS=agent:agent agent2:agent2 + +# The username and password the agent container will use when connecting to the RabbitMQ +# relay, would correspond to MIGRELAYAGENTS in a demo environment since the agent container +# will be connecting to our relay container. +AGENTUSER=agent +AGENTPASSWORD=agent + +# If yes, when the API container is started, it will generate an investigator +# key pair in the miginvestigator volume if it does not exist. After generation, it +# will start itself up with authentication disabled and add the new investigator +# key to the database if missing. +# +# This also causes other containers, for example the workstation container and agent +# containers to import this investigator key into the agent keyring and make use of it +# for queries. Setting this to yes is generally useful in a demo or sandbox environment. +GENERATEINVESTIGATOR=yes diff --git a/tools/compose/docker-compose.yml b/tools/compose/docker-compose.yml new file mode 100644 index 00000000..20e28a64 --- /dev/null +++ b/tools/compose/docker-compose.yml @@ -0,0 +1,82 @@ +--- + +version: '2.2' + +services: + migdb: + build: migdb/ + env_file: compose.env + networks: + - default + migrelay: + build: migrelay/ + env_file: compose.env + depends_on: + - migdb + networks: + - default + migscheduler: + build: migscheduler/ + env_file: compose.env + depends_on: + - migrelay + networks: + - default + migapi: + build: migapi/ + env_file: compose.env + depends_on: + - migrelay + networks: + - default + volumes: + - miginvestigator:/miginvestigator + migworkstation: + build: migworkstation/ + environment: + CLIENTMODULES: modmemory + env_file: compose.env + depends_on: + - migapi + networks: + - default + volumes: + - miginvestigator:/miginvestigator + migagenthost1: + build: migagenthost1/ + env_file: compose.env + depends_on: + - migapi + networks: + - default + volumes: + - miginvestigator:/miginvestigator + migagenthost2: + build: migagenthost2/ + env_file: compose.env + depends_on: + - migapi + networks: + - default + volumes: + - miginvestigator:/miginvestigator + migagenthost3: + build: migagenthost3/ + environment: + AGENTMODULES: modaudit moddispatch modmemory + env_file: compose.env + privileged: true + pid: "host" + depends_on: + - migapi + networks: + - default + volumes: + - miginvestigator:/miginvestigator + + +networks: + default: + +volumes: + miginvestigator: diff --git a/tools/compose/migagenthost1/Dockerfile b/tools/compose/migagenthost1/Dockerfile new file mode 100644 index 00000000..62e4b7b4 --- /dev/null +++ b/tools/compose/migagenthost1/Dockerfile @@ -0,0 +1,11 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +RUN sudo mkdir -p /etc/mig +COPY mig-agent.cfg /etc/mig/mig-agent.cfg + +COPY mig-agent.conf /etc/supervisor/conf.d/mig-agent.conf + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migagenthost1/mig-agent.cfg b/tools/compose/migagenthost1/mig-agent.cfg new file mode 100644 index 00000000..1cba648f --- /dev/null +++ b/tools/compose/migagenthost1/mig-agent.cfg @@ -0,0 +1,28 @@ +[agent] + relay = "amqp://AGENTUSER:AGENTPASSWORD@MIGRELAYHOST:5672/mig" + api = "http://MIGAPIHOST:1664/api/v1/" + socket = "127.0.0.1:51664" + heartbeatfreq = "30s" + moduletimeout = "300s" + isimmortal = on + installservice = on + discoverpublicip = on + checkin = off + refreshenv = "5m" + extraprivacymode = off + nopersistmods = off + onlyVerifyPubKey = false + tags = "environment:migdemo" + tags = "build:docker" + +[stats] + maxactions = 15 + +[certs] + ca = "" + cert= "" + key = "" + +[logging] + mode = "stdout" + level = "debug" diff --git a/tools/compose/migagenthost1/mig-agent.conf b/tools/compose/migagenthost1/mig-agent.conf new file mode 100644 index 00000000..e6f595b8 --- /dev/null +++ b/tools/compose/migagenthost1/mig-agent.conf @@ -0,0 +1,4 @@ +[program:mig-agent] +command=/go/bin/mig-agent -d +startretries=9999 +autorestart=true diff --git a/tools/compose/migagenthost1/run.sh b/tools/compose/migagenthost1/run.sh new file mode 100644 index 00000000..dc612d9f --- /dev/null +++ b/tools/compose/migagenthost1/run.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# If AGENTMODULES is set, rebuild the agent with the indicated module tags +if [[ ! -z "$AGENTMODULES" ]]; then + sudo env GOPATH=/go \ + go install -tags "$AGENTMODULES" mig.ninja/mig/mig-agent +fi + +# Update API configuration using the environment +sudo sed -i "s/AGENTUSER/$AGENTUSER/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/AGENTPASSWORD/$AGENTPASSWORD/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGRELAYHOST/$MIGRELAYHOST/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGAPIHOST/$MIGAPIHOST/g" /etc/mig/mig-agent.cfg + +# If the environment indicates investigator generation is enabled, stage the key material +# and an ACL in the agents keyring/configuration; we also build the command line tools +# configuration using the same data. +if [[ $GENERATEINVESTIGATOR == "yes" ]]; then + while [[ ! -f /miginvestigator/fingerprint.txt ]]; do + sleep 1 + done + sudo mkdir -p /etc/mig/agentkeys + sudo cp /miginvestigator/pubkey.asc /etc/mig/agentkeys/pubkey.asc + keyid=`head -1 /miginvestigator/fingerprint.txt` + sudo sh -c "cat > /etc/mig/acl.cfg << EOF +{ + \"default\": { + \"minimumweight\": 1, + \"investigators\": { + \"mig\": { + \"fingerprint\": \"${keyid}\", + \"weight\": 1 + } + } + } +} +EOF +" + sudo cp -R /miginvestigator/mig ~/.mig + sudo chown -R $(whoami) ~/.mig + cat > ~/.migrc << EOF +[api] + url = "http://$MIGAPIHOST:1664/api/v1/" + skipverifycert = on +[gpg] + home = "$HOME/.mig/" + keyid = "$keyid" +[targets] + macro = all:status='online' +EOF +fi + +sudo /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -n diff --git a/tools/compose/migagenthost2/Dockerfile b/tools/compose/migagenthost2/Dockerfile new file mode 100644 index 00000000..9a2c4e67 --- /dev/null +++ b/tools/compose/migagenthost2/Dockerfile @@ -0,0 +1,17 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +RUN sudo mkdir -p /etc/mig +COPY mig-agent.cfg /etc/mig/mig-agent.cfg + +COPY mig-agent.conf /etc/supervisor/conf.d/mig-agent.conf + +# Stage a few files that can be used as part of a demo/sandbox +COPY demofiles/samplefile1.txt /etc/samplefile1.txt + +RUN sudo mkdir -p /root/.ssh +COPY demofiles/demokey /root/.ssh/demokey + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migagenthost2/demofiles/demokey b/tools/compose/migagenthost2/demofiles/demokey new file mode 100644 index 00000000..9998b3e5 --- /dev/null +++ b/tools/compose/migagenthost2/demofiles/demokey @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAvnBR6tEXqbX2TNZBuMdXVSF3kuOrZfWBkRywBX40pRiIdIra +n+q8aZgwut7VSqGIHuHNjKURk0vCQuwS7MN3r58DVWVOMN3iBpu9HlSye5pIcL2S +wHcNJY48kYFlYvfiuJIkHIdxvwDJyZMSjbwWFsbrWoTz4OvQCQrX/qWGfZ0f2HRl +AV6VrRGPcXQNBGh8NNrUMjbBVoGBemi2Cf4yJaH99Sdjg7b2fT9vG/MyVJ336mWs +/r3cNbH2A/1XzrkmHJKdGjuFpNQNLoZKkAJcOKwo3v806utC32jBpDhAanakpapk +fUuuI65i/eRBe82oTDlIZMeJlAN/ZDC8wpJ/IwIDAQABAoIBAET9b8+/m/DxaJ8B +Z6HHJY5Dp4vhSrCPNjcZy3LR+8RrlNOTkgwo/8TJS64HGAIHbsaxSsPO8QUJhwCz +gU+gqHVJB+4h2ye82itGtEaOrHI2vFOLeeSbvHmEg9C6rb4iy1HOL6LeNojZgkCy +NSB/qQjrW0vZqngrNZxw/zrTQFu0thvQxm8Yd7xYQQxigUuPVJg991rio7kJ6Y6M +z3uXkLuakvV0AryJH7jjo5r2jgyluipkl9pnyTOkWFNrk/lODOOL0KdK+LTpcmbx +Q65hMWa95SLZymGZ216jilDgluVcpm6qMvryH0zViP1DMV8n0EuyvF4nIEIshXlj +FXkAlmECgYEA6ecniuKrUMZRSo9g+XfMMMhr1qJANYtYtp7CWsuhZU8c5ikgcdMh +5qIWDthL0/Y4kWLJgEVXkEUl0KppvZr9fVg2u++8BiNXU/zQE6pmQT0QX6HzAEZj +VB+lfLuafZ5cWUoKjNcKsty/K83vP7+T5JCHgLYavMTDCkwpg8+A/AsCgYEA0G4A +gjjDSHjw/cage4knQkNTBloM3AT6vo+V5t+omSI4mGPFTDbkPxVqf1rQIy5Ua4KO +Pa9rN70/sqA0GMew+37Nvc8LU5m0BhznrZ3weH11vxhE4xyEV6L8K1l0+9DqqL4Q +AbBrsxEOP+8m28xjAMoWv1O392MCAiyS5AZj4EkCgYEAiJcLGWi6wEW/bPoDmErj ++8zeZI2FT39bQ/G774egbWfgqWt03eN8lZErrmmTBmlvqT72CqjztHUc2xjY/PdI +dm0QMXmS5Ea5FXu6CvHkGs7NA/uvywZPtO1yFz/6vMbAiQfa11owjjiojq0IvMJk +XAImjvxI6tlASIAncii32vcCgYAl5zs0czogHc5FeC+LB6AaDsA0cX/RKx6mNNpf +x+CTKIP+QnaIq5rKVgyt6EfdfflgK7fOQ+5Njp7czwFqWDY5JhAoKlS2+UahrfP7 +wLMQVlBpVfKBEpiGj18oSvZZLN7fnOCwTfLn93wN6GG6qup+JEIKvSDBXkHpP03f +JcJX8QKBgQCOrE1t5aVU+5DeJTi8n4LqcliQfoi6IVPV3BQtPw4dIDxHM/nGnbyL +xZ//BaF7Z0Q9w2WwHeNLHDfzbzc7MsjfmX6ZrJfAKaGDJuBEgKWpHJ54rrVnUkHF +v4XtoF11aYUMq1keEUXhfczITRvPxdQ98gtmiAAv+TM/TI9OaRbSwQ== +-----END RSA PRIVATE KEY----- diff --git a/tools/compose/migagenthost2/demofiles/demokey.pub b/tools/compose/migagenthost2/demofiles/demokey.pub new file mode 100644 index 00000000..0f614a3a --- /dev/null +++ b/tools/compose/migagenthost2/demofiles/demokey.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+cFHq0ReptfZM1kG4x1dVIXeS46tl9YGRHLAFfjSlGIh0itqf6rxpmDC63tVKoYge4c2MpRGTS8JC7BLsw3evnwNVZU4w3eIGm70eVLJ7mkhwvZLAdw0ljjyRgWVi9+K4kiQch3G/AMnJkxKNvBYWxutahPPg69AJCtf+pYZ9nR/YdGUBXpWtEY9xdA0EaHw02tQyNsFWgYF6aLYJ/jIlof31J2ODtvZ9P28b8zJUnffqZaz+vdw1sfYD/VfOuSYckp0aO4Wk1A0uhkqQAlw4rCje/zTq60LfaMGkOEBqdqSlqmR9S64jrmL95EF7zahMOUhkx4mUA39kMLzCkn8j demo key diff --git a/tools/compose/migagenthost2/demofiles/samplefile1.txt b/tools/compose/migagenthost2/demofiles/samplefile1.txt new file mode 100644 index 00000000..4e8aa2fd --- /dev/null +++ b/tools/compose/migagenthost2/demofiles/samplefile1.txt @@ -0,0 +1 @@ +DEMOCONTENT diff --git a/tools/compose/migagenthost2/mig-agent.cfg b/tools/compose/migagenthost2/mig-agent.cfg new file mode 100644 index 00000000..1cba648f --- /dev/null +++ b/tools/compose/migagenthost2/mig-agent.cfg @@ -0,0 +1,28 @@ +[agent] + relay = "amqp://AGENTUSER:AGENTPASSWORD@MIGRELAYHOST:5672/mig" + api = "http://MIGAPIHOST:1664/api/v1/" + socket = "127.0.0.1:51664" + heartbeatfreq = "30s" + moduletimeout = "300s" + isimmortal = on + installservice = on + discoverpublicip = on + checkin = off + refreshenv = "5m" + extraprivacymode = off + nopersistmods = off + onlyVerifyPubKey = false + tags = "environment:migdemo" + tags = "build:docker" + +[stats] + maxactions = 15 + +[certs] + ca = "" + cert= "" + key = "" + +[logging] + mode = "stdout" + level = "debug" diff --git a/tools/compose/migagenthost2/mig-agent.conf b/tools/compose/migagenthost2/mig-agent.conf new file mode 100644 index 00000000..e6f595b8 --- /dev/null +++ b/tools/compose/migagenthost2/mig-agent.conf @@ -0,0 +1,4 @@ +[program:mig-agent] +command=/go/bin/mig-agent -d +startretries=9999 +autorestart=true diff --git a/tools/compose/migagenthost2/run.sh b/tools/compose/migagenthost2/run.sh new file mode 100644 index 00000000..dc612d9f --- /dev/null +++ b/tools/compose/migagenthost2/run.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# If AGENTMODULES is set, rebuild the agent with the indicated module tags +if [[ ! -z "$AGENTMODULES" ]]; then + sudo env GOPATH=/go \ + go install -tags "$AGENTMODULES" mig.ninja/mig/mig-agent +fi + +# Update API configuration using the environment +sudo sed -i "s/AGENTUSER/$AGENTUSER/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/AGENTPASSWORD/$AGENTPASSWORD/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGRELAYHOST/$MIGRELAYHOST/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGAPIHOST/$MIGAPIHOST/g" /etc/mig/mig-agent.cfg + +# If the environment indicates investigator generation is enabled, stage the key material +# and an ACL in the agents keyring/configuration; we also build the command line tools +# configuration using the same data. +if [[ $GENERATEINVESTIGATOR == "yes" ]]; then + while [[ ! -f /miginvestigator/fingerprint.txt ]]; do + sleep 1 + done + sudo mkdir -p /etc/mig/agentkeys + sudo cp /miginvestigator/pubkey.asc /etc/mig/agentkeys/pubkey.asc + keyid=`head -1 /miginvestigator/fingerprint.txt` + sudo sh -c "cat > /etc/mig/acl.cfg << EOF +{ + \"default\": { + \"minimumweight\": 1, + \"investigators\": { + \"mig\": { + \"fingerprint\": \"${keyid}\", + \"weight\": 1 + } + } + } +} +EOF +" + sudo cp -R /miginvestigator/mig ~/.mig + sudo chown -R $(whoami) ~/.mig + cat > ~/.migrc << EOF +[api] + url = "http://$MIGAPIHOST:1664/api/v1/" + skipverifycert = on +[gpg] + home = "$HOME/.mig/" + keyid = "$keyid" +[targets] + macro = all:status='online' +EOF +fi + +sudo /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -n diff --git a/tools/compose/migagenthost3/Dockerfile b/tools/compose/migagenthost3/Dockerfile new file mode 100644 index 00000000..5a02dab9 --- /dev/null +++ b/tools/compose/migagenthost3/Dockerfile @@ -0,0 +1,14 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +RUN sudo mkdir -p /etc/mig +COPY mig-agent.cfg /etc/mig/mig-agent.cfg +COPY audit.cfg /etc/mig/audit.cfg +COPY dispatch.cfg /etc/mig/dispatch.cfg +COPY audit.rules.json /etc/mig/audit.rules.json + +COPY mig-agent.conf /etc/supervisor/conf.d/mig-agent.conf + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migagenthost3/audit.cfg b/tools/compose/migagenthost3/audit.cfg new file mode 100644 index 00000000..b4caee4a --- /dev/null +++ b/tools/compose/migagenthost3/audit.cfg @@ -0,0 +1,4 @@ +[audit] +rulespath = /etc/mig/audit.rules.json +ratelimit = 500 +backloglimit = 500 diff --git a/tools/compose/migagenthost3/audit.rules.json b/tools/compose/migagenthost3/audit.rules.json new file mode 100644 index 00000000..ff5bbb62 --- /dev/null +++ b/tools/compose/migagenthost3/audit.rules.json @@ -0,0 +1,30 @@ +{ + "delete": true, + "enable": "1", + "buffer": "16348", + "rate": "500", + "audit_rules": [ + { + "strict_path_check": false, + "path": "/etc/passwd", + "key": "user", + "permission": "wa" + }, + { + "strict_path_check": false, + "path": "/etc/shadow", + "key": "user", + "permission": "wa" + }, + { + "syscalls": [ + "execve" + ], + "key": "exec", + "actions": [ + "exit", + "always" + ] + } + ] +} diff --git a/tools/compose/migagenthost3/dispatch.cfg b/tools/compose/migagenthost3/dispatch.cfg new file mode 100644 index 00000000..e445b446 --- /dev/null +++ b/tools/compose/migagenthost3/dispatch.cfg @@ -0,0 +1,3 @@ +[dispatch] +outputmozdef = no +httpurl = "http://migworkstation:2222/endpoint" diff --git a/tools/compose/migagenthost3/mig-agent.cfg b/tools/compose/migagenthost3/mig-agent.cfg new file mode 100644 index 00000000..1cba648f --- /dev/null +++ b/tools/compose/migagenthost3/mig-agent.cfg @@ -0,0 +1,28 @@ +[agent] + relay = "amqp://AGENTUSER:AGENTPASSWORD@MIGRELAYHOST:5672/mig" + api = "http://MIGAPIHOST:1664/api/v1/" + socket = "127.0.0.1:51664" + heartbeatfreq = "30s" + moduletimeout = "300s" + isimmortal = on + installservice = on + discoverpublicip = on + checkin = off + refreshenv = "5m" + extraprivacymode = off + nopersistmods = off + onlyVerifyPubKey = false + tags = "environment:migdemo" + tags = "build:docker" + +[stats] + maxactions = 15 + +[certs] + ca = "" + cert= "" + key = "" + +[logging] + mode = "stdout" + level = "debug" diff --git a/tools/compose/migagenthost3/mig-agent.conf b/tools/compose/migagenthost3/mig-agent.conf new file mode 100644 index 00000000..e6f595b8 --- /dev/null +++ b/tools/compose/migagenthost3/mig-agent.conf @@ -0,0 +1,4 @@ +[program:mig-agent] +command=/go/bin/mig-agent -d +startretries=9999 +autorestart=true diff --git a/tools/compose/migagenthost3/run.sh b/tools/compose/migagenthost3/run.sh new file mode 100644 index 00000000..dc612d9f --- /dev/null +++ b/tools/compose/migagenthost3/run.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# If AGENTMODULES is set, rebuild the agent with the indicated module tags +if [[ ! -z "$AGENTMODULES" ]]; then + sudo env GOPATH=/go \ + go install -tags "$AGENTMODULES" mig.ninja/mig/mig-agent +fi + +# Update API configuration using the environment +sudo sed -i "s/AGENTUSER/$AGENTUSER/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/AGENTPASSWORD/$AGENTPASSWORD/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGRELAYHOST/$MIGRELAYHOST/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGAPIHOST/$MIGAPIHOST/g" /etc/mig/mig-agent.cfg + +# If the environment indicates investigator generation is enabled, stage the key material +# and an ACL in the agents keyring/configuration; we also build the command line tools +# configuration using the same data. +if [[ $GENERATEINVESTIGATOR == "yes" ]]; then + while [[ ! -f /miginvestigator/fingerprint.txt ]]; do + sleep 1 + done + sudo mkdir -p /etc/mig/agentkeys + sudo cp /miginvestigator/pubkey.asc /etc/mig/agentkeys/pubkey.asc + keyid=`head -1 /miginvestigator/fingerprint.txt` + sudo sh -c "cat > /etc/mig/acl.cfg << EOF +{ + \"default\": { + \"minimumweight\": 1, + \"investigators\": { + \"mig\": { + \"fingerprint\": \"${keyid}\", + \"weight\": 1 + } + } + } +} +EOF +" + sudo cp -R /miginvestigator/mig ~/.mig + sudo chown -R $(whoami) ~/.mig + cat > ~/.migrc << EOF +[api] + url = "http://$MIGAPIHOST:1664/api/v1/" + skipverifycert = on +[gpg] + home = "$HOME/.mig/" + keyid = "$keyid" +[targets] + macro = all:status='online' +EOF +fi + +sudo /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -n diff --git a/tools/compose/migapi/Dockerfile b/tools/compose/migapi/Dockerfile new file mode 100644 index 00000000..f6f1e05e --- /dev/null +++ b/tools/compose/migapi/Dockerfile @@ -0,0 +1,11 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +RUN sudo mkdir -p /etc/mig +COPY api.cfg /etc/mig/api.cfg + +COPY mig-api.conf /etc/supervisor/conf.d/mig-api.conf + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migapi/api.cfg b/tools/compose/migapi/api.cfg new file mode 100644 index 00000000..b791efe9 --- /dev/null +++ b/tools/compose/migapi/api.cfg @@ -0,0 +1,27 @@ +; api configuration used in standalone docker demo mode + +[authentication] + enabled = on + tokenduration = 10m + +[manifest] + requiredsignatures = 1 + +[server] + ip = "0.0.0.0" + port = 1664 + host = "http://localhost:4444" + baseroute = "/api/v1" + clientpublicip = peer + +[postgres] + host = "MIGDBHOST" + port = 5432 + dbname = "mig" + user = "migapi" + password = "MIGDBAPIPASSWORD" + sslmode = "disable" + +[logging] + mode = "stdout" + level = "debug" diff --git a/tools/compose/migapi/mig-api.conf b/tools/compose/migapi/mig-api.conf new file mode 100644 index 00000000..e1201633 --- /dev/null +++ b/tools/compose/migapi/mig-api.conf @@ -0,0 +1,3 @@ +[program:mig-api] +command=/go/bin/mig-api +startretries=9999 diff --git a/tools/compose/migapi/run.sh b/tools/compose/migapi/run.sh new file mode 100644 index 00000000..e57922c2 --- /dev/null +++ b/tools/compose/migapi/run.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Update API configuration using the environment +sudo sed -i "s/MIGDBHOST/$MIGDBHOST/g" /etc/mig/api.cfg +sudo sed -i "s/MIGDBAPIPASSWORD/$MIGDBAPIPASSWORD/g" /etc/mig/api.cfg + +# If we have been asked to generate an investigator, do so and apply the new user +# to the database. +if [[ $GENERATEINVESTIGATOR == "yes" && ! -f /miginvestigator/fingerprint.txt ]]; then + mkdir -p ~/.mig + gpg --batch --no-default-keyring --keyring ~/.mig/pubring.gpg --secret-keyring \ + ~/.mig/secring.gpg --gen-key << EOF +Key-Type: 1 +Key-Length: 1024 +Subkey-Type: 1 +Subkey-Length: 1024 +Name-Real: $(whoami) Investigator +Name-Email: $(whoami)@localhost +Expire-Date: 12m +EOF + keyid=$(gpg --no-default-keyring --keyring ~/.mig/pubring.gpg \ + --secret-keyring ~/.mig/secring.gpg --fingerprint \ + --with-colons $(whoami)@localhost | grep '^fpr' | cut -f 10 -d ':') + cat > ~/.migrc << EOF +[api] + url = "http://localhost:1664/api/v1/" + skipverifycert = on +[gpg] + home = "$HOME/.mig/" + keyid = "$keyid" +[targets] + macro = all:status='online' +EOF + # Temporarily start the API up with API authentication disabled to add the initial + # investigator + sudo sed -i 's/enabled = on/enabled = off/' /etc/mig/api.cfg + sudo service supervisor start + gpg --no-default-keyring --keyring ~/.mig/pubring.gpg \ + --secret-keyring ~/.mig/secring.gpg \ + --export -a $(whoami)@localhost \ + > ~/.mig/$(whoami)-pubkey.asc + # Make sure the database is ready before we try this + while true; do + env PGHOST=$MIGDBHOST pg_isready + if [[ $? -eq 0 ]]; then + break + fi + sleep 1 + done + + echo -e "create investigator\n$(whoami)\nyes\nyes\nyes\nyes\n$HOME/.mig/$(whoami)-pubkey.asc\ny\n" | \ + /go/bin/mig-console -q + sudo service supervisor stop + sudo rm -f /var/run/supervisor.sock + sudo sed -i 's/enabled = off/enabled = on/' /etc/mig/api.cfg + # Populate /miginvestigator with the key material we created so other containers have access to it + sudo cp ~/.mig/$(whoami)-pubkey.asc /miginvestigator/pubkey.asc + sudo sh -c "echo $keyid >> /miginvestigator/fingerprint.txt" + sudo cp -R ~/.mig /miginvestigator/mig +fi + +sudo /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -n diff --git a/tools/compose/migdb/Dockerfile b/tools/compose/migdb/Dockerfile new file mode 100644 index 00000000..7463b1a9 --- /dev/null +++ b/tools/compose/migdb/Dockerfile @@ -0,0 +1,9 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +COPY build.sh /mig/build.sh +RUN bash /mig/build.sh + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migdb/build.sh b/tools/compose/migdb/build.sh new file mode 100644 index 00000000..f51c9a98 --- /dev/null +++ b/tools/compose/migdb/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Make sure we allow connections from external hosts +sudo sh -c 'echo "host all all samenet password" >> /etc/postgresql/9.5/main/pg_hba.conf' + +sudo service postgresql start || exit 1 + +# Add our users, just assign a random password during image build so we can add the schema +dbpass=$(cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c${1:-32}) +sudo -u postgres sh -c "psql -c 'CREATE ROLE migadmin;'" || exit 1 +sudo -u postgres sh -c "psql -c \"ALTER ROLE migadmin WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD '$dbpass';\"" || exit 1 + +sudo -u postgres sh -c "psql -c 'CREATE ROLE migapi;'" || exit 1 +sudo -u postgres sh -c "psql -c \"ALTER ROLE migapi WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD '$dbpass';\"" || exit 1 + +sudo -u postgres sh -c "psql -c 'CREATE ROLE migscheduler;'" || exit 1 +sudo -u postgres sh -c "psql -c \"ALTER ROLE migscheduler WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD '$dbpass';\"" || exit 1 + +# Add the schema +sudo -u postgres sh -c "psql -c 'CREATE DATABASE mig';" || exit 1 +sudo -u postgres sh -c "psql -f /go/src/mig.ninja/mig/database/schema.sql mig;" + +sudo service postgresql stop || exit 1 diff --git a/tools/compose/migdb/run.sh b/tools/compose/migdb/run.sh new file mode 100644 index 00000000..1e7a48eb --- /dev/null +++ b/tools/compose/migdb/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +sudo service postgresql start || exit 1 +while true; do + pg_isready + if [[ $? -eq 0 ]]; then + break + fi + sleep 1 +done +sudo -u postgres sh -c "psql -c \"ALTER ROLE migadmin PASSWORD '$MIGDBADMINPASSWORD';\"" || exit 1 +sudo -u postgres sh -c "psql -c \"ALTER ROLE migapi PASSWORD '$MIGDBAPIPASSWORD';\"" || exit 1 +sudo -u postgres sh -c "psql -c \"ALTER ROLE migscheduler PASSWORD '$MIGDBSCHEDULERPASSWORD';\"" || exit 1 +sudo service postgresql stop || exit 1 + +sudo -u postgres /usr/lib/postgresql/9.5/bin/postgres -D /var/lib/postgresql/9.5/main \ + -h '*' -c 'config_file=/etc/postgresql/9.5/main/postgresql.conf' diff --git a/tools/compose/migrelay/Dockerfile b/tools/compose/migrelay/Dockerfile new file mode 100644 index 00000000..7463b1a9 --- /dev/null +++ b/tools/compose/migrelay/Dockerfile @@ -0,0 +1,9 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +COPY build.sh /mig/build.sh +RUN bash /mig/build.sh + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migrelay/build.sh b/tools/compose/migrelay/build.sh new file mode 100644 index 00000000..cc15797e --- /dev/null +++ b/tools/compose/migrelay/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +echo 'NODENAME=rabbit@localhost' | sudo tee --append /etc/rabbitmq/rabbitmq-env.conf + +sudo service rabbitmq-server start + +mqpass=$(cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c${1:-32}) + +# Configure RabbitMQ with the users we want, and some random passwords for now in the image. Note we do +# not agent agent users here; these are created on image execution. +sudo rabbitmqctl add_user admin $mqpass || exit 1 +sudo rabbitmqctl set_user_tags admin administrator || exit 1 + +sudo rabbitmqctl delete_user guest || exit 1 + +sudo rabbitmqctl add_vhost mig || exit 1 + +sudo rabbitmqctl add_user scheduler $mqpass || exit 1 +sudo rabbitmqctl set_permissions -p mig scheduler \ + '^(toagents|toschedulers|toworkers|mig\.agt\..*)$' \ + '^(toagents|toworkers|mig\.agt\.(heartbeats|results))$' \ + '^(toagents|toschedulers|toworkers|mig\.agt\.(heartbeats|results))$' || exit 1 + +sudo rabbitmqctl add_user worker $mqpass || exit 1 +sudo rabbitmqctl set_permissions -p mig worker \ + '^migevent\..*$' \ + '^migevent(|\..*)$' \ + '^(toworkers|migevent\..*)$' + +sudo service rabbitmq-server stop diff --git a/tools/compose/migrelay/run.sh b/tools/compose/migrelay/run.sh new file mode 100644 index 00000000..fd2980a7 --- /dev/null +++ b/tools/compose/migrelay/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +sudo service rabbitmq-server start || exit 1 + + +sudo rabbitmqctl change_password admin $MIGRELAYADMINPASSWORD || exit 1 +sudo rabbitmqctl change_password scheduler $MIGRELAYSCHEDULERPASSWORD || exit 1 +sudo rabbitmqctl change_password worker $MIGRELAYWORKERPASSWORD || exit 1 + +for agent in $MIGRELAYAGENTS; do + username=`echo $agent | awk -F: '{print $1}'` + pw=`echo $agent | awk -F: '{print $2}'` + + # It's possible if the container was restarted the user could already exist, dont + # bail if the user add fails + sudo rabbitmqctl add_user $username $pw + sudo rabbitmqctl change_password $username $pw || exit 1 + + sudo rabbitmqctl set_permissions -p mig $username \ + '^mig\.agt\..*$' \ + '^(toschedulers|mig\.agt\..*)$' \ + '^(toagents|mig\.agt\..*)$' || exit 1 + +done + +sudo service rabbitmq-server stop || exit 1 + +sudo /usr/sbin/rabbitmq-server diff --git a/tools/compose/migscheduler/Dockerfile b/tools/compose/migscheduler/Dockerfile new file mode 100644 index 00000000..7aa2ffb2 --- /dev/null +++ b/tools/compose/migscheduler/Dockerfile @@ -0,0 +1,12 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +USER root +RUN mkdir -p /etc/mig +COPY scheduler.cfg /etc/mig/scheduler.cfg + +COPY mig-scheduler.conf /etc/supervisor/conf.d/mig-scheduler.conf + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migscheduler/build.sh b/tools/compose/migscheduler/build.sh new file mode 100644 index 00000000..cc15797e --- /dev/null +++ b/tools/compose/migscheduler/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +echo 'NODENAME=rabbit@localhost' | sudo tee --append /etc/rabbitmq/rabbitmq-env.conf + +sudo service rabbitmq-server start + +mqpass=$(cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c${1:-32}) + +# Configure RabbitMQ with the users we want, and some random passwords for now in the image. Note we do +# not agent agent users here; these are created on image execution. +sudo rabbitmqctl add_user admin $mqpass || exit 1 +sudo rabbitmqctl set_user_tags admin administrator || exit 1 + +sudo rabbitmqctl delete_user guest || exit 1 + +sudo rabbitmqctl add_vhost mig || exit 1 + +sudo rabbitmqctl add_user scheduler $mqpass || exit 1 +sudo rabbitmqctl set_permissions -p mig scheduler \ + '^(toagents|toschedulers|toworkers|mig\.agt\..*)$' \ + '^(toagents|toworkers|mig\.agt\.(heartbeats|results))$' \ + '^(toagents|toschedulers|toworkers|mig\.agt\.(heartbeats|results))$' || exit 1 + +sudo rabbitmqctl add_user worker $mqpass || exit 1 +sudo rabbitmqctl set_permissions -p mig worker \ + '^migevent\..*$' \ + '^migevent(|\..*)$' \ + '^(toworkers|migevent\..*)$' + +sudo service rabbitmq-server stop diff --git a/tools/compose/migscheduler/mig-scheduler.conf b/tools/compose/migscheduler/mig-scheduler.conf new file mode 100644 index 00000000..bba3997f --- /dev/null +++ b/tools/compose/migscheduler/mig-scheduler.conf @@ -0,0 +1,4 @@ +[program:mig-scheduler] +command=/go/bin/mig-scheduler +startretries=9999 +autorestart=true diff --git a/tools/compose/migscheduler/run.sh b/tools/compose/migscheduler/run.sh new file mode 100644 index 00000000..3d16444e --- /dev/null +++ b/tools/compose/migscheduler/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Update scheduler configuration using the environment +sed -i "s/MIGDBHOST/$MIGDBHOST/g" /etc/mig/scheduler.cfg +sed -i "s/MIGDBSCHEDULERPASSWORD/$MIGDBSCHEDULERPASSWORD/g" /etc/mig/scheduler.cfg +sed -i "s/MIGRELAYHOST/$MIGRELAYHOST/g" /etc/mig/scheduler.cfg +sed -i "s/MIGRELAYSCHEDULERPASSWORD/$MIGRELAYSCHEDULERPASSWORD/g" /etc/mig/scheduler.cfg + +/usr/bin/supervisord -c /etc/supervisor/supervisord.conf -n diff --git a/tools/compose/migscheduler/scheduler.cfg b/tools/compose/migscheduler/scheduler.cfg new file mode 100644 index 00000000..b38d6679 --- /dev/null +++ b/tools/compose/migscheduler/scheduler.cfg @@ -0,0 +1,39 @@ +[agent] + timeout = "60s" + heartbeatfreq = "30s" + whitelist = "" + detectmultiagents = true + killdupagents = true + logactions = true + +[collector] + freq = "1s" + +[periodic] + freq = "3s" + deleteafter = "72h" + queuescleanupfreq = "24h" + +[directories] + spool = "/var/cache/mig/" + tmp = "/var/tmp/" + +[postgres] + host = "MIGDBHOST" + port = 5432 + dbname = "mig" + user = "migscheduler" + password = "MIGDBSCHEDULERPASSWORD" + sslmode = "disable" + maxconn = 10 + +[mq] + host = "MIGRELAYHOST" + port = 5672 + user = "scheduler" + pass = "MIGRELAYSCHEDULERPASSWORD" + vhost = "mig" + +[logging] + mode = "stdout" + level = "debug" diff --git a/tools/compose/migworkstation/Dockerfile b/tools/compose/migworkstation/Dockerfile new file mode 100644 index 00000000..e0f875ff --- /dev/null +++ b/tools/compose/migworkstation/Dockerfile @@ -0,0 +1,14 @@ +FROM mozilla/mig:latest + +MAINTAINER Mozilla + +RUN sudo mkdir -p /etc/mig +COPY mig-agent.cfg /etc/mig/mig-agent.cfg + +COPY mig-agent.conf /etc/supervisor/conf.d/mig-agent.conf + +COPY simpleweb.py /mig/simpleweb.py +COPY simpleweb.conf /etc/supervisor/conf.d/simpleweb.conf + +COPY run.sh /mig/run.sh +CMD bash /mig/run.sh diff --git a/tools/compose/migworkstation/mig-agent.cfg b/tools/compose/migworkstation/mig-agent.cfg new file mode 100644 index 00000000..1cba648f --- /dev/null +++ b/tools/compose/migworkstation/mig-agent.cfg @@ -0,0 +1,28 @@ +[agent] + relay = "amqp://AGENTUSER:AGENTPASSWORD@MIGRELAYHOST:5672/mig" + api = "http://MIGAPIHOST:1664/api/v1/" + socket = "127.0.0.1:51664" + heartbeatfreq = "30s" + moduletimeout = "300s" + isimmortal = on + installservice = on + discoverpublicip = on + checkin = off + refreshenv = "5m" + extraprivacymode = off + nopersistmods = off + onlyVerifyPubKey = false + tags = "environment:migdemo" + tags = "build:docker" + +[stats] + maxactions = 15 + +[certs] + ca = "" + cert= "" + key = "" + +[logging] + mode = "stdout" + level = "debug" diff --git a/tools/compose/migworkstation/mig-agent.conf b/tools/compose/migworkstation/mig-agent.conf new file mode 100644 index 00000000..e6f595b8 --- /dev/null +++ b/tools/compose/migworkstation/mig-agent.conf @@ -0,0 +1,4 @@ +[program:mig-agent] +command=/go/bin/mig-agent -d +startretries=9999 +autorestart=true diff --git a/tools/compose/migworkstation/run.sh b/tools/compose/migworkstation/run.sh new file mode 100644 index 00000000..aa4e1525 --- /dev/null +++ b/tools/compose/migworkstation/run.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# If AGENTMODULES is set, rebuild the agent with the indicated module tags +if [[ ! -z "$AGENTMODULES" ]]; then + sudo env GOPATH=/go \ + go install -tags "$AGENTMODULES" mig.ninja/mig/mig-agent +fi + +# If CLIENTMODULES is set, rebuild the clients with the indicated module tags +if [[ ! -z "$CLIENTMODULES" ]]; then + sudo env GOPATH=/go \ + go install -tags "$CLIENTMODULES" mig.ninja/mig/client/mig-console + sudo env GOPATH=/go \ + go install -tags "$CLIENTMODULES" mig.ninja/mig/client/mig +fi + +# Update API configuration using the environment +sudo sed -i "s/AGENTUSER/$AGENTUSER/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/AGENTPASSWORD/$AGENTPASSWORD/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGRELAYHOST/$MIGRELAYHOST/g" /etc/mig/mig-agent.cfg +sudo sed -i "s/MIGAPIHOST/$MIGAPIHOST/g" /etc/mig/mig-agent.cfg + +# If the environment indicates investigator generation is enabled, stage the key material +# and an ACL in the agents keyring/configuration; we also build the command line tools +# configuration using the same data. +if [[ $GENERATEINVESTIGATOR == "yes" ]]; then + while [[ ! -f /miginvestigator/fingerprint.txt ]]; do + sleep 1 + done + sudo mkdir -p /etc/mig/agentkeys + sudo cp /miginvestigator/pubkey.asc /etc/mig/agentkeys/pubkey.asc + keyid=`head -1 /miginvestigator/fingerprint.txt` + sudo sh -c "cat > /etc/mig/acl.cfg << EOF +{ + \"default\": { + \"minimumweight\": 1, + \"investigators\": { + \"mig\": { + \"fingerprint\": \"${keyid}\", + \"weight\": 1 + } + } + } +} +EOF +" + sudo rm -rf ~/.mig + sudo cp -R /miginvestigator/mig ~/.mig + sudo chown -R $(whoami) ~/.mig + cat > ~/.migrc << EOF +[api] + url = "http://$MIGAPIHOST:1664/api/v1/" + skipverifycert = on +[gpg] + home = "$HOME/.mig/" + keyid = "$keyid" +[targets] + macro = all:status='online' +EOF +fi + +sudo /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -n diff --git a/tools/compose/migworkstation/simpleweb.conf b/tools/compose/migworkstation/simpleweb.conf new file mode 100644 index 00000000..14ded0a5 --- /dev/null +++ b/tools/compose/migworkstation/simpleweb.conf @@ -0,0 +1,4 @@ +[program:simpleweb] +command=/mig/simpleweb.py 2222 +startretries=9999 +autorestart=true diff --git a/tools/compose/migworkstation/simpleweb.py b/tools/compose/migworkstation/simpleweb.py new file mode 100755 index 00000000..6fe6a257 --- /dev/null +++ b/tools/compose/migworkstation/simpleweb.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import sys +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + +class S(BaseHTTPRequestHandler): + def do_POST(self): + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.end_headers() + con_length = int(self.headers['Content-Length']) + data = self.rfile.read(con_length) + print data + self.wfile.write('accepted\n') + +def run(server_class=HTTPServer, handler_class=S, port=8080): + server_address = ('', port) + httpd = server_class(server_address, handler_class) + print 'Starting httpd...' + httpd.serve_forever() + +if __name__ == "__main__": + from sys import argv + + if len(argv) == 2: + run(port=int(argv[1])) + else: + run() + +sys.exit(0) diff --git a/tools/docker_install.sh b/tools/docker_install.sh deleted file mode 100644 index 60f97fa7..00000000 --- a/tools/docker_install.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -fail() { - echo configuration failed - exit 1 -} - -go install mig.ninja/mig/mig-scheduler || fail -go install mig.ninja/mig/mig-api || fail -go install -tags 'modmemory' mig.ninja/mig/client/mig || fail -go install -tags 'modmemory' mig.ninja/mig/client/mig-console || fail -go install -tags 'modmemory' mig.ninja/mig/mig-agent || fail - -sudo sh -c "echo 'host all all 127.0.0.1/32 password' >> /etc/postgresql/9.5/main/pg_hba.conf" -sudo service postgresql restart || fail -dbpass=$(cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c${1:-32}) - -for user in 'migadmin' 'migapi' 'migscheduler'; do - sudo -u postgres sh -c "psql -c 'CREATE ROLE $user;'" || fail - sudo -u postgres sh -c "psql -c \"ALTER ROLE $user WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD '$dbpass';\"" || fail -done -sudo -u postgres sh -c "psql -c 'CREATE DATABASE mig';" || fail -sudo -u postgres sh -c "psql -f /go/src/mig.ninja/mig/database/schema.sql mig;" - -sudo sh -c "cat > /etc/supervisor/conf.d/mig-scheduler.inactive << EOF -[program:mig-scheduler] -command=/go/bin/mig-scheduler -startretries=20 -EOF" - -sudo sh -c "cat > /etc/supervisor/conf.d/mig-api.inactive << EOF -[program:mig-api] -command=/go/bin/mig-api -startretries=20 -EOF" - -sudo sh -c "cat > /etc/supervisor/conf.d/mig-agent.inactive << EOF -[program:mig-agent] -command=/go/bin/mig-agent -d -startretries=20 -EOF" - -echo 'NODENAME=rabbit@localhost' | sudo tee --append /etc/rabbitmq/rabbitmq-env.conf -sudo service rabbitmq-server start -mqpass=$(cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c${1:-32}) - -sudo rabbitmqctl add_user admin $mqpass || fail -sudo rabbitmqctl set_user_tags admin administrator || fail - -sudo rabbitmqctl delete_user guest || fail - -sudo rabbitmqctl add_vhost mig || fail -sudo rabbitmqctl list_vhosts || fail - -sudo rabbitmqctl add_user scheduler $mqpass || fail -sudo rabbitmqctl set_permissions -p mig scheduler \ - '^(toagents|toschedulers|toworkers|mig\.agt\..*)$' \ - '^(toagents|toworkers|mig\.agt\.(heartbeats|results))$' \ - '^(toagents|toschedulers|toworkers|mig\.agt\.(heartbeats|results))$' || fail - -sudo rabbitmqctl add_user agent $mqpass || fail -sudo rabbitmqctl set_permissions -p mig agent \ - '^mig\.agt\..*$' \ - '^(toschedulers|mig\.agt\..*)$' \ - '^(toagents|mig\.agt\..*)$' || fail - -sudo rabbitmqctl add_user worker $mqpass || fail -sudo rabbitmqctl set_permissions -p mig worker \ - '^migevent\..*$' \ - '^migevent(|\..*)$' \ - '^(toworkers|migevent\..*)$' - -sudo service rabbitmq-server stop -sudo service postgresql stop - -sudo mkdir -p /etc/mig || fail -sudo sh -c "cat /go/src/mig.ninja/mig/tools/api.cfg.demo | \ - sed \"s,APIPASS,${dbpass},\" > /etc/mig/api.cfg.demo" -sudo sh -c "cat /go/src/mig.ninja/mig/tools/scheduler.cfg.demo | \ - sed \"s,SCHEDULERDBPASS,${dbpass},\" | \ - sed \"s,SCHEDULERMQPASS,${mqpass},\" > /etc/mig/scheduler.cfg.demo" -sudo sh -c "cat /go/src/mig.ninja/mig/tools/mig-agent.cfg.demo | \ - sed \"s,AGENTPASS,${mqpass},\" > /etc/mig/mig-agent.cfg.demo" - diff --git a/tools/docker_start.sh b/tools/docker_start.sh index 637ce1b4..957bf86e 100644 --- a/tools/docker_start.sh +++ b/tools/docker_start.sh @@ -1,6 +1,102 @@ #!/usr/bin/env bash -standalone_services() { +# Default entry point for standard MIG docker image. When run with no specific environment +# variables, this configures the docker image to run as a standalone MIG demo. If MIGMODE +# is set to test, the docker image is used to execute integration tests. + +fail() { + echo configuration failed + exit 1 +} + +# Configure the docker container for standalone execution of a MIG demo environment. +standalone_configure() { + echo Performing initial container configuration... + + go install mig.ninja/mig/mig-scheduler || fail + go install mig.ninja/mig/mig-api || fail + go install -tags 'modmemory' mig.ninja/mig/client/mig || fail + go install -tags 'modmemory' mig.ninja/mig/client/mig-console || fail + go install -tags 'modmemory' mig.ninja/mig/mig-agent || fail + + sudo sh -c "echo 'host all all 127.0.0.1/32 password' >> /etc/postgresql/9.5/main/pg_hba.conf" + sudo service postgresql restart || fail + dbpass=$(cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c${1:-32}) + + for user in 'migadmin' 'migapi' 'migscheduler'; do + sudo -u postgres sh -c "psql -c 'CREATE ROLE $user;'" || fail + sudo -u postgres sh -c "psql -c \"ALTER ROLE $user WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD '$dbpass';\"" || fail + done + sudo -u postgres sh -c "psql -c 'CREATE DATABASE mig';" || fail + sudo -u postgres sh -c "psql -f /go/src/mig.ninja/mig/database/schema.sql mig;" + + sudo sh -c "cat > /etc/supervisor/conf.d/mig-scheduler.inactive << EOF +[program:mig-scheduler] +command=/go/bin/mig-scheduler +startretries=9999 +autorestart=true +EOF" + + sudo sh -c "cat > /etc/supervisor/conf.d/mig-api.inactive << EOF +[program:mig-api] +command=/go/bin/mig-api +startretries=9999 +autorestart=true +EOF" + + sudo sh -c "cat > /etc/supervisor/conf.d/mig-agent.inactive << EOF +[program:mig-agent] +command=/go/bin/mig-agent -d +startretries=9999 +autorestart=true +EOF" + + echo 'NODENAME=rabbit@localhost' | sudo tee --append /etc/rabbitmq/rabbitmq-env.conf + sudo service rabbitmq-server start + mqpass=$(cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c${1:-32}) + + sudo rabbitmqctl add_user admin $mqpass || fail + sudo rabbitmqctl set_user_tags admin administrator || fail + + sudo rabbitmqctl delete_user guest || fail + + sudo rabbitmqctl add_vhost mig || fail + sudo rabbitmqctl list_vhosts || fail + + sudo rabbitmqctl add_user scheduler $mqpass || fail + sudo rabbitmqctl set_permissions -p mig scheduler \ + '^(toagents|toschedulers|toworkers|mig\.agt\..*)$' \ + '^(toagents|toworkers|mig\.agt\.(heartbeats|results))$' \ + '^(toagents|toschedulers|toworkers|mig\.agt\.(heartbeats|results))$' || fail + + sudo rabbitmqctl add_user agent $mqpass || fail + sudo rabbitmqctl set_permissions -p mig agent \ + '^mig\.agt\..*$' \ + '^(toschedulers|mig\.agt\..*)$' \ + '^(toagents|mig\.agt\..*)$' || fail + + sudo rabbitmqctl add_user worker $mqpass || fail + sudo rabbitmqctl set_permissions -p mig worker \ + '^migevent\..*$' \ + '^migevent(|\..*)$' \ + '^(toworkers|migevent\..*)$' + + sudo service rabbitmq-server stop + sudo service postgresql stop + + sudo mkdir -p /etc/mig || fail + sudo sh -c "cat /go/src/mig.ninja/mig/tools/api.cfg.demo | \ + sed \"s,APIPASS,${dbpass},\" > /etc/mig/api.cfg.demo" + sudo sh -c "cat /go/src/mig.ninja/mig/tools/scheduler.cfg.demo | \ + sed \"s,SCHEDULERDBPASS,${dbpass},\" | \ + sed \"s,SCHEDULERMQPASS,${mqpass},\" > /etc/mig/scheduler.cfg.demo" + sudo sh -c "cat /go/src/mig.ninja/mig/tools/mig-agent.cfg.demo | \ + sed \"s,AGENTPASS,${mqpass},\" > /etc/mig/mig-agent.cfg.demo" +} + +# Generate a demonistration investigator and associated key material and configure the +# standalone docker image to make use of it. +standalone_userconfig() { sudo service rabbitmq-server restart sudo service postgresql restart @@ -74,19 +170,48 @@ EOF sudo service supervisor start } +# Start integration tests. start_test() { - standalone_services - # Sleep for a few seconds to give the agent time to register - sleep 10 + # Sleep a number of seconds to give the agent time to register before we run the + # test, the heartbeat interval is 30 seconds so 45 should be sufficient + sleep 45 mig -i /go/src/mig.ninja/mig/actions/integration_tests.json || exit 1 } +# Start demo environment, just spawns a shell. start_demo() { - standalone_services bash } PATH=/go/bin:$PATH; export PATH +GOPATH=/go; export GOPATH + +if [[ ! -f /.migconfigured ]]; then + # The container hasn't been configured with a standalone configuration yet, apply + # the configuration and note it as having completed. + standalone_configure + standalone_userconfig + sudo touch /.migconfigured +fi + +sudo service rabbitmq-server stop +sudo service postgresql stop +sudo service supervisor stop +sudo mkdir -p /var/run/rabbitmq +sudo chown rabbitmq:rabbitmq /var/run/rabbitmq +sudo rm -f /var/run/supervisor.sock /var/run/supervisord.pid +sudo service rabbitmq-server restart +sudo service postgresql start +echo Waiting for Postgres to be ready... +while true; do + pg_isready + if [[ $? -eq 0 ]]; then + break + fi + sleep 1 +done +sudo service supervisor start + if [[ $MIGMODE = "test" ]]; then start_test else diff --git a/tools/mig-agent.cfg.demo b/tools/mig-agent.cfg.demo index 3de6869b..463b104c 100644 --- a/tools/mig-agent.cfg.demo +++ b/tools/mig-agent.cfg.demo @@ -4,7 +4,7 @@ relay = "amqp://agent:AGENTPASS@localhost:5672/mig" api = "http://localhost:1664/api/v1/" socket = "127.0.0.1:51664" - heartbeatfreq = "5s" + heartbeatfreq = "30s" moduletimeout = "300s" isimmortal = on installservice = on diff --git a/tools/scheduler.cfg.demo b/tools/scheduler.cfg.demo index 6345bff3..dcbe41fa 100644 --- a/tools/scheduler.cfg.demo +++ b/tools/scheduler.cfg.demo @@ -1,8 +1,8 @@ ; scheduler configuration file used in standalone docker demo mode [agent] - timeout = "20m" - heartbeatfreq = "5m" + timeout = "60m" + heartbeatfreq = "30s" whitelist = "" detectmultiagents = true killdupagents = true