mozilla
/
mig
зеркало из https://github.com/mozilla/mig.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[medium] add tags.Team to asset hints and compliance items

This commit is contained in:
Julien Vehent 2015-05-01 18:48:06 -04:00
Родитель 54be9b37da
Коммит b535a206d1
7 изменённых файлов: 217 добавлений и 156 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
Makefile
Просмотреть файл

@ -295,9 +295,9 @@ worker-agent-verif:
$(MKDIR) -p $(BINDIR)
$(GO) build $(GOOPTS) -o $(BINDIR)/mig_agent_verif_worker $(GOLDFLAGS) mig/workers/agent_verif
worker-mozdef-asset:
worker-agent-intel:
$(MKDIR) -p $(BINDIR)
$(GO) build $(GOOPTS) -o $(BINDIR)/mig_mozdef_asset_worker $(GOLDFLAGS) mig/workers/mozdef_asset
$(GO) build $(GOOPTS) -o $(BINDIR)/mig_agent_intel_worker $(GOLDFLAGS) mig/workers/agent_intel
worker-compliance-item:
$(MKDIR) -p $(BINDIR)

90
doc/worker_agent_intel.html Normal file
Просмотреть файл

@ -0,0 +1,90 @@
<!DOCTYPE html>
<html>
<head>
<title>Mozilla InvestiGator: Agent Intel Worker</title>
<meta charset="utf-8" />
<link href="docstyle.css" rel="stylesheet" />
<meta content="Julien Vehent &lt;jvehent@mozilla.com&gt;" name="author" />
</head>
<body>
<h1>Mozilla InvestiGator: Agent Intel Worker</h1>
<aside class="topic contents" id="table-of-contents">
<h1>Table of Contents</h1>
<ul class="auto-toc">
<li><a href="#configuration">1   Configuration</a>
<ul class="auto-toc">
<li><a href="#upstart">1.1   Upstart</a></li>
</ul>
</li>
</ul>
</aside>
<p>The Agent Intel Worker in a separate program that listens for event about agents that newly joined the platform, and create asset hints that are published to MozDef. This worker serves a very specific purpose in the collection of asset data performed by Mozilla OpSec. It may not be very useful to anyone else.</p>
<section id="configuration">
<h2>1   Configuration</h2>
<p>This worker retrieves agents hearbeats in the MIG Agent format from the MIG Relay, transforms them into Asset Hints, and publishes them to some other rabbitmq endpoint when MozDef will retrieve them.</p>
<pre class="code">; mozdef rabbitmq endpoint
[mozdef]
host = "mozdef.rabbitmq.example.net"
port = 5671
user = "migassetworker"
pass = "secretpassphrase"
vhost = "mozdef"
exchange = "eventtask"
routingkey = "eventtask"
usetls = true
cacertpath = "/etc/certs/ca.crt"
clientcertpath = "/etc/certs/migassetworker.crt"
clientkeypath = "/etc/certs/migassetworker.key"
timeout = "10s"
; https://github.com/ameihm0912/vmintgr
[vmintgr]
bin = "/opt/vmintgr/bin/grouptest"
; mig rabbitmq endpoint
[mq]
host = "hostname.mig.relay.example.net"
port = 5671
user = "migworker"
pass = "somepassphrase"
vhost = "mig"
usetls = true
cacert = "/path/to/ca.crt"
tlscert = "/path/to/client.crt"
tlskey = "/path/to/client.key"
timeout = "10s"
[logging]
mode = "syslog" ; stdout | file | syslog
level = "info" ; debug | info | warning | error | critical
host = "localhost"
port = 514
protocol = "udp"</pre>
<section id="upstart">
<h3>1.1   Upstart</h3>
<pre class="code"># Mozilla InvestiGator Agent Intel Worker
description "MIG Agent Intel Worker"
start on filesystem or runlevel [2345]
stop on runlevel [!2345]
setuid mig
limit nofile 640000 640000
respawn
respawn limit 10 5
umask 022
console none
pre-start script
test /opt/mig_agent_intel_worker || { stop; exit 0; }
end script
# Start
exec /opt/mig_agent_intel_worker</pre>
</section>
</section>
</body>
</html>

18
doc/worker_compliance_item.html
Просмотреть файл

@ -1,9 +1,9 @@
<!DOCTYPE html>
<html>
<head>
<title>Mozilla InvestiGator: MozDef Compliance Item Worker</title>
<meta charset="utf-8" />
<link href="docstyle.css" rel="stylesheet" />
<title>Mozilla InvestiGator: MozDef Compliance Item Worker</title>
<meta content="Julien Vehent &lt;jvehent@mozilla.com&gt;" name="author" />
</head>
<body>
@ -11,8 +11,7 @@
<aside class="topic contents" id="table-of-contents">
<h1>Table of Contents</h1>
<ul class="auto-toc">
<li>
<p><a href="#configuration">1   Configuration</a></p>
<li><a href="#configuration">1   Configuration</a>
<ul class="auto-toc">
<li><a href="#upstart">1.1   Upstart</a></li>
</ul>
@ -23,7 +22,7 @@
<section id="configuration">
<h2>1   Configuration</h2>
<p>The configuration needs standard access to MIG's rabbitmq relay, configured in the <cite>[mq]</cite> section. Access to MozDef's rabbitmq endpoint is configured in the <cite>[mozdef]</cite> section. The <cite>[api]</cite> section must only contain the location of MIG's API endpoint, used to build links to commands in compliance items. Standard logging can be configured in <cite>[logging]</cite>.</p>
<pre><code class="">[api]
<pre class="code">[api]
host = "https://api.mig.example.net"
[mozdef]
host = "mozdef.rabbitmq.example.net"
@ -38,6 +37,11 @@
clientcertpath = "/etc/certs/migcomplianceworker.crt"
clientkeypath = "/etc/certs/migcomplianceworker.key"
timeout = "10s"
; https://github.com/ameihm0912/vmintgr
[vmintgr]
bin = "/opt/vmintgr/bin/grouptest"
[mq]
host = "relay.mig.example.net"
port = 5671
@ -54,11 +58,11 @@
level = "info"
host = "localhost"
port = 514
protocol = "udp"</code></pre>
protocol = "udp"</pre>
<section id="upstart">
<h3>1.1   Upstart</h3>
<p>To manage this worker with upstart, use the configuration below, for example in <cite>/etc/init/mig-compliance-item-worker.conf</cite>.</p>
<pre><code class=""># Mozilla InvestiGator MozDef Compliance Item Worker
<pre class="code"># Mozilla InvestiGator MozDef Compliance Item Worker
description "MIG MozDef Compliance Item Worker"
@ -79,7 +83,7 @@ pre-start script
end script
# Start
exec /opt/mig_compliance_item_worker</code></pre>
exec /opt/mig_compliance_item_worker</pre>
</section>
</section>
</body>

20
src/mig/workers/mozdef_asset/doc.rst → src/mig/workers/agent_intel/doc.rst
Просмотреть файл

@ -1,12 +1,12 @@
=========================================
Mozilla InvestiGator: MozDef Asset Worker
=========================================
========================================
Mozilla InvestiGator: Agent Intel Worker
========================================
:Author: Julien Vehent <jvehent@mozilla.com>
.. sectnum::
.. contents:: Table of Contents
The MozDef Asset Worker in a separate program that listens for event about
The Agent Intel Worker in a separate program that listens for event about
agents that newly joined the platform, and create asset hints that are
published to MozDef. This worker serves a very specific purpose in the
collection of asset data performed by Mozilla OpSec. It may not be very useful
@ -37,6 +37,10 @@ rabbitmq endpoint when MozDef will retrieve them.
clientkeypath = "/etc/certs/migassetworker.key"
timeout = "10s"
; https://github.com/ameihm0912/vmintgr
[vmintgr]
bin = "/opt/vmintgr/bin/grouptest"
; mig rabbitmq endpoint
[mq]
host = "hostname.mig.relay.example.net"
@ -62,9 +66,9 @@ Upstart
.. code::
# Mozilla InvestiGator MozDef Asset Worker
# Mozilla InvestiGator Agent Intel Worker
description "MIG MozDef Asset Worker"
description "MIG Agent Intel Worker"
start on filesystem or runlevel [2345]
stop on runlevel [!2345]
@ -79,8 +83,8 @@ Upstart
console none
pre-start script
test /opt/mig_mozdef_asset_worker || { stop; exit 0; }
test /opt/mig_agent_intel_worker || { stop; exit 0; }
end script
# Start
exec /opt/mig_mozdef_asset_worker
exec /opt/mig_agent_intel_worker

59
src/mig/workers/mozdef_asset/main.go → src/mig/workers/agent_intel/main.go
Просмотреть файл

@ -15,15 +15,21 @@ import (
"mig/event"
"mig/workers"
"os"
"os/exec"
"regexp"
)
const workerName = "mozdef_asset"
const workerName = "agent_intel"
type Config struct {
Mq workers.MqConf
MozDef gozdef.MqConf
Logging mig.Logging
Vmintgr vmintgrconf
}
type vmintgrconf struct {
Bin string
}
func main() {
@ -36,7 +42,7 @@ func main() {
fmt.Fprintf(os.Stderr, "%s - a worker that listens to new endpoints and sends them as assets to mozdef\n", os.Args[0])
flag.PrintDefaults()
}
var configPath = flag.String("c", "/etc/mig/mozdef_asset_worker.cfg", "Load configuration from file")
var configPath = flag.String("c", "/etc/mig/agent_intel_worker.cfg", "Load configuration from file")
flag.Parse()
err = gcfg.ReadFileInto(&conf, *configPath)
if err != nil {
@ -67,10 +73,16 @@ func main() {
err = json.Unmarshal(event.Body, &agt)
if err != nil {
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("invalid agent description: %v", err)}.Err())
continue
}
agt, err = populateTeam(agt, conf)
if err != nil {
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("failed to populate agent team: %v", err)}.Err())
}
hint, err = makeHintFromAgent(agt)
if err != nil {
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("failed to build asset hint: %v", err)}.Err())
continue
}
err = publishHintToMozdef(hint, gp)
if err != nil {
@ -83,6 +95,46 @@ func main() {
return
}
type VmintgrOutput struct {
Host string `json:"host"`
Ip string `json:"ip"`
Team string `json:"team"`
}
func populateTeam(orig_agt mig.Agent, conf Config) (agt mig.Agent, err error) {
agt = orig_agt
var (
out []byte
vmout VmintgrOutput
query string
)
if conf.Vmintgr.Bin == "" {
return agt, fmt.Errorf("vmintgr is not configured")
}
for i := 0; i <= len(agt.Env.Addresses); i++ {
switch i {
case 0:
query = "host:" + agt.Name
default:
query = "ip:" + agt.Env.Addresses[i-1]
}
out, err = exec.Command(conf.Vmintgr.Bin, query).Output()
if err != nil {
return
}
err = json.Unmarshal(out, &vmout)
if err != nil {
return
}
if vmout.Team != "default" {
goto finish
}
}
finish:
agt.Tags.(map[string]interface{})["team"] = vmout.Team
return
}
func makeHintFromAgent(agt mig.Agent) (hint gozdef.HostAssetHint, err error) {
hint.Type = "host"
hint.Name = agt.Name
@ -102,6 +154,9 @@ func makeHintFromAgent(agt mig.Agent) (hint gozdef.HostAssetHint, err error) {
if _, ok := agt.Tags.(map[string]interface{})["operator"]; ok {
hint.Operator = agt.Tags.(map[string]interface{})["operator"].(string)
}
if _, ok := agt.Tags.(map[string]interface{})["team"]; ok {
hint.Team = agt.Tags.(map[string]interface{})["team"].(string)
}
return
}

50
src/mig/workers/compliance_item/main.go
Просмотреть файл

@ -17,6 +17,7 @@ import (
"mig/modules/file"
"mig/workers"
"os"
"os/exec"
"regexp"
"time"
)
@ -30,6 +31,9 @@ type Config struct {
API struct {
Host string
}
Vmintgr struct {
Bin string
}
}
func main() {
@ -123,12 +127,17 @@ func makeComplianceItem(cmd mig.Command, conf Config) (items []gozdef.Compliance
ci.Check.Description = cmd.Action.Name
ci.Link = fmt.Sprintf("%s/command?commandid=%.0f", conf.API.Host, cmd.ID)
if cmd.Agent.Tags != nil {
operator := ""
if _, ok := cmd.Agent.Tags.(map[string]interface{})["operator"]; ok {
ci.Tags = struct {
Operator string `json:"operator"`
}{
Operator: cmd.Agent.Tags.(map[string]interface{})["operator"].(string),
}
operator = cmd.Agent.Tags.(map[string]interface{})["operator"].(string)
}
team := getTeam(cmd.Agent, conf)
ci.Tags = struct {
Operator string `json:"operator"`
Team string `json:"team"`
}{
Operator: operator,
Team: team,
}
}
for i, result := range cmd.Results {
@ -260,3 +269,34 @@ func makeComplianceItem(cmd mig.Command, conf Config) (items []gozdef.Compliance
}
return
}
type VmintgrOutput struct {
Host string `json:"host"`
Ip string `json:"ip"`
Team string `json:"team"`
}
func getTeam(agt mig.Agent, conf Config) string {
var vmout VmintgrOutput
if conf.Vmintgr.Bin == "" {
return ""
}
for i := 0; i <= len(agt.Env.Addresses); i++ {
query := "host:" + agt.Name
if i > 0 {
query = "ip:" + agt.Env.Addresses[i-1]
}
out, err := exec.Command(conf.Vmintgr.Bin, query).Output()
if err != nil {
return ""
}
err = json.Unmarshal(out, &vmout)
if err != nil {
return ""
}
if vmout.Team != "default" {
return vmout.Team
}
}
return "default"
}

132
src/mig/workers/mozdef_asset/qa/main.go
Просмотреть файл

@ -1,132 +0,0 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main
import (
"code.google.com/p/gcfg"
"encoding/json"
"flag"
"fmt"
"github.com/jvehent/gozdef"
"mig"
"mig/event"
"mig/workers"
"os"
"regexp"
)
const workerName = "mozdef_asset_qa"
type Config struct {
Mq workers.MqConf
MozDef gozdef.MqConf
Logging mig.Logging
}
func main() {
var (
err error
conf Config
hint gozdef.HostAssetHint
evctr int
ctrmod int
)
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "%s - a worker that listens to new endpoints and sends them as assets to mozdef\n", os.Args[0])
flag.PrintDefaults()
}
var configPath = flag.String("c", "/etc/mig/mozdef_asset_worker_qa.cfg", "Load configuration from file")
var sampling = flag.Int("s", 1, "Sampling percentage, default sends 1% of events seen")
flag.Parse()
if *sampling > 100 || *sampling < 1 {
panic("invalid sampling percentage, must be an unsigned integer between 1 and 100")
}
ctrmod = 100 / *sampling
err = gcfg.ReadFileInto(&conf, *configPath)
if err != nil {
panic(err)
}
logctx, err := mig.InitLogger(conf.Logging, workerName)
if err != nil {
panic(err)
}
// bind to the MIG even queue
workerQueue := "migevent.worker." + workerName
consumerChan, err := workers.InitMqWithConsumer(conf.Mq, workerQueue, event.Q_Agt_New)
if err != nil {
panic(err)
}
// bind to the mozdef relay exchange
gp, err := gozdef.InitAmqp(conf.MozDef)
if err != nil {
panic(err)
}
mig.ProcessLog(logctx, mig.Log{Desc: "worker started, consuming queue " + workerQueue + " from key " + event.Q_Agt_New})
for event := range consumerChan {
evctr++
if evctr != ctrmod {
continue
} else {
evctr = 0
}
var agt mig.Agent
err = json.Unmarshal(event.Body, &agt)
if err != nil {
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("invalid agent description: %v", err)}.Err())
}
hint, err = makeHintFromAgent(agt)
if err != nil {
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("failed to build asset hint: %v", err)}.Err())
}
err = publishHintToMozdef(hint, gp)
if err != nil {
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("failed to publish to mozdef: %v", err)}.Err())
}
mig.ProcessLog(logctx, mig.Log{Desc: "published asset hint for agent '" + hint.Name + "' to mozdef"}.Info())
}
return
}
func makeHintFromAgent(agt mig.Agent) (hint gozdef.HostAssetHint, err error) {
hint.Type = "host"
hint.Name = agt.Name
reipv4 := regexp.MustCompile(`([0-9]{1,3}\.){3}([0-9]{1,3})`)
for _, ip := range agt.Env.Addresses {
if reipv4.MatchString(ip) {
hint.IPv4 = append(hint.IPv4, ip)
} else {
hint.IPv6 = append(hint.IPv6, ip)
}
}
hint.OS = agt.Env.OS
hint.Arch = agt.Env.Arch
hint.Ident = agt.Env.Ident
hint.Init = agt.Env.Init
hint.IsProxied = agt.Env.IsProxied
if _, ok := agt.Tags.(map[string]interface{})["operator"]; ok {
hint.Operator = agt.Tags.(map[string]interface{})["operator"].(string)
}
return
}
func publishHintToMozdef(hint gozdef.HostAssetHint, gp gozdef.Publisher) error {
// create a new event and set values in the fields
ev, err := gozdef.NewEvent()
if err != nil {
return err
}
ev.Category = "asset_hint"
ev.Source = "mig"
ev.Summary = fmt.Sprintf("mig discovered endpoint %s", hint.Name)
ev.Tags = append(ev.Tags, "mig")
ev.Tags = append(ev.Tags, "asset")
ev.Info()
ev.Details = hint
return gp.Send(ev)
}