зеркало из https://github.com/mozilla/mig.git
[minor] various fixes to build server packages
This commit is contained in:
Julien Vehent
Родитель
bbf462f3f9
Коммит
85787c1d3c
36
Makefile
36
Makefile
|
|
@ -288,21 +288,27 @@ else
|
|||
-o ./mig-clients-$(BUILDREV)-$(FPMARCH).dmg tmpdmg
|
||||
endif
|
||||
|
||||
rpm-scheduler: mig-scheduler
|
||||
deb-server: go_get_platform_deps mig-scheduler mig-api worker-agent-intel worker-compliance-item
|
||||
rm -rf tmp
|
||||
$(INSTALL) -D -m 0755 $(BINDIR)/mig-scheduler tmp/usr/bin/mig-scheduler
|
||||
$(INSTALL) -D -m 0640 conf/mig-scheduler.cfg.inc tmp/etc/mig/mig-scheduler.cfg
|
||||
# add binaries
|
||||
$(INSTALL) -D -m 0755 $(BINDIR)/mig-scheduler tmp/opt/mig/bin/mig-scheduler
|
||||
$(INSTALL) -D -m 0755 $(BINDIR)/mig-api tmp/opt/mig/bin/mig-api
|
||||
$(INSTALL) -D -m 0755 $(BINDIR)/mig-agent-intel-worker tmp/opt/mig/bin//mig-agent-intel-worker
|
||||
$(INSTALL) -D -m 0755 $(BINDIR)/mig-compliance-item-worker tmp/opt/mig/bin/mig-compliance-item-worker
|
||||
$(INSTALL) -D -m 0755 tools/list_new_agents.sh tmp/opt/mig/bin/list_new_agents.sh
|
||||
# add configuration templates
|
||||
$(INSTALL) -D -m 0640 conf/scheduler.cfg.inc tmp/etc/mig/scheduler.cfg
|
||||
$(INSTALL) -D -m 0640 conf/api.cfg.inc tmp/etc/mig/api.cfg
|
||||
$(INSTALL) -D -m 0640 conf/agent-intel-worker.cfg.inc tmp/etc/mig/agent-intel-worker.cfg
|
||||
$(INSTALL) -D -m 0640 conf/compliance-item-worker.cfg.inc tmp/etc/mig/compliance-item-worker.cfg
|
||||
# add upstart configs
|
||||
$(INSTALL) -D -m 0640 conf/upstart/mig-scheduler.conf tmp/etc/init/mig-scheduler.conf
|
||||
$(INSTALL) -D -m 0640 conf/upstart/mig-api.conf tmp/etc/init/mig-api.conf
|
||||
$(INSTALL) -D -m 0640 conf/upstart/mig-compliance-item-worker.conf tmp/etc/init/mig-compliance-item-worker.conf
|
||||
$(INSTALL) -D -m 0640 conf/upstart/mig-agent-intel-worker.conf tmp/etc/init/mig-agent-intel-worker.conf
|
||||
$(MKDIR) -p tmp/var/cache/mig
|
||||
fpm -C tmp -n mig-scheduler --license GPL --vendor mozilla --description "Mozilla InvestiGator Scheduler" \
|
||||
-m "Mozilla OpSec" --url http://mig.mozilla.org --architecture $(FPMARCH) -v $(BUILDREV) -s dir -t rpm .
|
||||
|
||||
rpm-api: mig-api
|
||||
rm -rf tmp
|
||||
$(INSTALL) -D -m 0755 $(BINDIR)/mig-api tmp/usr/bin/mig-api
|
||||
$(INSTALL) -D -m 0640 conf/mig-api.cfg.inc tmp/etc/mig/mig-api.cfg
|
||||
$(MKDIR) -p tmp/var/cache/mig
|
||||
fpm -C tmp -n mig-api --license GPL --vendor mozilla --description "Mozilla InvestiGator API" \
|
||||
-m "Mozilla OpSec" --url http://mig.mozilla.org --architecture $(FPMARCH) -v $(BUILDREV) -s dir -t rpm .
|
||||
fpm -C tmp -n mig-server --license GPL --vendor mozilla --description "Mozilla InvestiGator Server" \
|
||||
-m "Mozilla OpSec" --url http://mig.mozilla.org --architecture $(FPMARCH) -v $(BUILDREV) -s dir -t deb .
|
||||
|
||||
worker-agent-verif:
|
||||
$(MKDIR) -p $(BINDIR)
|
||||
|
|
@ -310,11 +316,11 @@ worker-agent-verif:
|
|||
|
||||
worker-agent-intel:
|
||||
$(MKDIR) -p $(BINDIR)
|
||||
$(GO) build $(GOOPTS) -o $(BINDIR)/mig_agent_intel_worker $(GOLDFLAGS) mig/workers/agent_intel
|
||||
$(GO) build $(GOOPTS) -o $(BINDIR)/mig-agent-intel-worker $(GOLDFLAGS) mig/workers/agent_intel
|
||||
|
||||
worker-compliance-item:
|
||||
$(MKDIR) -p $(BINDIR)
|
||||
$(GO) build $(GOOPTS) -o $(BINDIR)/mig_compliance_item_worker $(GOLDFLAGS) mig/workers/compliance_item
|
||||
$(GO) build $(GOOPTS) -o $(BINDIR)/mig-compliance-item-worker $(GOLDFLAGS) mig/workers/compliance_item
|
||||
|
||||
doc:
|
||||
make -C doc doc
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
[mozdef]
|
||||
host = "mozdef-relay.example.net"
|
||||
port = 5671
|
||||
user = "mig-agent-intel-worker"
|
||||
pass = "secretpassphrase"
|
||||
vhost = "prod"
|
||||
exchange = "eventtask"
|
||||
routingkey = "eventtask"
|
||||
usetls = true
|
||||
cacertpath = "/etc/mig/certs/ca.crt"
|
||||
clientcertpath = "/etc/mig/certs/worker.crt"
|
||||
clientkeypath = "/etc/mig/certs/worker.key"
|
||||
timeout = "10s"
|
||||
[mq]
|
||||
host = "somerelay.somedomain.example.net"
|
||||
port = 443
|
||||
user = "worker"
|
||||
pass = "secretpassphrase2"
|
||||
vhost = "mig"
|
||||
usetls = true
|
||||
cacert = "/etc/mig/certs/ca.crt"
|
||||
tlscert = "/etc/mig/certs/worker.crt"
|
||||
tlskey = "/etc/mig/certs/worker.key"
|
||||
timeout = "10s"
|
||||
[vmintgr]
|
||||
bin = "/opt/vmintgr/bin/grouptest"
|
||||
[logging]
|
||||
mode = "syslog" ; stdout | file | syslog
|
||||
level = "info"
|
||||
host = "localhost"
|
||||
port = 514
|
||||
protocol = "udp"
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
[api]
|
||||
host = "https://api.mig.example.net"
|
||||
[mozdef]
|
||||
host = "mozdef-relay.example.net"
|
||||
port = 5671
|
||||
user = "mig-agent-intel-worker"
|
||||
pass = "secretpassphrase"
|
||||
vhost = "prod"
|
||||
exchange = "eventtask"
|
||||
routingkey = "eventtask"
|
||||
usetls = true
|
||||
cacertpath = "/etc/mig/certs/ca.crt"
|
||||
clientcertpath = "/etc/mig/certs/worker.crt"
|
||||
clientkeypath = "/etc/mig/certs/worker.key"
|
||||
timeout = "10s"
|
||||
[mq]
|
||||
host = "somerelay.somedomain.example.net"
|
||||
port = 443
|
||||
user = "worker"
|
||||
pass = "secretpassphrase2"
|
||||
vhost = "mig"
|
||||
usetls = true
|
||||
cacert = "/etc/mig/certs/ca.crt"
|
||||
tlscert = "/etc/mig/certs/worker.crt"
|
||||
tlskey = "/etc/mig/certs/worker.key"
|
||||
timeout = "10s"
|
||||
[vmintgr]
|
||||
bin = "/opt/vmintgr/bin/grouptest"
|
||||
[logging]
|
||||
mode = "syslog" ; stdout | file | syslog
|
||||
level = "info"
|
||||
host = "localhost"
|
||||
port = 514
|
||||
protocol = "udp"
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Mozilla InvestiGator MozDef Asset Worker
|
||||
|
||||
description "MIG Agent Intel Worker"
|
||||
|
||||
start on filesystem or runlevel [2345]
|
||||
stop on runlevel [!2345]
|
||||
|
||||
setuid mig
|
||||
limit nofile 640000 640000
|
||||
|
||||
respawn
|
||||
respawn limit 10 5
|
||||
umask 022
|
||||
|
||||
console none
|
||||
|
||||
pre-start script
|
||||
test /opt/mig/bin/mig-agent-intel-worker || { stop; exit 0; }
|
||||
end script
|
||||
|
||||
# Start
|
||||
exec /opt/mig/bin/mig-agent-intel-worker
|
||||
|
|
@ -14,8 +14,8 @@ umask 022
|
|||
console none
|
||||
|
||||
pre-start script
|
||||
test /opt/mig-api || { stop; exit 0; }
|
||||
test /opt/mig/bin/mig-api || { stop; exit 0; }
|
||||
end script
|
||||
|
||||
# Start
|
||||
exec /opt/mig-api
|
||||
exec /opt/mig/bin/mig-api
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
# Mozilla InvestiGator MozDef Compliance Item Worker
|
||||
|
||||
description "MIG MozDef Compliance Item Worker"
|
||||
|
||||
start on filesystem or runlevel [2345]
|
||||
stop on runlevel [!2345]
|
||||
|
||||
setuid mig
|
||||
limit nofile 640000 640000
|
||||
|
||||
respawn
|
||||
respawn limit 10 5
|
||||
umask 022
|
||||
|
||||
console none
|
||||
|
||||
pre-start script
|
||||
test /opt/mig/bin/mig-compliance-item-worker || { stop; exit 0; }
|
||||
end script
|
||||
|
||||
# Start
|
||||
exec /opt/mig/bin/mig-compliance-item-worker
|
||||
|
|
@ -15,8 +15,8 @@ umask 022
|
|||
console none
|
||||
|
||||
pre-start script
|
||||
test /opt/mig-scheduler || { stop; exit 0; }
|
||||
test /opt/mig/bin/mig-scheduler || { stop; exit 0; }
|
||||
end script
|
||||
|
||||
# Start
|
||||
exec /opt/mig-scheduler
|
||||
exec /opt/mig/bin/mig-scheduler
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ func main() {
|
|||
fmt.Fprintf(os.Stderr, "%s - a worker that listens to new endpoints and sends them as assets to mozdef\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
var configPath = flag.String("c", "/etc/mig/agent_intel_worker.cfg", "Load configuration from file")
|
||||
var configPath = flag.String("c", "/etc/mig/agent-intel-worker.cfg", "Load configuration from file")
|
||||
flag.Parse()
|
||||
err = gcfg.ReadFileInto(&conf, *configPath)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func main() {
|
|||
fmt.Fprintf(os.Stderr, "%s - a worker that transform commands results into compliance items and publishes them to mozdef\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
var configPath = flag.String("c", "/etc/mig/compliance_item_worker.cfg", "Load configuration from file")
|
||||
var configPath = flag.String("c", "/etc/mig/compliance-item-worker.cfg", "Load configuration from file")
|
||||
flag.Parse()
|
||||
err = gcfg.ReadFileInto(&conf, *configPath)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -1,267 +0,0 @@
|
|||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
|
||||
package main
|
||||
|
||||
import (
|
||||
"code.google.com/p/gcfg"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/jvehent/gozdef"
|
||||
"mig"
|
||||
"mig/event"
|
||||
"mig/modules/file"
|
||||
"mig/workers"
|
||||
"os"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
const workerName = "compliance_item_qa"
|
||||
|
||||
type Config struct {
|
||||
Mq workers.MqConf
|
||||
MozDef gozdef.MqConf
|
||||
Logging mig.Logging
|
||||
API struct {
|
||||
Host string
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
err error
|
||||
conf Config
|
||||
items []gozdef.ComplianceItem
|
||||
evctr int
|
||||
ctrmod int
|
||||
)
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "%s - a worker that transform commands results into compliance items and publishes them to mozdef\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
var configPath = flag.String("c", "/etc/mig/compliance_item_qa_worker.cfg", "Load configuration from file")
|
||||
var sampling = flag.Int("s", 1, "Sampling percentage, default sends 1% of events seen")
|
||||
flag.Parse()
|
||||
if *sampling > 100 || *sampling < 1 {
|
||||
panic("invalid sampling percentage, must be an unsigned integer between 1 and 100")
|
||||
}
|
||||
ctrmod = 100 / *sampling
|
||||
err = gcfg.ReadFileInto(&conf, *configPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
logctx, err := mig.InitLogger(conf.Logging, workerName)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// bind to the MIG even queue
|
||||
workerQueue := "migevent.worker." + workerName
|
||||
consumerChan, err := workers.InitMqWithConsumer(conf.Mq, workerQueue, event.Q_Cmd_Res)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// bind to the mozdef relay exchange
|
||||
gp, err := gozdef.InitAmqp(conf.MozDef)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
mig.ProcessLog(logctx, mig.Log{Desc: "worker started, consuming queue " + workerQueue + " from key " + event.Q_Cmd_Res})
|
||||
tFamRe := regexp.MustCompile("(?i)^compliance$")
|
||||
for event := range consumerChan {
|
||||
evctr++
|
||||
if evctr != ctrmod {
|
||||
continue
|
||||
} else {
|
||||
evctr = 0
|
||||
}
|
||||
var cmd mig.Command
|
||||
err = json.Unmarshal(event.Body, &cmd)
|
||||
if err != nil {
|
||||
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("invalid command: %v", err)}.Err())
|
||||
}
|
||||
// discard actions that aren't threat.family=compliance
|
||||
if !tFamRe.MatchString(cmd.Action.Threat.Family) {
|
||||
continue
|
||||
}
|
||||
items, err = makeComplianceItem(cmd, conf)
|
||||
if err != nil {
|
||||
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("failed to make compliance items: %v", err)}.Err())
|
||||
}
|
||||
for _, item := range items {
|
||||
// create a new event and set values in the fields
|
||||
ev, err := gozdef.NewEvent()
|
||||
if err != nil {
|
||||
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("failed to make new mozdef event: %v", err)}.Err())
|
||||
}
|
||||
ev.Category = "complianceitems"
|
||||
ev.Source = "mig"
|
||||
cverb := "fails"
|
||||
if item.Compliance {
|
||||
cverb = "passes"
|
||||
}
|
||||
ev.Summary = fmt.Sprintf("%s %s compliance with %s", item.Target, cverb, item.Check.Ref)
|
||||
ev.Tags = append(ev.Tags, "mig")
|
||||
ev.Tags = append(ev.Tags, "compliance")
|
||||
ev.Info()
|
||||
ev.Details = item
|
||||
err = gp.Send(ev)
|
||||
if err != nil {
|
||||
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("failed to publish to mozdef: %v", err)}.Err())
|
||||
}
|
||||
}
|
||||
mig.ProcessLog(logctx, mig.Log{Desc: fmt.Sprintf("published %d items from command %.0f to mozdef", len(items), cmd.ID)}.Info())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func makeComplianceItem(cmd mig.Command, conf Config) (items []gozdef.ComplianceItem, err error) {
|
||||
var ci gozdef.ComplianceItem
|
||||
ci.Utctimestamp = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
ci.Target = cmd.Agent.Name
|
||||
ci.Policy.Name = cmd.Action.Threat.Type
|
||||
ci.Policy.URL = cmd.Action.Description.URL
|
||||
ci.Policy.Level = cmd.Action.Threat.Level
|
||||
ci.Check.Ref = cmd.Action.Threat.Ref
|
||||
ci.Check.Description = cmd.Action.Name
|
||||
ci.Link = fmt.Sprintf("%s/command?commandid=%.0f", conf.API.Host, cmd.ID)
|
||||
if cmd.Agent.Tags != nil {
|
||||
if _, ok := cmd.Agent.Tags.(map[string]interface{})["operator"]; ok {
|
||||
ci.Tags = struct {
|
||||
Operator string `json:"operator"`
|
||||
}{
|
||||
Operator: cmd.Agent.Tags.(map[string]interface{})["operator"].(string),
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, result := range cmd.Results {
|
||||
buf, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
if i > (len(cmd.Action.Operations) - 1) {
|
||||
// skip this entry if the lookup fails
|
||||
continue
|
||||
}
|
||||
switch cmd.Action.Operations[i].Module {
|
||||
case "file":
|
||||
var r file.Results
|
||||
err = json.Unmarshal(buf, &r)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
for label, sr := range r.Elements {
|
||||
for _, mf := range sr {
|
||||
ci.Check.Location = mf.File
|
||||
ci.Check.Name = label
|
||||
ci.Check.Test.Type = "file"
|
||||
ci.Check.Test.Value = ""
|
||||
for _, v := range mf.Search.Names {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("name='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.Sizes {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("size='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.Modes {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("mode='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.Mtimes {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("mtime='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.Contents {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("content='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.MD5 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("md5='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA1 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha1='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA256 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha256='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA384 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha384='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA512 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha512='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA3_224 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha3_224='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA3_256 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha3_256='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA3_384 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha3_384='%s'", v)
|
||||
}
|
||||
for _, v := range mf.Search.SHA3_512 {
|
||||
if len(ci.Check.Test.Value) > 0 {
|
||||
ci.Check.Test.Value += " and "
|
||||
}
|
||||
ci.Check.Test.Value += fmt.Sprintf("sha3_512='%s'", v)
|
||||
}
|
||||
if mf.File == "" {
|
||||
for i, p := range mf.Search.Paths {
|
||||
if i > 0 {
|
||||
ci.Check.Location += ", "
|
||||
}
|
||||
ci.Check.Location += p
|
||||
}
|
||||
ci.Compliance = false
|
||||
} else {
|
||||
ci.Compliance = true
|
||||
}
|
||||
items = append(items, ci)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
for t in $(tail -10000 /var/log/daemon.log | grep 'is not authorized' |awk '{print $12}'|sort|uniq|sed "s/'//g")
|
||||
do
|
||||
if [ "$(grep $t /etc/mig/agents_whitelist.txt)" == "" ]
|
||||
then
|
||||
echo $t
|
||||
fi
|
||||
done
|
||||
Загрузка…
Ссылка в новой задаче