mozilla
/
mig
зеркало из https://github.com/mozilla/mig.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #297 from ameihm0912/modpersist-config 

[medium] persistent module config file support
This commit is contained in:
Aaron Meihm 2016-12-16 15:58:15 -06:00 коммит произвёл GitHub
Родитель ff977a6a30 3b35161a0f
Коммит cf76532726
7 изменённых файлов: 166 добавлений и 22 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

25
conf/mig-agent-conf.go.inc
Просмотреть файл

@ -5,7 +5,7 @@
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main
import(
import (
"mig.ninja/mig"
"time"
)
@ -43,13 +43,21 @@ var EXTRAPRIVACYMODE = false
// disabled at run-time using a config option or command line flag
var SPAWNPERSISTENT = true
// The directory the agent will look for persistent module configuration files
// in.
//
// XXX This should be improved to take into account Windows paths, but at this
// time persistent module support is not available on Windows. The agent will
// attempt to locate a configuration using the module name, e.g., modulename.cfg.
var MODULECONFIGDIR = "/etc/mig"
// how often the agent will refresh its environment. if 0 agent
// will only update environment at initialization.
var REFRESHENV time.Duration = 0
var LOGGINGCONF = mig.Logging{
Mode: "stdout", // stdout | file | syslog
Level: "debug", // debug | info | ...
Mode: "stdout", // stdout | file | syslog
Level: "debug", // debug | info | ...
//File: "/tmp/migagt.log",
//MaxFileSize: 0,
//Host: "syslog_hostname",
@ -69,6 +77,7 @@ var APIURL string = "http://localhost:1664/api/v1/"
// if the connection still fails after looking for a HTTP_PROXY, try to use the
// proxies listed below
var PROXIES = []string{"proxy.example.net:3128", "proxy2.example.net:8080"}
// If you don't want proxies in the built-in configuration, use the following
// instead.
// var PROXIES = []string{}
@ -84,7 +93,7 @@ var MODULETIMEOUT time.Duration = 300 * time.Second
// Control modules permissions by PGP keys
var AGENTACL = [...]string{
`{
`{
"default": {
"minimumweight": 2,
"investigators": {
@ -99,7 +108,7 @@ var AGENTACL = [...]string{
}
}
}`,
`{
`{
"agentdestroy": {
"minimumweight": 1,
"investigators": {
@ -112,12 +121,11 @@ var AGENTACL = [...]string{
}`,
}
// PGP public keys that are authorized to sign actions
// this is an array of strings, put each public key block
// into its own array entry, as shown below
var PUBLICPGPKEYS = [...]string{
`
`
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1; Name: User for MIG test (Another test user for Mozilla Investigator) <usertest+mig@example.org>
@ -128,7 +136,7 @@ lMVXz7c/B8T79KIH0EDAG8o6AbvZQdTMSZp+Ap562smLkV+xsPo1O1Zd/hDJKYuY
=SWyb
-----END PGP PUBLIC KEY BLOCK-----
`,
`
`
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1; Name: Test User (This is a test user for Mozilla Investigator) <testuser+mig@example.net>
@ -140,7 +148,6 @@ QnD9SDA9/d80
-----END PGP PUBLIC KEY BLOCK-----
`}
// CA cert that signs the rabbitmq server certificate, for verification
// of the chain of trust. If rabbitmq uses a self-signed cert, add this
// cert below

9
mig-agent/config.go
Просмотреть файл

@ -32,6 +32,7 @@ type config struct {
Api string
RefreshEnv string
NoPersistMods bool
PersistConfigDir string
ExtraPrivacyMode bool
}
Certs struct {
@ -88,6 +89,9 @@ type globals struct {
// disabled at run-time using a config option or command line flag
spawnPersistent bool
// directory to look in for persistent module configuration files
persistConfigDir string
// how often the agent will refresh its environment. if 0 agent
// will only update environment at initialization.
refreshEnv time.Duration
@ -151,6 +155,7 @@ func newGlobals() *globals {
checkin: CHECKIN,
extraPrivacyMode: EXTRAPRIVACYMODE,
spawnPersistent: SPAWNPERSISTENT,
persistConfigDir: MODULECONFIGDIR,
refreshEnv: REFRESHENV,
loggingConf: LOGGINGCONF,
amqBroker: AMQPBROKER,
@ -179,6 +184,9 @@ func (g globals) parseConfig(config config) error {
if config.Agent.NoPersistMods {
g.spawnPersistent = false
}
if config.Agent.PersistConfigDir != "" {
g.persistConfigDir = config.Agent.PersistConfigDir
}
if config.Agent.RefreshEnv != "" {
g.refreshEnv, err = time.ParseDuration(config.Agent.RefreshEnv)
if err != nil {
@ -242,6 +250,7 @@ func (g globals) apply() {
CHECKIN = g.checkin
EXTRAPRIVACYMODE = g.extraPrivacyMode
SPAWNPERSISTENT = g.spawnPersistent
MODULECONFIGDIR = g.persistConfigDir
REFRESHENV = g.refreshEnv
LOGGINGCONF = g.loggingConf
AMQPBROKER = g.amqBroker

25
mig-agent/configuration.go
Просмотреть файл

@ -5,7 +5,7 @@
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main
import(
import (
"mig.ninja/mig"
"time"
)
@ -43,13 +43,21 @@ var EXTRAPRIVACYMODE = false
// disabled at run-time using a config option or command line flag
var SPAWNPERSISTENT = true
// The directory the agent will look for persistent module configuration files
// in.
//
// XXX This should be improved to take into account Windows paths, but at this
// time persistent module support is not available on Windows. The agent will
// attempt to locate a configuration using the module name, e.g., modulename.cfg.
var MODULECONFIGDIR = "/etc/mig"
// how often the agent will refresh its environment. if 0 agent
// will only update environment at initialization.
var REFRESHENV time.Duration = 0
var LOGGINGCONF = mig.Logging{
Mode: "stdout", // stdout | file | syslog
Level: "debug", // debug | info | ...
Mode: "stdout", // stdout | file | syslog
Level: "debug", // debug | info | ...
//File: "/tmp/migagt.log",
//MaxFileSize: 0,
//Host: "syslog_hostname",
@ -69,6 +77,7 @@ var APIURL string = "http://localhost:1664/api/v1/"
// if the connection still fails after looking for a HTTP_PROXY, try to use the
// proxies listed below
var PROXIES = []string{"proxy.example.net:3128", "proxy2.example.net:8080"}
// If you don't want proxies in the built-in configuration, use the following
// instead.
// var PROXIES = []string{}
@ -84,7 +93,7 @@ var MODULETIMEOUT time.Duration = 300 * time.Second
// Control modules permissions by PGP keys
var AGENTACL = [...]string{
`{
`{
"default": {
"minimumweight": 2,
"investigators": {
@ -99,7 +108,7 @@ var AGENTACL = [...]string{
}
}
}`,
`{
`{
"agentdestroy": {
"minimumweight": 1,
"investigators": {
@ -112,12 +121,11 @@ var AGENTACL = [...]string{
}`,
}
// PGP public keys that are authorized to sign actions
// this is an array of strings, put each public key block
// into its own array entry, as shown below
var PUBLICPGPKEYS = [...]string{
`
`
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1; Name: User for MIG test (Another test user for Mozilla Investigator) <usertest+mig@example.org>
@ -128,7 +136,7 @@ lMVXz7c/B8T79KIH0EDAG8o6AbvZQdTMSZp+Ap562smLkV+xsPo1O1Zd/hDJKYuY
=SWyb
-----END PGP PUBLIC KEY BLOCK-----
`,
`
`
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1; Name: Test User (This is a test user for Mozilla Investigator) <testuser+mig@example.net>
@ -140,7 +148,6 @@ QnD9SDA9/d80
-----END PGP PUBLIC KEY BLOCK-----
`}
// CA cert that signs the rabbitmq server certificate, for verification
// of the chain of trust. If rabbitmq uses a self-signed cert, add this
// cert below

39
mig-agent/persist.go
Просмотреть файл

@ -10,12 +10,15 @@ import (
"encoding/json"
"fmt"
"os/exec"
"path"
"strings"
"sync"
"time"
"mig.ninja/mig"
"mig.ninja/mig/modules"
"gopkg.in/gcfg.v1"
)
// persistModuleRegister maintains a map of the running persistent modules, and
@ -63,6 +66,19 @@ func (p *persistModuleRegister) remove(modname string) {
var persistModRegister persistModuleRegister
// Load the configuration file for a persistent module if it exists, and return it
// as a JSON byte slice so we can send it from the agent to the module after the
// module is started. If the configuration file cannot be loaded, just return the
// config struct for the module uninitialized.
func getPersistConfig(modname string) (ret interface{}) {
cfg := modules.Available[modname].NewRun().(modules.PersistRunner).PersistModConfig()
confpath := path.Join(MODULECONFIGDIR, modname+".cfg")
// An error here isn't fatal, we just continue with cfg as is
gcfg.ReadFileInto(cfg, confpath)
ret = cfg
return
}
// Start all the persistent modules available to the agent.
func startPersist(ctx *Context) (err error) {
defer func() {
@ -139,6 +155,7 @@ func managePersistModule(ctx *Context, name string) {
continue
}
pipein = modules.NewModuleReader(cmdpipein)
cfg := getPersistConfig(name)
err = cmd.Start()
if err != nil {
logfunc("error starting module, %v", err)
@ -160,6 +177,28 @@ func managePersistModule(ctx *Context, name string) {
}()
isRunning = true
// The module is now running, send any configuration parameters we have
// to it.
cm, err := modules.MakeMessageConfig(cfg)
if err != nil {
// This should never happen, but if it does we will just
// kill the executing module as we are unable to send any
// configuration to it
killModule = true
break
}
err = modules.WriteOutput(cm, pipeout)
if err != nil {
// XXX This should be revisited, both here and later on when
// sending a ping. If this write fails, we just assume the
// process is down, where it may not be.
logfunc("config write failed, %v", err)
isRunning = false
persistModRegister.remove(name)
failDelay = true
continue
}
}
select {
case msg, ok := <-inChan:

51
modules/examplepersist/examplepersist.go
Просмотреть файл

@ -62,12 +62,35 @@ var logChan chan string
// returning and the module exiting.
var handlerErrChan chan error
// When the agent sends the persistent module it's configuration, it will come in via
// the config channel as a JSON byte slice so we can unmarshal it into our configuration
var configChan chan []byte
// An example background task the module will execute while it is being supervised by
// the agent. This example just logs the current time up to the agent every 30
// seconds.
// seconds by default, or if a configuration file existed for the module it will use
// the interval value set there.
func runSomeTasks() {
var cfg config
// After the agent starts this module, it will send any module configuration
// which we can read immediately here. The configuration will come in via
// configChan as a JSON document, which we unmarshal into our config struct.
incfg := <-configChan
err := json.Unmarshal(incfg, &cfg)
if err != nil {
handlerErrChan <- err
return
}
logChan <- "module received configuration"
if cfg.ExamplePersist.Interval <= 0 {
logChan <- "config interval was <= 0, defaulting to 30 seconds"
cfg.ExamplePersist.Interval = 30
}
for {
time.Sleep(time.Second * 30)
time.Sleep(time.Duration(cfg.ExamplePersist.Interval) * time.Second)
// Send a log message up to the agent
logChan <- fmt.Sprintf("running, current time is %v", time.Now())
}
@ -109,6 +132,26 @@ func requestHandler(p interface{}) (ret string) {
return string(resp)
}
// The configuration for the persistent module; in the case of a persistent module
// that does not require configuration, our config struct would just be empty, but
// we need to define something to return so we can satisfy the PersistRunner
// interface.
//
// We need to make sure we have JSON tags here as this structure will be marshalled
// and sent to the running module by the agent.
type config struct {
ExamplePersist struct {
Interval int `json:"interval"`
}
}
// PersistModConfig must be implemented in persistent modules so we can satisfy
// the PersistRunner interface. Typically here we will just return a new config
// structure that will get used to load our configuration.
func (r *run) PersistModConfig() interface{} {
return &config{}
}
// RunPersist is the function used to initialize the persistent component
// of the module. It should not return. In this example, we do our initialization
// and call modules.DefaultPersistHandlers, which looks after handling all
@ -130,6 +173,8 @@ func (r *run) RunPersist(in modules.ModuleReader, out modules.ModuleWriter) {
// error to this channel will cause DefaultPersistHandlers() to return
// and the module to exit.
handlerErrChan = make(chan error, 64)
// Create a config channel we will read our configuration from.
configChan = make(chan []byte, 1)
// Start up an example background task we want our module to run
// continuously.
go runSomeTasks()
@ -146,7 +191,7 @@ func (r *run) RunPersist(in modules.ModuleReader, out modules.ModuleWriter) {
go modules.HandlePersistRequest(l, requestHandler, handlerErrChan)
// Finally, enter the standard module management function. This will not return
// unless an error occurs.
modules.DefaultPersistHandlers(in, out, logChan, handlerErrChan, regChan)
modules.DefaultPersistHandlers(in, out, logChan, handlerErrChan, regChan, configChan)
}
// Module Run function, used to make queries using the module.

38
modules/modules.go
Просмотреть файл

@ -49,6 +49,7 @@ const (
MsgClassPing MessageClass = "ping"
MsgClassLog MessageClass = "log"
MsgClassRegister MessageClass = "register"
MsgClassConfig MessageClass = "config"
)
// Parameter format expected for a log message
@ -61,6 +62,10 @@ type RegParams struct {
SockPath string `json:"sockpath"`
}
type ConfigParams struct {
Config interface{} `json:"config"`
}
// Result implement the base type for results returned by modules.
// All modules must return this type of result. The fields are:
//
@ -97,6 +102,7 @@ type Runner interface {
// PersistRunner.
type PersistRunner interface {
RunPersist(ModuleReader, ModuleWriter)
PersistModConfig() interface{}
}
// ModuleReader is used to read module communications. It's intent is to
@ -193,6 +199,18 @@ func MakeMessageRegister(spec string) (rawMsg []byte, err error) {
return
}
// Creates a new message of class config.
func MakeMessageConfig(cfgdata interface{}) (rawMsg []byte, err error) {
param := ConfigParams{Config: cfgdata}
msg := Message{Class: MsgClassConfig, Parameters: param}
rawMsg, err = json.Marshal(&msg)
if err != nil {
err = fmt.Errorf("Failed to make module config message: %v", err)
return
}
return
}
// Keep reading until we get a full line or an error, and return
func readInputLine(rdr *bufio.Reader) ([]byte, error) {
var ret []byte
@ -329,7 +347,7 @@ func WatchForStop(r ModuleReader, stopChan *chan bool) error {
// RunPersist function. Looks after replying to ping messages, writing logs, and other
// communication between the agent and the running persistent module.
func DefaultPersistHandlers(in ModuleReader, out ModuleWriter, logch chan string,
errch chan error, regch chan string) {
errch chan error, regch chan string, confch chan []byte) {
inChan := make(chan Message, 0)
go func() {
for {
@ -399,6 +417,24 @@ func DefaultPersistHandlers(in ModuleReader, out ModuleWriter, logch chan string
failed = true
break
}
case "config":
var confparam ConfigParams
buf, err := json.Marshal(msg.Parameters)
if err != nil {
failed = true
break
}
err = json.Unmarshal(buf, &confparam)
if err != nil {
failed = true
break
}
buf, err = json.Marshal(confparam.Config)
if err != nil {
failed = true
break
}
confch <- buf
}
}

1
tools/standalone_install.sh
Просмотреть файл

@ -311,6 +311,7 @@ var HEARTBEATFREQ time.Duration = 30 * time.Second
var REFRESHENV time.Duration = 60 * time.Second
var MODULETIMEOUT time.Duration = 300 * time.Second
var SPAWNPERSISTENT bool = true
var MODULECONFIGDIR = "/etc/mig"
var AGENTACL = [...]string{
\`{
"default": {