[medium] addition of fswatch module

fswatch is a simple file system monitoring module that takes advantage
of the persistent module framework in MIG to do continuous file
integrity scans of specific paths on the file system. When changes are
detected, alerts are generated and appear in the agent's log file.

client/mig/available_modules.go

@@ -9,6 +9,7 @@ import (
 	_ "mig.ninja/mig/modules/agentdestroy"
 	_ "mig.ninja/mig/modules/examplepersist"
 	_ "mig.ninja/mig/modules/file"
+	_ "mig.ninja/mig/modules/fswatch"
 	_ "mig.ninja/mig/modules/memory"
 	_ "mig.ninja/mig/modules/netstat"
 	_ "mig.ninja/mig/modules/ping"

mig-agent/available_modules.go

@@ -9,6 +9,7 @@ import (
 	_ "mig.ninja/mig/modules/agentdestroy"
 	_ "mig.ninja/mig/modules/examplepersist"
 	_ "mig.ninja/mig/modules/file"
+	_ "mig.ninja/mig/modules/fswatch"
 	_ "mig.ninja/mig/modules/memory"
 	_ "mig.ninja/mig/modules/netstat"
 	_ "mig.ninja/mig/modules/ping"

modules/fswatch/fswatch.go

@@ -0,0 +1,215 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Contributor: Aaron Meihm ameihm@mozilla.com [:alm]
+
+package fswatch /* import "mig.ninja/mig/modules/fswatch" */
+
+import (
+	"encoding/json"
+	"fmt"
+	"runtime"
+
+	"mig.ninja/mig/modules"
+)
+
+type module struct {
+}
+
+func (m *module) NewRun() modules.Runner {
+	return new(run)
+}
+
+func init() {
+	modules.Register("fswatch", new(module))
+}
+
+type run struct {
+	Parameters Parameters
+	Results    modules.Result
+}
+
+func buildResults(e elements, r *modules.Result) (buf []byte, err error) {
+	r.Success = true
+	r.Elements = e
+	r.FoundAnything = true
+	buf, err = json.Marshal(r)
+	return
+}
+
+var logChan chan string
+var handlerErrChan chan error
+var configChan chan []byte
+
+var alertChan chan FSWatchAlert
+
+// Alert severities
+const (
+	_ = iota
+	ALERT_CRITICAL
+	ALERT_HIGH
+	ALERT_MEDIUM
+	ALERT_LOW
+)
+
+// Create a new alert and send it to channel ch
+func newAlert(sev int, f string, a ...interface{}) {
+	alertChan <- FSWatchAlert{
+		Severity: sev,
+		Alert:    fmt.Sprintf(f, a...),
+	}
+}
+
+// An alert generated by the fswatch module
+type FSWatchAlert struct {
+	Severity int    `json:"severity"` // ALERT_CRITICAL, etc...
+	Alert    string `json:"alert"`    // alert text
+}
+
+// Convert FSWatchAlert to a string
+func (a FSWatchAlert) String() string {
+	sev := "unknown"
+	switch a.Severity {
+	case ALERT_CRITICAL:
+		sev = "critical"
+	case ALERT_HIGH:
+		sev = "high"
+	case ALERT_MEDIUM:
+		sev = "medium"
+	case ALERT_LOW:
+		sev = "low"
+	}
+	return fmt.Sprintf("[%v] %v", sev, a.Alert)
+}
+
+func moduleMain() {
+	var cfg config
+
+	// Initialize the channel any alerts will come in on from any
+	// goroutines we create
+	alertChan = make(chan FSWatchAlert, 16)
+
+	incfg := <-configChan
+	err := json.Unmarshal(incfg, &cfg)
+	if err != nil {
+		handlerErrChan <- err
+		return
+	}
+	logChan <- "module received configuration"
+
+	go fsWatch(cfg)
+	for {
+		a := <-alertChan
+		// For now just send the alert up to the agent's log, but we will
+		// eventually want to handle this differently here.
+		logChan <- a.String()
+	}
+}
+
+func requestHandler(p interface{}) (ret string) {
+	var results modules.Result
+	defer func() {
+		if e := recover(); e != nil {
+			results.Errors = append(results.Errors, fmt.Sprintf("%v", e))
+			results.Success = false
+			err, _ := json.Marshal(results)
+			ret = string(err)
+			return
+		}
+	}()
+	e := elements{Ok: true}
+	resp, err := buildResults(e, &results)
+	if err != nil {
+		panic(err)
+	}
+	return string(resp)
+}
+
+type config struct {
+	FSWatch struct {
+		Interval string
+	}
+	FSWatchPaths struct {
+		Path []string
+	}
+}
+
+func (r *run) PersistModConfig() interface{} {
+	return &config{}
+}
+
+func (r *run) RunPersist(in modules.ModuleReader, out modules.ModuleWriter) {
+	logChan = make(chan string, 64)
+	regChan := make(chan string, 64)
+	handlerErrChan = make(chan error, 64)
+	configChan = make(chan []byte, 1)
+
+	go moduleMain()
+	l, spec, err := modules.GetPersistListener("fswatch")
+	if err != nil {
+		handlerErrChan <- err
+	} else {
+		regChan <- spec
+	}
+	go modules.HandlePersistRequest(l, requestHandler, handlerErrChan)
+	modules.DefaultPersistHandlers(in, out, logChan, handlerErrChan, regChan, configChan)
+}
+
+func (r *run) Run(in modules.ModuleReader) (resStr string) {
+	defer func() {
+		if e := recover(); e != nil {
+			// return error in json
+			r.Results.Errors = append(r.Results.Errors, fmt.Sprintf("%v", e))
+			r.Results.Success = false
+			err, _ := json.Marshal(r.Results)
+			resStr = string(err)
+			return
+		}
+	}()
+	runtime.GOMAXPROCS(1)
+	sockspec, err := modules.ReadPersistInputParameters(in, &r.Parameters)
+	if err != nil {
+		panic(err)
+	}
+	err = r.ValidateParameters()
+	if err != nil {
+		panic(err)
+	}
+	resStr = modules.SendPersistRequest(r.Parameters, sockspec)
+	return
+}
+
+func (r *run) ValidateParameters() (err error) {
+	return
+}
+
+func (r *run) PrintResults(result modules.Result, foundOnly bool) (prints []string, err error) {
+	var (
+		elem elements
+	)
+
+	err = result.GetElements(&elem)
+	if err != nil {
+		panic(err)
+	}
+	resStr := fmt.Sprintf("ok:%v", elem.Ok)
+	prints = append(prints, resStr)
+	if !foundOnly {
+		for _, we := range result.Errors {
+			prints = append(prints, we)
+		}
+	}
+	return
+}
+
+type elements struct {
+	Ok bool `json:"ok"`
+}
+
+type Parameters struct {
+}
+
+func newParameters() *Parameters {
+	return &Parameters{}
+}

modules/fswatch/fswatch_profile_linux.go

@@ -0,0 +1,22 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Contributor: Aaron Meihm ameihm@mozilla.com [:alm]
+
+package fswatch /* import "mig.ninja/mig/modules/fswatch" */
+
+var localFsWatchProfile = fsWatchProfile{
+	[]fsWatchProfileEntry{
+		{"/boot", []fsWatchObject{}},
+		{"/etc/cron.d", []fsWatchObject{}},
+		{"/var/spool/cron", []fsWatchObject{}},
+		{"/bin", []fsWatchObject{}},
+		{"/sbin", []fsWatchObject{}},
+		{"/usr/bin", []fsWatchObject{}},
+		{"/usr/sbin", []fsWatchObject{}},
+		{"/etc", []fsWatchObject{}},
+		{"/etc/init.d", []fsWatchObject{}},
+		{"/etc/systemd/system", []fsWatchObject{}},
+	},
+}

modules/fswatch/fswatch_test.go

@@ -0,0 +1,16 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Contributor: Aaron Meihm ameihm@mozilla.com [:alm]
+
+package fswatch /* import "mig.ninja/mig/modules/fswatch" */
+
+import (
+	"mig.ninja/mig/testutil"
+	"testing"
+)
+
+func TestRegistration(t *testing.T) {
+	testutil.CheckModuleRegistration(t, "fswatch")
+}

modules/fswatch/paramscreator.go

@@ -0,0 +1,22 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Contributor: Aaron Meihm ameihm@mozilla.com [:alm]
+
+package fswatch /* import "mig.ninja/mig/modules/fswatch" */
+
+import (
+	"fmt"
+)
+
+func printHelp(isCmd bool) {
+	fmt.Printf(`Query parameters
+----------------
+This module has no parameters.
+`)
+}
+
+func (r *run) ParamsParser(args []string) (interface{}, error) {
+	return r.Parameters, r.ValidateParameters()
+}

modules/fswatch/watch.go

@@ -0,0 +1,256 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Contributor: Aaron Meihm ameihm@mozilla.com [:alm]
+
+package fswatch /* import "mig.ninja/mig/modules/fswatch" */
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"time"
+)
+
+// fsWatchProfile specifies which paths to monitor on the file system, as a
+// list of fsWatchProfileEntry types
+type fsWatchProfile struct {
+	entries []fsWatchProfileEntry
+}
+
+// fsWatchProfileEntry describes a path to monitor on the file system. path
+// can reference either a file or a directory. In the case of a directory,
+// the first level contents of that directory will be read and objects will
+// be populated with these vales. In the case of just a file, objects will
+// contain a single entry for the file path.
+type fsWatchProfileEntry struct {
+	path    string
+	objects []fsWatchObject
+}
+
+// Collect all entries in a directory indicated by a profile entry, note that we do
+// not do recursion here and only read the first directory level
+func (f *fsWatchProfileEntry) collectDirectory() error {
+	dirents, err := ioutil.ReadDir(f.path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			if len(f.objects) != 0 {
+				newAlert(ALERT_MEDIUM, "monitored directory %v disappeared", f.path)
+				// Since it is gone, also remove any objects referenced by it
+				f.objects = nil
+			}
+			return nil
+		}
+		return err
+	}
+
+	oldobjs := f.objects[:]
+	foundents := make([]string, 0)
+
+	for _, fname := range dirents {
+		fpath := path.Join(f.path, fname.Name())
+		finfo, err := os.Stat(fpath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				// We saw the entry in ReadDir but it's gone now, note this and
+				// continue
+				newAlert(ALERT_MEDIUM, "path %v disappeared from %v during directory refresh",
+					fpath, f.path)
+				continue
+			}
+			return err
+		}
+		// Only monitor regular files in a directory, anything else ignore
+		if !finfo.Mode().IsRegular() {
+			continue
+		}
+		foundents = append(foundents, fpath)
+	}
+
+	f.objects = nil
+	for i := range oldobjs {
+		var objval *fsWatchObject
+		for _, x := range foundents {
+			if oldobjs[i].path == x {
+				objval = &oldobjs[i]
+				break
+			}
+		}
+		if objval != nil { // Existed in oldobjs and foundents, retain the entry
+			f.objects = append(f.objects, *objval)
+		} else {
+			newAlert(ALERT_MEDIUM, "monitored path %v disappeared", oldobjs[i].path)
+		}
+	}
+	// Add any new entries
+	for _, x := range foundents {
+		found := false
+		for _, y := range f.objects {
+			if y.path == x {
+				found = true
+				break
+			}
+		}
+		if found {
+			continue
+		}
+		f.objects = append(f.objects, fsWatchObject{x, nil, nil})
+		logChan <- fmt.Sprintf("fswatch added %v from directory", x)
+	}
+
+	return nil
+}
+
+// Return true if a fsWatchProfileEntry f has an object entry for path
+func (f *fsWatchProfileEntry) hasObject(path string) bool {
+	for _, x := range f.objects {
+		if x.path == path {
+			return true
+		}
+	}
+	return false
+}
+
+// Do hash comparisons of all objects in fsWatchProfileEntry f
+func (f *fsWatchProfileEntry) hashcheck() error {
+	for i := range f.objects {
+		err := f.objects[i].hash()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Scan all entries in the profile, and populate the object list for each entry
+func (f *fsWatchProfileEntry) refresh() error {
+	finfo, err := os.Stat(f.path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			if len(f.objects) != 0 {
+				newAlert(ALERT_MEDIUM, "monitored path %v disappeared", f.path)
+				// Since it is gone, also remove any objects referenced by it
+				f.objects = nil
+			}
+			return nil
+		}
+		return err
+	}
+	if finfo.Mode().IsDir() {
+		return f.collectDirectory()
+	} else if finfo.Mode().IsRegular() {
+		if !f.hasObject(f.path) {
+			f.objects = append(f.objects, fsWatchObject{f.path, nil, nil})
+			logChan <- fmt.Sprintf("fswatch added %v", f.path)
+		}
+	} else {
+		return fmt.Errorf("fswatch entry is not a directory or regular file")
+	}
+	return nil
+}
+
+// fsWatchObject describes an individual object identified in a profile entry
+type fsWatchObject struct {
+	path     string // object path
+	previous []byte // hash previously calculated
+	current  []byte // hash currently calculated
+}
+
+// Hash object f, setting previous to current and recalculating current
+func (f *fsWatchObject) hash() error {
+	fd, err := os.Open(f.path)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+	f.previous = f.current
+	h := sha256.New()
+	buf := make([]byte, 4096)
+	for {
+		n, err := fd.Read(buf)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		if n > 0 {
+			h.Write(buf[:n])
+		}
+	}
+	f.current = h.Sum(nil)
+	if f.previous == nil {
+		return nil
+	}
+	if bytes.Compare(f.previous, f.current) != 0 {
+		newAlert(ALERT_HIGH, "signature for %v changed: %x -> %x", f.path,
+			f.previous, f.current)
+	}
+	return nil
+}
+
+// Primary function that identifies all entries in our watch lists, hashes
+// identified objects, and creates alert messages
+func fsWatchRefreshEntries(profile *fsWatchProfile) (err error) {
+	for i := range profile.entries {
+		err = profile.entries[i].refresh()
+		if err != nil {
+			return
+		}
+		err = profile.entries[i].hashcheck()
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Main entry routine for file system monitor
+func fsWatch(cfg config) {
+	var err error
+	profile := localFsWatchProfile
+
+	sdur := 5 * time.Minute
+	if cfg.FSWatch.Interval != "" {
+		sdur, err = time.ParseDuration(cfg.FSWatch.Interval)
+		if err != nil {
+			handlerErrChan <- err
+			return
+		}
+	}
+	logChan <- fmt.Sprintf("fswatch interval set to %v", sdur)
+
+	// If custom paths have been indicated in the config file, override the
+	// local profile
+	if len(cfg.FSWatchPaths.Path) != 0 {
+		logChan <- "fswatch using monitoring paths set it configuration file"
+		profile.entries = nil
+		for _, x := range cfg.FSWatchPaths.Path {
+			newent := fsWatchProfileEntry{
+				path:    x,
+				objects: nil,
+			}
+			profile.entries = append(profile.entries, newent)
+			logChan <- fmt.Sprintf("fswatch watching: %v", x)
+		}
+	} else {
+		logChan <- "fswatch using built-in monitoring paths"
+		for _, x := range profile.entries {
+			logChan <- fmt.Sprintf("fswatch watching: %v", x.path)
+		}
+	}
+
+	for {
+		err = fsWatchRefreshEntries(&profile)
+		if err != nil {
+			handlerErrChan <- err
+			return
+		}
+		time.Sleep(sdur)
+	}
+}