This PR adds support for file integrity monitoring. This is done by providing a simplified API that can be used to PATCH/GET FIM configurations. There is also code to build the FIM configuration to send back to osquery. Each PATCH request, if successful, replaces Fleet's existing FIM configuration. For example:

curl -X "PATCH" "https://localhost:8080/api/v1/kolide/fim" \
     -H "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzZXNzaW9uX2tleSI6IkVhaFhvZWswMGtWSEdaTTNCWndIMnhpYWxkNWZpcVFDR2hEcW1HK2UySmRNOGVFVE1DeTNTaUlFWmhZNUxhdW1ueFZDV2JiR1Bwdm5TKzdyK3NJUzNnPT0ifQ.SDCHAUA1vTuWGjXtcQds2GZLM27HAAiOUhR4WvgvTNY" \
     -H "Content-Type: application/json; charset=utf-8" \
     -d $'{
  "interval": 500,
  "file_paths": {
    "etc": [
      "/etc/%%"
    ],
    "users": [
      "/Users/%/Library/%%",
      "/Users/%/Documents/%%"
    ],
    "usr": [
      "/usr/bin/%%"
    ]
  }
}'
This commit is contained in:
John Murphy 2017-08-18 10:37:33 -05:00 коммит произвёл GitHub
Родитель 1e92f8fcfd
Коммит d5f9fcaeb2
21 изменённых файлов: 429 добавлений и 29 удалений

Просмотреть файл

@ -35,4 +35,11 @@ func testFileIntegrityMonitoring(t *testing.T, ds kolide.Datastore) {
assert.Len(t, actual, 2)
assert.Len(t, actual["fp1"], 3)
assert.Len(t, actual["fp2"], 2)
err = ds.ClearFIMSections()
require.Nil(t, err)
fs, err := ds.FIMSections()
assert.Nil(t, err)
assert.Len(t, fs, 0)
}

Просмотреть файл

@ -21,3 +21,7 @@ func (d *Datastore) FIMSections() (kolide.FIMSections, error) {
}
return result, nil
}
func (d *Datastore) ClearFIMSections() error {
panic("inmem is being deprecated")
}

Просмотреть файл

@ -1,22 +0,0 @@
package datastore
import (
"testing"
"github.com/kolide/fleet/server/config"
"github.com/kolide/fleet/server/datastore/inmem"
"github.com/stretchr/testify/require"
)
func TestInmem(t *testing.T) {
for _, f := range testFunctions {
t.Run(functionName(f), func(t *testing.T) {
ds, err := inmem.New(config.TestConfig())
require.Nil(t, err)
defer func() { require.Nil(t, ds.Drop()) }()
require.Nil(t, err)
f(t, ds)
})
}
}

Просмотреть файл

@ -41,6 +41,12 @@ func (d *Datastore) NewFIMSection(fp *kolide.FIMSection, opts ...kolide.Optional
return fp, nil
}
func (d *Datastore) ClearFIMSections() error {
sqlStatement := "DELETE FROM file_integrity_monitorings"
_, err := d.db.Exec(sqlStatement)
return err
}
func (d *Datastore) FIMSections() (kolide.FIMSections, error) {
sqlStatement := `
SELECT fim.section_name, mf.file FROM

Просмотреть файл

@ -0,0 +1,25 @@
package tables
import (
"database/sql"
)
func init() {
MigrationClient.AddMigration(Up20170519105648, Down20170519105648)
}
func Up20170519105648(tx *sql.Tx) error {
_, err := tx.Exec(
"ALTER TABLE `app_configs` " +
"ADD COLUMN `fim_interval` " +
"INT NOT NULL DEFAULT 300 AFTER `enable_sso`;",
)
return err
}
func Down20170519105648(tx *sql.Tx) error {
_, err := tx.Exec(
"ALTER TABLE `app_configs` DROP COLUMN `fim_interval` ;",
)
return err
}

Просмотреть файл

@ -127,6 +127,8 @@ type AppConfig struct {
IDPName string `db:"idp_name"`
// EnableSSO flag to determine whether or not to enable SSO
EnableSSO bool `db:"enable_sso"`
// FIMInterval defines the interval when file integrity checks will occurr
FIMInterval int `db:"fim_interval"`
}
// ModifyAppConfigRequest contains application configuration information

Просмотреть файл

@ -1,5 +1,7 @@
package kolide
import "context"
type FIMSections map[string][]string
type FileIntegrityMonitoringStore interface {
@ -7,9 +9,20 @@ type FileIntegrityMonitoringStore interface {
NewFIMSection(path *FIMSection, opts ...OptionalArg) (*FIMSection, error)
// FIMSections returns all named file sections
FIMSections() (FIMSections, error)
// ClearFIMSections removes all the FIM information
ClearFIMSections() error
}
// FilePath maps a name to a group of files for the osquery file_paths
// FileIntegrityMonitoringService methods to update
type FileIntegrityMonitoringService interface {
// GetFIM returns the FIM config
GetFIM(ctx context.Context) (*FIMConfig, error)
// ModifyFIM replaces existing FIM. To disable FIM send FIMConfig with
// empty FilePaths
ModifyFIM(ctx context.Context, fim FIMConfig) error
}
// FIMSection maps a name to a group of files for the osquery file_paths
// section.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type FIMSection struct {
@ -18,3 +31,13 @@ type FIMSection struct {
Description string
Paths []string `db:"-"`
}
// FIMConfig information to set up File Integrity Monitoring
type FIMConfig struct {
// Interval defines the frequency when the file monitor will run.
Interval uint `json:"interval"`
// FilePaths contains named groups of files to monitor. The hash key is the
// name, the array of strings contains paths to be monitored.
// See https://osquery.readthedocs.io/en/stable/deployment/file-integrity-monitoring/
FilePaths FIMSections `json:"file_paths,omitempty"`
}

Просмотреть файл

@ -62,9 +62,13 @@ type Decorators struct {
// OsqueryConfig is a struct that can be serialized into a valid osquery config
// using Go's JSON tooling.
type OsqueryConfig struct {
Options map[string]interface{} `json:"options"`
Decorators Decorators `json:"decorators,omitempty"`
Packs Packs `json:"packs,omitempty"`
Schedule map[string]QueryContent `json:"schedule,omitempty"`
Options map[string]interface{} `json:"options"`
Decorators Decorators `json:"decorators,omitempty"`
Packs Packs `json:"packs,omitempty"`
// FilePaths contains named collections of file paths used for
// FIM (File Integrity Monitoring)
FilePaths FIMSections `json:"file_paths,omitempty"`
}
// OsqueryResultLog is the format of an osquery result log (ie: a differential

Просмотреть файл

@ -18,4 +18,5 @@ type Service interface {
ImportConfigService
LicenseService
DecoratorService
FileIntegrityMonitoringService
}

Просмотреть файл

@ -9,6 +9,7 @@ package mock
//go:generate mockimpl -o datastore_options.go "s *OptionStore" "kolide.OptionStore"
//go:generate mockimpl -o datastore_packs.go "s *PackStore" "kolide.PackStore"
//go:generate mockimpl -o datastore_hosts.go "s *HostStore" "kolide.HostStore"
//go:generate mockimpl -o datastore_fim.go "s *FileIntegrityMonitoringStore" "kolide.FileIntegrityMonitoringStore"
import "github.com/kolide/fleet/server/kolide"
@ -20,9 +21,9 @@ type Store struct {
kolide.PasswordResetStore
kolide.QueryStore
kolide.ScheduledQueryStore
kolide.FileIntegrityMonitoringStore
kolide.YARAStore
kolide.TargetStore
FileIntegrityMonitoringStore
AppConfigStore
DecoratorStore
HostStore

Просмотреть файл

@ -0,0 +1,39 @@
// Automatically generated by mockimpl. DO NOT EDIT!
package mock
import "github.com/kolide/fleet/server/kolide"
var _ kolide.FileIntegrityMonitoringStore = (*FileIntegrityMonitoringStore)(nil)
type NewFIMSectionFunc func(path *kolide.FIMSection, opts ...kolide.OptionalArg) (*kolide.FIMSection, error)
type FIMSectionsFunc func() (kolide.FIMSections, error)
type ClearFIMSectionsFunc func() error
type FileIntegrityMonitoringStore struct {
NewFIMSectionFunc NewFIMSectionFunc
NewFIMSectionFuncInvoked bool
FIMSectionsFunc FIMSectionsFunc
FIMSectionsFuncInvoked bool
ClearFIMSectionsFunc ClearFIMSectionsFunc
ClearFIMSectionsFuncInvoked bool
}
func (s *FileIntegrityMonitoringStore) NewFIMSection(path *kolide.FIMSection, opts ...kolide.OptionalArg) (*kolide.FIMSection, error) {
s.NewFIMSectionFuncInvoked = true
return s.NewFIMSectionFunc(path, opts...)
}
func (s *FileIntegrityMonitoringStore) FIMSections() (kolide.FIMSections, error) {
s.FIMSectionsFuncInvoked = true
return s.FIMSectionsFunc()
}
func (s *FileIntegrityMonitoringStore) ClearFIMSections() error {
s.ClearFIMSectionsFuncInvoked = true
return s.ClearFIMSectionsFunc()
}

Просмотреть файл

@ -21,6 +21,7 @@ type HostFunc func(id uint) (*kolide.Host, error)
type ListHostsFunc func(opt kolide.ListOptions) ([]*kolide.Host, error)
type EnrollHostFunc func(osqueryHostId string, nodeKeySize int) (*kolide.Host, error)
type AuthenticateHostFunc func(nodeKey string) (*kolide.Host, error)
type MarkHostSeenFunc func(host *kolide.Host, t time.Time) error

Просмотреть файл

@ -0,0 +1,40 @@
package service
import (
"context"
"github.com/go-kit/kit/endpoint"
"github.com/kolide/fleet/server/kolide"
)
type modifyFIMResponse struct {
Err error `json:"error,omitempty"`
}
func (m modifyFIMResponse) error() error { return m.Err }
func makeModifyFIMEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, req interface{}) (interface{}, error) {
fimConfig := req.(kolide.FIMConfig)
var resp modifyFIMResponse
if err := svc.ModifyFIM(ctx, fimConfig); err != nil {
resp.Err = err
}
return resp, nil
}
}
type getFIMResponse struct {
Err error `json:"error,omitempty"`
Payload *kolide.FIMConfig `json:"payload,omitempty"`
}
func makeGetFIMEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, _ interface{}) (interface{}, error) {
fimConfig, err := svc.GetFIM(ctx)
if err != nil {
return getFIMResponse{Err: err}, nil
}
return getFIMResponse{Payload: fimConfig}, nil
}
}

Просмотреть файл

@ -86,6 +86,8 @@ type KolideEndpoints struct {
InitiateSSO endpoint.Endpoint
CallbackSSO endpoint.Endpoint
SSOSettings endpoint.Endpoint
GetFIM endpoint.Endpoint
ModifyFIM endpoint.Endpoint
}
// MakeKolideServerEndpoints creates the Kolide API endpoints.
@ -167,6 +169,8 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint
ChangeEmail: authenticatedUser(jwtKey, svc, makeChangeEmailEndpoint(svc)),
UpdateLicense: authenticatedUser(jwtKey, svc, mustBeAdmin(makeUpdateLicenseEndpoint(svc))),
GetLicense: authenticatedUser(jwtKey, svc, makeGetLicenseEndpoint(svc)),
GetFIM: authenticatedUser(jwtKey, svc, makeGetFIMEndpoint(svc)),
ModifyFIM: authenticatedUser(jwtKey, svc, makeModifyFIMEndpoint(svc)),
// Osquery endpoints
EnrollAgent: makeEnrollAgentEndpoint(svc),
@ -249,6 +253,8 @@ type kolideHandlers struct {
InitiateSSO http.Handler
CallbackSSO http.Handler
SettingsSSO http.Handler
ModifyFIM http.Handler
GetFIM http.Handler
}
func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *kolideHandlers {
@ -327,6 +333,8 @@ func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *koli
InitiateSSO: newServer(e.InitiateSSO, decodeInitiateSSORequest),
CallbackSSO: newServer(e.CallbackSSO, decodeCallbackSSORequest),
SettingsSSO: newServer(e.SSOSettings, decodeNoParamsRequest),
ModifyFIM: newServer(e.ModifyFIM, decodeModifyFIMRequest),
GetFIM: newServer(e.GetFIM, decodeNoParamsRequest),
}
}
@ -435,6 +443,9 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) {
r.Handle("/api/v1/kolide/hosts/{id}", h.GetHost).Methods("GET").Name("get_host")
r.Handle("/api/v1/kolide/hosts/{id}", h.DeleteHost).Methods("DELETE").Name("delete_host")
r.Handle("/api/v1/kolide/fim", h.GetFIM).Methods("GET").Name("get_fim")
r.Handle("/api/v1/kolide/fim", h.ModifyFIM).Methods("PATCH").Name("post_fim")
r.Handle("/api/v1/kolide/options", h.GetOptions).Methods("GET").Name("get_options")
r.Handle("/api/v1/kolide/options", h.ModifyOptions).Methods("PATCH").Name("modify_options")
r.Handle("/api/v1/kolide/options/reset", h.ResetOptions).Methods("GET").Name("reset_options")

Просмотреть файл

@ -0,0 +1,32 @@
package service
import (
"context"
"time"
"github.com/kolide/fleet/server/kolide"
)
func (lm loggingMiddleware) GetFIM(ctx context.Context) (cfg *kolide.FIMConfig, err error) {
defer func(begin time.Time) {
lm.logger.Log(
"method", "GetFIM",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
cfg, err = lm.Service.GetFIM(ctx)
return cfg, err
}
func (lm loggingMiddleware) ModifyFIM(ctx context.Context, fim kolide.FIMConfig) (err error) {
defer func(begin time.Time) {
lm.logger.Log(
"method", "ModifyFIM",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
err = lm.Service.ModifyFIM(ctx, fim)
return err
}

Просмотреть файл

@ -0,0 +1,29 @@
package service
import (
"context"
"fmt"
"time"
"github.com/kolide/fleet/server/kolide"
)
func (mw metricsMiddleware) GetFIM(ctx context.Context) (cfg *kolide.FIMConfig, err error) {
defer func(begin time.Time) {
lvs := []string{"method", "GetFIM", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
cfg, err = mw.Service.GetFIM(ctx)
return cfg, err
}
func (mw metricsMiddleware) ModifyFIM(ctx context.Context, fim kolide.FIMConfig) (err error) {
defer func(begin time.Time) {
lvs := []string{"method", "ModifyFIM", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
err = mw.Service.ModifyFIM(ctx, fim)
return err
}

Просмотреть файл

@ -0,0 +1,46 @@
package service
import (
"context"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (svc service) GetFIM(ctx context.Context) (*kolide.FIMConfig, error) {
config, err := svc.ds.AppConfig()
if err != nil {
return nil, errors.Wrap(err, "getting fim config")
}
paths, err := svc.ds.FIMSections()
if err != nil {
return nil, errors.Wrap(err, "getting fim paths")
}
result := &kolide.FIMConfig{
Interval: uint(config.FIMInterval),
FilePaths: paths,
}
return result, nil
}
// ModifyFIM will remove existing FIM settings and replace it
func (svc service) ModifyFIM(ctx context.Context, fim kolide.FIMConfig) error {
if err := svc.ds.ClearFIMSections(); err != nil {
return errors.Wrap(err, "updating fim")
}
config, err := svc.ds.AppConfig()
if err != nil {
return errors.Wrap(err, "updating fim")
}
config.FIMInterval = int(fim.Interval)
for sectionName, paths := range fim.FilePaths {
section := kolide.FIMSection{
SectionName: sectionName,
Paths: paths,
}
if _, err := svc.ds.NewFIMSection(&section); err != nil {
return errors.Wrap(err, "creating fim section")
}
}
return svc.ds.SaveAppConfig(config)
}

Просмотреть файл

@ -0,0 +1,89 @@
package service
import (
"context"
"testing"
"github.com/kolide/fleet/server/kolide"
"github.com/kolide/fleet/server/mock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetFIMService(t *testing.T) {
ds := &mock.Store{
AppConfigStore: mock.AppConfigStore{
AppConfigFunc: func() (*kolide.AppConfig, error) {
config := &kolide.AppConfig{
FIMInterval: 300,
}
return config, nil
},
},
FileIntegrityMonitoringStore: mock.FileIntegrityMonitoringStore{
FIMSectionsFunc: func() (kolide.FIMSections, error) {
result := kolide.FIMSections{
"etc": []string{
"/etc/config/%%",
"/etc/zipp",
},
}
return result, nil
},
},
}
svc := service{
ds: ds,
}
resp, err := svc.GetFIM(context.Background())
require.Nil(t, err)
require.NotNil(t, resp)
assert.Equal(t, resp.Interval, uint(300))
paths, ok := resp.FilePaths["etc"]
require.True(t, ok)
assert.Len(t, paths, 2)
}
func TestUpdateFIM(t *testing.T) {
ds := &mock.Store{
AppConfigStore: mock.AppConfigStore{
AppConfigFunc: func() (*kolide.AppConfig, error) {
config := &kolide.AppConfig{
FIMInterval: 300,
}
return config, nil
},
SaveAppConfigFunc: func(_ *kolide.AppConfig) error {
return nil
},
},
FileIntegrityMonitoringStore: mock.FileIntegrityMonitoringStore{
ClearFIMSectionsFunc: func() error {
return nil
},
NewFIMSectionFunc: func(fs *kolide.FIMSection, _ ...kolide.OptionalArg) (*kolide.FIMSection, error) {
fs.ID = 1
return fs, nil
},
},
}
svc := service{
ds: ds,
}
fim := kolide.FIMConfig{
Interval: uint(300),
FilePaths: kolide.FIMSections{
"etc": []string{
"/etc/config/%%",
"/etc/zipp",
},
},
}
err := svc.ModifyFIM(context.Background(), fim)
require.Nil(t, err)
assert.True(t, ds.NewFIMSectionFuncInvoked)
assert.True(t, ds.ClearFIMSectionsFuncInvoked)
assert.True(t, ds.AppConfigFuncInvoked)
assert.True(t, ds.SaveAppConfigFuncInvoked)
}

Просмотреть файл

@ -89,6 +89,25 @@ func (svc service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier
return host.NodeKey, nil
}
func (svc service) getFIMConfig(ctx context.Context, cfg *kolide.OsqueryConfig) (*kolide.OsqueryConfig, error) {
fimConfig, err := svc.GetFIM(ctx)
if err != nil {
return nil, osqueryError{message: "internal error: unable to fetch FIM configuration"}
}
if cfg.Schedule == nil {
cfg.Schedule = make(map[string]kolide.QueryContent)
}
removed := false
// file events scheduled query is required to run file integrity monitors
cfg.Schedule["file_events"] = kolide.QueryContent{
Query: "SELECT * FROM file_events;",
Interval: fimConfig.Interval,
Removed: &removed,
}
cfg.FilePaths = fimConfig.FilePaths
return cfg, nil
}
func (svc service) GetClientConfig(ctx context.Context) (*kolide.OsqueryConfig, error) {
host, ok := hostctx.FromContext(ctx)
if !ok {
@ -192,7 +211,7 @@ func (svc service) GetClientConfig(ctx context.Context) (*kolide.OsqueryConfig,
}
}
return config, nil
return svc.getFIMConfig(ctx, config)
}
// If osqueryWriters are based on bufio we want to flush after a batch of

Просмотреть файл

@ -972,6 +972,17 @@ func TestUpdateHostIntervals(t *testing.T) {
ds.ListLabelsForHostFunc = func(hid uint) ([]kolide.Label, error) {
return []kolide.Label{}, nil
}
ds.AppConfigFunc = func() (*kolide.AppConfig, error) {
return &kolide.AppConfig{FIMInterval: 400}, nil
}
ds.FIMSectionsFunc = func() (kolide.FIMSections, error) {
sections := kolide.FIMSections{
"etc": []string{
"/etc/%%",
},
}
return sections, nil
}
var testCases = []struct {
initHost kolide.Host
@ -1067,6 +1078,8 @@ func TestUpdateHostIntervals(t *testing.T) {
}
for _, tt := range testCases {
ds.FIMSectionsFuncInvoked = false
t.Run("", func(t *testing.T) {
ctx := hostctx.NewContext(context.Background(), tt.initHost)
@ -1081,9 +1094,22 @@ func TestUpdateHostIntervals(t *testing.T) {
return nil
}
_, err = svc.GetClientConfig(ctx)
cfg, err := svc.GetClientConfig(ctx)
require.Nil(t, err)
assert.Equal(t, tt.saveHostCalled, saveHostCalled)
require.True(t, ds.FIMSectionsFuncInvoked)
require.Condition(t, func() bool {
_, ok := cfg.Schedule["file_events"]
return ok
})
assert.Equal(t, 400, int(cfg.Schedule["file_events"].Interval))
assert.Equal(t, "SELECT * FROM file_events;", cfg.Schedule["file_events"].Query)
require.NotNil(t, cfg.FilePaths)
require.Condition(t, func() bool {
_, ok := cfg.FilePaths["etc"]
return ok
})
assert.Len(t, cfg.FilePaths["etc"], 1)
})
}

Просмотреть файл

@ -0,0 +1,17 @@
package service
import (
"context"
"encoding/json"
"net/http"
"github.com/kolide/fleet/server/kolide"
)
func decodeModifyFIMRequest(ctx context.Context, r *http.Request) (interface{}, error) {
var fimConfig kolide.FIMConfig
if err := json.NewDecoder(r.Body).Decode(&fimConfig); err != nil {
return nil, err
}
return fimConfig, nil
}