mozilla
/
mig
зеркало из https://github.com/mozilla/mig.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #87 from mozilla/fixsearch20150919 

Fix various DB Query issues
This commit is contained in:
Julien Vehent 2015-09-23 13:03:44 -04:00
Родитель 5c4fe0feb4 7e4d77c28f
Коммит 765db7647f
16 изменённых файлов: 1197 добавлений и 690 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
action.go
Просмотреть файл

@ -12,11 +12,12 @@ import (
"fmt"
"io"
"io/ioutil"
"mig.ninja/mig/pgp"
"os"
"strconv"
"sync"
"time"
"mig.ninja/mig/pgp"
)
// ActionVersion is the version of the syntax that is expected

15
client/client.go
Просмотреть файл

@ -13,14 +13,8 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"github.com/jvehent/cljs"
"golang.org/x/crypto/openpgp"
"gopkg.in/gcfg.v1"
"io"
"io/ioutil"
"mig.ninja/mig"
"mig.ninja/mig/modules"
"mig.ninja/mig/pgp"
"mime/multipart"
"net/http"
"net/url"
@ -28,6 +22,13 @@ import (
"os/user"
"strings"
"time"
"github.com/jvehent/cljs"
"golang.org/x/crypto/openpgp"
"gopkg.in/gcfg.v1"
"mig.ninja/mig"
"mig.ninja/mig/modules"
"mig.ninja/mig/pgp"
)
var version string
@ -695,7 +696,7 @@ func (cli Client) EvaluateAgentTarget(target string) (agents []mig.Agent, err er
err = fmt.Errorf("EvaluateAgentTarget() -> %v", e)
}
}()
query := "search?type=agent&target=" + url.QueryEscape(target)
query := "search?type=agent&limit=1000000&target=" + url.QueryEscape(target)
resource, err := cli.GetAPIResource(query)
if err != nil {
panic(err)

38
client/mig-console/action_reader.go
Просмотреть файл

@ -8,12 +8,14 @@ package main
import (
"encoding/json"
"fmt"
"github.com/bobappleyard/readline"
"io"
"mig.ninja/mig"
"mig.ninja/mig/client"
"strconv"
"strings"
"time"
"github.com/bobappleyard/readline"
"mig.ninja/mig"
"mig.ninja/mig/client"
)
// actionReader retrieves an action from the API using its numerical ID
@ -186,7 +188,7 @@ exit:
return
}
func actionPrintShort(data interface{}) (idstr, name, datestr, invs string, sent int, err error) {
func actionPrintShort(data interface{}) (idstr, name, target, datestr, invs, status string, sent int, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("actionPrintShort() -> %v", e)
@ -199,8 +201,8 @@ func actionPrintShort(data interface{}) (idstr, name, datestr, invs string, sent
invs = investigatorsStringFromAction(a.Investigators, 23)
idstr = fmt.Sprintf("%.0f", a.ID)
if len(idstr) < 20 {
for i := len(idstr); i < 20; i++ {
if len(idstr) < 14 {
for i := len(idstr); i < 14; i++ {
idstr += " "
}
}
@ -215,7 +217,24 @@ func actionPrintShort(data interface{}) (idstr, name, datestr, invs string, sent
name = name[0:27] + "..."
}
datestr = a.LastUpdateTime.Format("Mon Jan 2 3:04pm MST")
target = a.Target
if len(target) < 30 {
for i := len(target); i < 30; i++ {
target += " "
}
}
if len(target) > 30 {
target = target[0:27] + "..."
}
status = a.Status
if len(status) < 10 {
for i := len(status); i < 10; i++ {
status += " "
}
}
datestr = a.LastUpdateTime.UTC().Format(time.RFC3339)
if len(datestr) > 21 {
datestr = datestr[0:21]
}
@ -238,6 +257,11 @@ func investigatorsStringFromAction(invlist []mig.Investigator, strlen int) (inve
if len(investigators) > strlen {
investigators = investigators[0:(strlen-3)] + "..."
}
if len(investigators) < strlen {
for i := len(investigators); i < strlen; i++ {
investigators += " "
}
}
return
}

11
client/mig-console/console.go
Просмотреть файл

@ -9,16 +9,17 @@ import (
"encoding/json"
"flag"
"fmt"
"github.com/bobappleyard/readline"
"io"
"io/ioutil"
"log"
"mig.ninja/mig"
"mig.ninja/mig/client"
"net/http"
"os"
"strconv"
"strings"
"github.com/bobappleyard/readline"
"mig.ninja/mig"
"mig.ninja/mig/client"
)
// build version
@ -242,13 +243,13 @@ func printStatus(cli client.Client) (err error) {
var onlineagt, idleagt []string
actout := make([]string, 2)
actout[0] = "Latest Actions:"
actout[1] = "---- ID ---- + ---- Name ---- + -Sent- + ---- Date ---- + ---- Investigators ----"
actout[1] = "---- ID ---- + ---- Name ---- + -Sent- + ---- Date ---- + ---- Investigators ----"
var onlineagents, onlineendpoints, idleagents, idleendpoints, newendpoints, doubleagents, disappearedendpoints, flappingendpoints float64
for _, item := range st.Collection.Items {
for _, data := range item.Data {
switch data.Name {
case "action":
idstr, name, datestr, invs, sent, err := actionPrintShort(data.Value)
idstr, name, _, datestr, invs, _, sent, err := actionPrintShort(data.Value)
if err != nil {
panic(err)
}

15
client/mig-console/investigator.go
Просмотреть файл

@ -7,15 +7,16 @@ package main
import (
"fmt"
"github.com/bobappleyard/readline"
"io"
"io/ioutil"
"mig.ninja/mig/client"
"mig.ninja/mig/pgp"
"regexp"
"strconv"
"strings"
"time"
"github.com/bobappleyard/readline"
"mig.ninja/mig/client"
"mig.ninja/mig/pgp"
)
// investigatorReader retrieves an agent from the api
@ -63,7 +64,7 @@ func investigatorReader(input string, cli client.Client) (err error) {
fmt.Println("error: ", err)
break
}
orders := strings.Split(input, " ")
orders := strings.Split(strings.TrimSpace(input), " ")
switch orders[0] {
case "details":
fmt.Printf("Investigator ID %.0f\nname %s\nstatus %s\nkey id %s\ncreated %s\nmodified %s\n",
@ -117,6 +118,10 @@ setstatus <status> changes the status of the investigator to <status> (can be 'a
} else {
fmt.Println("Investigator status set to", newstatus)
}
inv, err = cli.GetInvestigator(iid)
if err != nil {
panic(err)
}
case "":
break
default:
@ -140,7 +145,7 @@ func printInvestigatorLastActions(iid float64, limit int, cli client.Client) (er
if err != nil {
panic(err)
}
fmt.Printf("------- ID ------- + -------- Action Name ------- + ----------- Target ---------- + ---- Date ---- + -- Status --\n")
fmt.Printf("----- ID ----- + -------- Action Name ------- + ----------- Target ---------- + ---- Date ---- + -- Status --\n")
for _, item := range resource.Collection.Items {
for _, data := range item.Data {
if data.Name != "action" {

369
client/mig-console/search.go
Просмотреть файл

@ -7,19 +7,14 @@ package main
import (
"fmt"
"github.com/jvehent/cljs"
"mig.ninja/mig"
"mig.ninja/mig/client"
"regexp"
"strconv"
"strings"
"time"
)
type searchParameters struct {
sType string
query string
version string
}
"mig.ninja/mig/client"
migdbsearch "mig.ninja/mig/database/search"
)
// search runs a search for actions, commands or agents
func search(input string, cli client.Client) (err error) {
@ -37,105 +32,161 @@ func search(input string, cli client.Client) (err error) {
case "action", "agent", "command", "investigator":
sType = orders[1]
case "", "help":
fmt.Printf(`usage: search <action|agent|command|investigator> where <parameters> [<and|or> <parameters>]
The following search parameters are available:
fmt.Printf(`usage: search <action|agent|command|investigator> where <key>=<value> [and <key>=<value>...]
Example:
mig> search command where agentname=%%khazad%% and investigatorname=%%vehent%% and actionname=%%memory%% and after=2015-09-09T17:00:00Z
---- ID ---- + ---- Name ---- + --- Last Updated ---
4886304327951 memory -c /home/ulfr/.migrc... 2015-09-09T13:01:03-04:00
The following search parameters are available, per search type:
* action:
- name=<str> search actions by name <str>
- before=<rfc3339> search actions that expired before <rfc3339 date>
- after=<rfc3339> search actions were valid after <rfc3339 date>
- commandid=<id> search action that spawned a given command
- agentid=<id> search actions that ran on a given agent
- agentname=<str> search actions that ran on an agent named <str>
- investigatorid=<id> search actions signed by a given investigator
- investigatorname=<str>search actions signed by investigator named <str>
- status=<str> search actions with a given status amongst:
pending, scheduled, preparing, invalid, inflight, completed
* command:
- name=<str> search commands by action name <str>
- before=<rfc3339> search commands that started before <rfc3339 date>
- after=<rfc3339> search commands that started after <rfc3339 date>
- actionid=<id> search commands spawned action <id>
- actionname=<str> search commands spawned by an action named <str>
- agentname=<str> search commands that ran on an agent named <str>
- agentid=<id> search commands that ran on a given agent
- investigatorid=<id> search commands signed by investigator <id>
- investigatorname=<str>search commands signed by investigator named <str>
- status=<str> search commands with a given status amongst:
prepared, sent, success, timeout, cancelled, expired, failed
* agent:
- name=<str> search agents by hostname
- before=<rfc3339> search agents that have sent a heartbeat before <rfc3339 date>
- after=<rfc3339> search agents that have sent a heartbeat after <rfc3339 date>
- actionid=<id> search agents that ran action <id>
- actionname=<str> search agents that ran action named <str>
- commandid=<id> search agents that ran command <id>
- investigatorid=<id> search agents that ran an action signed by investigator <id>
- investigatorname=<str>search agents that ran an action signed by investigator named <str>
- version=<str> search agents by version <str>
- status=<str> search agents with a given status amongst:
online, upgraded, destroyed, offline, idle
* investigator:
- name=<str> search investigators by name
- before=<rfc3339> search investigators created or modified before <rfc3339 date>
- after=<rfc3339> search investigators created or modified after <rfc3339 date>
- actionid=<id> search investigators that signed action <id>
- actionname=<str> search investigators that signed action named <str>
- commandid=<id> search investigators that ran command <id>
- agentid=<id> search investigators that ran a command on a given agent
- agentname=<str> search investigators that ran actions on an agent named <str>,
- status=<str> search investigators by status amongst: active, disabled
All searches accept the 'limit=<num>' parameter to limits the number of results returned by a search, defaults to 100
Parameters that accept a <str> can use wildcards * and % (ex: name=jul%veh% ).
No spaces are permitted within parameters. Spaces are used to separate search parameters.
`)
return nil
default:
return fmt.Errorf("Invalid search '%s'. Try `search help`.\n", input)
}
sp, err := parseSearchQuery(orders)
p, err := parseSearchQuery(orders)
if err != nil {
panic(err)
}
items, err := runSearchQuery(sp, cli)
fmt.Printf("Searching %s after %s and before %s, limited to %.0f results\n", p.Type,
p.After.Format(time.RFC3339), p.Before.Format(time.RFC3339), p.Limit)
resources, err := cli.GetAPIResource("search?" + p.String())
if err != nil {
if strings.Contains(fmt.Sprintf("%v", err), "HTTP 404") {
panic("No results found for search query: " + p.String())
}
panic(err)
}
switch sType {
case "agent":
agents, err := filterAgentItems(sp, items, cli)
if err != nil {
panic(err)
}
fmt.Println("---- ID ---- + ---- Name ---- + -- Status -- + -- Last Heartbeat --")
for _, agt := range agents {
name := agt.Name
if len(name) < 30 {
for i := len(name); i < 30; i++ {
name += " "
}
}
if len(name) > 30 {
name = name[0:27] + "..."
}
status := agt.Status
if len(status) < 12 {
for i := len(status); i < 12; i++ {
status += " "
}
}
if len(status) > 12 {
status = status[0:12]
}
fmt.Printf("%20.0f %s %s %s\n", agt.ID, name, status, agt.HeartBeatTS.Format(time.RFC3339))
}
case "action", "command":
fmt.Println("---- ID ---- + ---- Name ---- + --- Last Updated ---")
for _, item := range items {
for _, data := range item.Data {
if data.Name != sType {
continue
}
switch data.Name {
case "action":
idstr, name, datestr, _, _, err := actionPrintShort(data.Value)
if err != nil {
panic(err)
}
fmt.Printf("%s %s %s\n", idstr, name, datestr)
case "command":
cmd, err := client.ValueToCommand(data.Value)
if err != nil {
panic(err)
}
name := cmd.Action.Name
if len(name) < 30 {
for i := len(name); i < 30; i++ {
name += " "
}
}
if len(name) > 30 {
name = name[0:27] + "..."
}
fmt.Printf("%20.0f %s %s\n", cmd.ID, name, cmd.FinishTime.Format(time.RFC3339))
}
}
}
fmt.Println("--- ID ---- + ---- Name ---- + -- Status -- + -- Last Heartbeat --")
case "action":
fmt.Println("----- ID ----- + -------- Action Name ------- + ----------- Target ---------- + ---- Investigators ---- + - Sent - + - Status - + --- Last Updated --- ")
case "command":
fmt.Println("---- ID ---- + ---- Name ---- + --- Last Updated ---")
case "investigator":
fmt.Println("- ID - + ---- Name ---- + --- Status ---")
for _, item := range items {
for _, data := range item.Data {
if data.Name != sType {
continue
}
for _, item := range resources.Collection.Items {
for _, data := range item.Data {
if data.Name != sType {
continue
}
switch data.Name {
case "action":
idstr, name, target, datestr, invs, status, sent, err := actionPrintShort(data.Value)
if err != nil {
panic(err)
}
switch data.Name {
case "investigator":
inv, err := client.ValueToInvestigator(data.Value)
if err != nil {
panic(err)
}
name := inv.Name
if len(name) < 30 {
for i := len(name); i < 30; i++ {
name += " "
}
}
if len(name) > 30 {
name = name[0:27] + "..."
}
fmt.Printf("%6.0f %s %s\n", inv.ID, name, inv.Status)
fmt.Printf("%s %s %s %s %8d %s %s\n", idstr, name, target, invs, sent,
status, datestr)
case "command":
cmd, err := client.ValueToCommand(data.Value)
if err != nil {
panic(err)
}
name := cmd.Action.Name
if len(name) < 30 {
for i := len(name); i < 30; i++ {
name += " "
}
}
if len(name) > 30 {
name = name[0:27] + "..."
}
fmt.Printf("%14.0f %s %s\n", cmd.ID, name,
cmd.FinishTime.UTC().Format(time.RFC3339))
case "agent":
agt, err := client.ValueToAgent(data.Value)
if err != nil {
panic(err)
}
name := agt.Name
if len(name) < 30 {
for i := len(name); i < 30; i++ {
name += " "
}
}
if len(name) > 30 {
name = name[0:27] + "..."
}
status := agt.Status
if len(status) < 12 {
for i := len(status); i < 12; i++ {
status += " "
}
}
if len(status) > 12 {
status = status[0:12]
}
fmt.Printf("%20.0f %s %s %s\n", agt.ID, name, status,
agt.HeartBeatTS.UTC().Format(time.RFC3339))
case "investigator":
inv, err := client.ValueToInvestigator(data.Value)
if err != nil {
panic(err)
}
name := inv.Name
if len(name) < 30 {
for i := len(name); i < 30; i++ {
name += " "
}
}
if len(name) > 30 {
name = name[0:27] + "..."
}
fmt.Printf("%6.0f %s %s\n", inv.ID, name, inv.Status)
}
}
}
@ -143,14 +194,14 @@ The following search parameters are available:
}
// parseSearchQuery transforms a search string into an API query
func parseSearchQuery(orders []string) (sp searchParameters, err error) {
func parseSearchQuery(orders []string) (p migdbsearch.Parameters, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("parseSearchQuery() -> %v", e)
}
}()
sType := orders[1]
query := "search?type=" + sType
p = migdbsearch.NewParameters()
p.Type = orders[1]
if len(orders) < 4 {
panic("Invalid search syntax. try `search help`.")
}
@ -158,114 +209,64 @@ func parseSearchQuery(orders []string) (sp searchParameters, err error) {
panic(fmt.Sprintf("Expected keyword 'where' after search type. Got '%s'", orders[2]))
}
for _, order := range orders[3:len(orders)] {
if order == "and" || order == "or" {
if order == "and" {
continue
}
params := strings.Split(order, "=")
if len(params) != 2 {
panic(fmt.Sprintf("Invalid `key=value` for in parameter '%s'", order))
panic(fmt.Sprintf("Invalid `key=value` in search parameter '%s'", order))
}
key := params[0]
value := params[1]
// if the string contains % characters, used in postgres's pattern matching,
// escape them properly
value := strings.Replace(params[1], "%", "%25", -1)
// wildcards are converted to postgres's % pattern matching
value = strings.Replace(value, "*", "%25", -1)
switch key {
case "and", "or":
continue
case "agentname":
query += "&agentname=" + value
case "after":
query += "&after=" + value
case "before":
query += "&before=" + value
case "id":
panic("If you already know the ID, don't use the search. Use (action|command|agent) <id> directly")
case "actionname":
p.ActionName = value
case "actionid":
query += "&actionid=" + value
case "commandid":
query += "&commandid=" + value
p.ActionID = value
case "after":
p.After, err = time.Parse(time.RFC3339, value)
if err != nil {
panic("after date not in RFC3339 format, ex: 2015-09-23T14:14:16Z")
}
case "agentid":
query += "&agentid=" + value
case "name":
switch sType {
case "action", "command":
query += "&actionname=" + value
case "agent":
query += "&agentname=" + value
p.AgentID = value
case "agentname":
p.AgentName = value
case "agentversion":
p.AgentVersion = value
case "before":
p.Before, err = time.Parse(time.RFC3339, value)
if err != nil {
panic("before date not in RFC3339 format, ex: 2015-09-23T14:14:16Z")
}
case "commandid":
p.CommandID = value
case "investigatorid":
p.InvestigatorID = value
case "investigatorname":
p.InvestigatorName = value
case "limit":
p.Limit, err = strconv.ParseFloat(value, 64)
if err != nil {
panic("invalid limit parameter")
}
case "status":
switch sType {
case "action":
panic("'status' is not a valid action search parameter")
case "command", "agent":
query += "&status=" + value
p.Status = value
case "name":
switch p.Type {
case "action", "command":
p.ActionName = value
case "agent":
p.AgentName = value
case "investigator":
p.InvestigatorName = value
}
case "limit":
query += "&limit=" + value
case "version":
if sType != "agent" {
panic("'version' is only valid when searching for agents")
}
sp.version = value
default:
panic(fmt.Sprintf("Unknown search key '%s'", key))
}
}
sp.sType = sType
sp.query = query
return
}
// runSearchQuery executes a search string against the API
func runSearchQuery(sp searchParameters, cli client.Client) (items []cljs.Item, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("runSearchQuery() -> %v", e)
}
}()
fmt.Println("Search query:", sp.query)
target := sp.query
resource, err := cli.GetAPIResource(target)
if err != nil {
panic(err)
}
items = resource.Collection.Items
return
}
func filterAgentItems(sp searchParameters, items []cljs.Item, cli client.Client) (agents []mig.Agent, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("filterAgentItems() -> %v", e)
}
}()
for _, item := range items {
for _, data := range item.Data {
if data.Name != sp.sType {
continue
}
switch sp.sType {
case "agent":
agt, err := client.ValueToAgent(data.Value)
if err != nil {
panic(err)
}
if sp.version != "" {
tests := strings.Split(sp.version, "%")
for _, test := range tests {
if !strings.Contains(agt.Version, test) {
// this agent doesn't have the version we are looking for, skip it
goto skip
}
}
}
agents = append(agents, agt)
}
skip:
}
}
return
}

26
database/actions.go
Просмотреть файл

@ -10,9 +10,10 @@ import (
"database/sql"
"encoding/json"
"fmt"
"mig.ninja/mig"
"time"
"mig.ninja/mig"
_ "github.com/lib/pq"
)
@ -22,8 +23,10 @@ func (db *DB) LastActions(limit int) (actions []mig.Action, err error) {
validfrom, expireafter, starttime, finishtime, lastupdatetime,
status, pgpsignatures, syntaxversion
FROM actions ORDER BY starttime DESC LIMIT $1`, limit)
if rows != nil {
defer rows.Close()
}
if err != nil && err != sql.ErrNoRows {
rows.Close()
err = fmt.Errorf("Error while listing actions: '%v'", err)
return
}
@ -34,31 +37,26 @@ func (db *DB) LastActions(limit int) (actions []mig.Action, err error) {
&jDesc, &jThreat, &jOps, &a.ValidFrom, &a.ExpireAfter,
&a.StartTime, &a.FinishTime, &a.LastUpdateTime, &a.Status, &jSig, &a.SyntaxVersion)
if err != nil {
rows.Close()
err = fmt.Errorf("Error while retrieving action: '%v'", err)
return
}
err = json.Unmarshal(jDesc, &a.Description)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action description: '%v'", err)
return
}
err = json.Unmarshal(jThreat, &a.Threat)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action threat: '%v'", err)
return
}
err = json.Unmarshal(jOps, &a.Operations)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action operations: '%v'", err)
return
}
err = json.Unmarshal(jSig, &a.PGPSignatures)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action signatures: '%v'", err)
return
}
@ -235,8 +233,10 @@ func (db *DB) InsertSignature(aid, iid float64, sig string) (err error) {
func (db *DB) GetActionCounters(aid float64) (counters mig.ActionCounters, err error) {
rows, err := db.c.Query(`SELECT DISTINCT(status), COUNT(id) FROM commands
WHERE actionid = $1 GROUP BY status`, aid)
if rows != nil {
defer rows.Close()
}
if err != nil && err != sql.ErrNoRows {
rows.Close()
err = fmt.Errorf("Error while retrieving counters: '%v'", err)
return
}
@ -245,7 +245,6 @@ func (db *DB) GetActionCounters(aid float64) (counters mig.ActionCounters, err e
var status string
err = rows.Scan(&status, &count)
if err != nil {
rows.Close()
err = fmt.Errorf("Error while retrieving counter: '%v'", err)
}
switch status {
@ -288,8 +287,10 @@ func (db *DB) SetupRunnableActions() (actions []mig.Action, err error) {
WHERE status='pending' AND validfrom < NOW() AND expireafter > NOW()
RETURNING id, name, target, description, threat, operations,
validfrom, expireafter, status, pgpsignatures, syntaxversion`)
if rows != nil {
defer rows.Close()
}
if err != nil && err != sql.ErrNoRows {
rows.Close()
err = fmt.Errorf("Error while setting up runnable actions: '%v'", err)
return
}
@ -299,31 +300,26 @@ func (db *DB) SetupRunnableActions() (actions []mig.Action, err error) {
err = rows.Scan(&a.ID, &a.Name, &a.Target, &jDesc, &jThreat, &jOps,
&a.ValidFrom, &a.ExpireAfter, &a.Status, &jSig, &a.SyntaxVersion)
if err != nil {
rows.Close()
err = fmt.Errorf("Error while retrieving action: '%v'", err)
return
}
err = json.Unmarshal(jDesc, &a.Description)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action description: '%v'", err)
return
}
err = json.Unmarshal(jThreat, &a.Threat)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action threat: '%v'", err)
return
}
err = json.Unmarshal(jOps, &a.Operations)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action operations: '%v'", err)
return
}
err = json.Unmarshal(jSig, &a.PGPSignatures)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action signatures: '%v'", err)
return
}

56
database/agents.go
Просмотреть файл

@ -10,9 +10,10 @@ import (
"database/sql"
"encoding/json"
"fmt"
"mig.ninja/mig"
"time"
"mig.ninja/mig"
_ "github.com/lib/pq"
)
@ -66,6 +67,9 @@ func (db *DB) AgentsActiveSince(pointInTime time.Time) (agents []mig.Agent, err
rows, err := db.c.Query(`SELECT DISTINCT(agents.queueloc), agents.name FROM agents
WHERE agents.heartbeattime >= $1 AND agents.heartbeattime <= NOW()
GROUP BY agents.queueloc, agents.name`, pointInTime)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while finding agents: '%v'", err)
return
@ -74,7 +78,6 @@ func (db *DB) AgentsActiveSince(pointInTime time.Time) (agents []mig.Agent, err
var agent mig.Agent
err = rows.Scan(&agent.QueueLoc, &agent.Name)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve agent data: '%v'", err)
return
}
@ -127,6 +130,9 @@ func (db *DB) ListMultiAgentsQueues(pointInTime time.Time) (queues []string, err
rows, err := db.c.Query(`SELECT queueloc FROM agents
WHERE heartbeattime > $1 AND mode != 'checkin'
GROUP BY queueloc HAVING COUNT(queueloc) > 1`, pointInTime)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while listing multi agents queues: '%v'", err)
return
@ -135,7 +141,6 @@ func (db *DB) ListMultiAgentsQueues(pointInTime time.Time) (queues []string, err
var q string
err = rows.Scan(&q)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve agent queue: '%v'", err)
return
}
@ -151,6 +156,9 @@ func (db *DB) ListMultiAgentsQueues(pointInTime time.Time) (queues []string, err
func (db *DB) ActiveAgentsByQueue(queueloc string, pointInTime time.Time) (agents []mig.Agent, err error) {
rows, err := db.c.Query(`SELECT id, name, queueloc, mode, version, pid, starttime, heartbeattime, status
FROM agents WHERE agents.heartbeattime > $1 AND agents.queueloc=$2`, pointInTime, queueloc)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while finding agents: '%v'", err)
return
@ -160,7 +168,6 @@ func (db *DB) ActiveAgentsByQueue(queueloc string, pointInTime time.Time) (agent
err = rows.Scan(&agent.ID, &agent.Name, &agent.QueueLoc, &agent.Mode, &agent.Version,
&agent.PID, &agent.StartTime, &agent.HeartBeatTS, &agent.Status)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve agent data: '%v'", err)
return
}
@ -175,7 +182,6 @@ func (db *DB) ActiveAgentsByQueue(queueloc string, pointInTime time.Time) (agent
// ActiveAgentsByTarget runs a search for all agents that match a given target string.
// For safety, it does so in a transaction that runs as a readonly user.
func (db *DB) ActiveAgentsByTarget(target string) (agents []mig.Agent, err error) {
var jTags, jEnv []byte
// save current user
var dbuser string
err = db.c.QueryRow("SELECT CURRENT_USER").Scan(&dbuser)
@ -191,10 +197,12 @@ func (db *DB) ActiveAgentsByTarget(target string) (agents []mig.Agent, err error
_ = txn.Rollback()
return
}
rows, err := txn.Query(fmt.Sprintf(`SELECT DISTINCT ON (queueloc) id, name, queueloc, version, pid, status,
environment, tags, mode, starttime, heartbeattime, destructiontime
rows, err := txn.Query(fmt.Sprintf(`SELECT DISTINCT ON (queueloc) id, name, queueloc, pid, mode
FROM agents WHERE agents.status IN ('%s', '%s') AND (%s)
ORDER BY agents.queueloc, agents.heartbeattime DESC`, mig.AgtStatusOnline, mig.AgtStatusIdle, target))
ORDER BY agents.queueloc ASC`, mig.AgtStatusOnline, mig.AgtStatusIdle, target))
if rows != nil {
defer rows.Close()
}
if err != nil {
_ = txn.Rollback()
err = fmt.Errorf("Error while finding agents: '%v'", err)
@ -202,23 +210,11 @@ func (db *DB) ActiveAgentsByTarget(target string) (agents []mig.Agent, err error
}
for rows.Next() {
var agent mig.Agent
err = rows.Scan(&agent.ID, &agent.Name, &agent.QueueLoc, &agent.Version, &agent.PID, &agent.Status,
&jEnv, &jTags, &agent.Mode, &agent.StartTime, &agent.HeartBeatTS, &agent.DestructionTime)
err = rows.Scan(&agent.ID, &agent.Name, &agent.QueueLoc, &agent.PID, &agent.Mode)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve agent data: '%v'", err)
return
}
err = json.Unmarshal(jTags, &agent.Tags)
if err != nil {
err = fmt.Errorf("failed to unmarshal agent tags")
return
}
err = json.Unmarshal(jEnv, &agent.Env)
if err != nil {
err = fmt.Errorf("failed to unmarshal agent environment")
return
}
agents = append(agents, agent)
}
if err := rows.Err(); err != nil {
@ -266,6 +262,9 @@ func (db *DB) GetAgentsStats(limit int) (stats []mig.AgentsStats, err error) {
online_endpoints, idle_agents, idle_agents_by_version, idle_endpoints, new_endpoints,
multi_agents_endpoints, disappeared_endpoints, flapping_endpoints
FROM agents_stats ORDER BY timestamp DESC LIMIT $1`, limit)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while retrieving agent statistics: '%v'", err)
return
@ -277,19 +276,16 @@ func (db *DB) GetAgentsStats(limit int) (stats []mig.AgentsStats, err error) {
&s.IdleAgents, &jIdlAgtVer, &s.IdleEndpoints, &s.NewEndpoints,
&s.MultiAgentsEndpoints, &s.DisappearedEndpoints, &s.FlappingEndpoints)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve agent statistics data: '%v'", err)
return
}
err = json.Unmarshal(jOnlAgtVer, &s.OnlineAgentsByVersion)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal online agent by version statistics: '%v'", err)
return
}
err = json.Unmarshal(jIdlAgtVer, &s.IdleAgentsByVersion)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal idle agent by version statistics: '%v'", err)
return
}
@ -331,6 +327,9 @@ func (db *DB) StoreAgentsStats(stats mig.AgentsStats) (err error) {
func (db *DB) SumOnlineAgentsByVersion() (sum []mig.AgentsVersionsSum, err error) {
rows, err := db.c.Query(`SELECT COUNT(*), version FROM agents
WHERE agents.status=$1 GROUP BY version`, mig.AgtStatusOnline)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while counting agents: '%v'", err)
return
@ -339,7 +338,6 @@ func (db *DB) SumOnlineAgentsByVersion() (sum []mig.AgentsVersionsSum, err error
var asum mig.AgentsVersionsSum
err = rows.Scan(&asum.Count, &asum.Version)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve summary data: '%v'", err)
return
}
@ -359,6 +357,9 @@ func (db *DB) SumIdleAgentsByVersion() (sum []mig.AgentsVersionsSum, err error)
SELECT distinct(queueloc) FROM agents
WHERE agents.status=$2)
GROUP BY version`, mig.AgtStatusIdle, mig.AgtStatusOnline)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while counting agents: '%v'", err)
return
@ -367,7 +368,6 @@ func (db *DB) SumIdleAgentsByVersion() (sum []mig.AgentsVersionsSum, err error)
var asum mig.AgentsVersionsSum
err = rows.Scan(&asum.Count, &asum.Version)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve summary data: '%v'", err)
return
}
@ -491,6 +491,9 @@ func (db *DB) GetDisappearedEndpoints(oldest time.Time) (queues []string, err er
GROUP BY queueloc
)
GROUP BY queueloc`, oldest)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while retrieving disappeared endpoints: '%v'", err)
return
@ -499,7 +502,6 @@ func (db *DB) GetDisappearedEndpoints(oldest time.Time) (queues []string, err er
var q string
err = rows.Scan(&q)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve endpoint queue: '%v'", err)
return
}

13
database/commands.go
Просмотреть файл

@ -10,9 +10,10 @@ import (
"database/sql"
"encoding/json"
"fmt"
"mig.ninja/mig"
"time"
"mig.ninja/mig"
_ "github.com/lib/pq"
)
@ -74,6 +75,9 @@ func (db *DB) CommandsByActionID(actionid float64) (commands []mig.Command, err
agents.id, agents.name, agents.version
FROM commands, actions, agents
WHERE commands.actionid=actions.id AND commands.agentid=agents.id AND actions.id=$1`, actionid)
if rows != nil {
defer rows.Close()
}
if err != nil {
err = fmt.Errorf("Error while finding commands: '%v'", err)
return
@ -86,37 +90,31 @@ func (db *DB) CommandsByActionID(actionid float64) (commands []mig.Command, err
&cmd.Action.ValidFrom, &cmd.Action.ExpireAfter, &jSig, &cmd.Action.SyntaxVersion,
&cmd.Agent.ID, &cmd.Agent.Name, &cmd.Agent.Version)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve command: '%v'", err)
return
}
err = json.Unmarshal(jRes, &cmd.Results)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal command results: '%v'", err)
return
}
err = json.Unmarshal(jDesc, &cmd.Action.Description)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action description: '%v'", err)
return
}
err = json.Unmarshal(jThreat, &cmd.Action.Threat)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action threat: '%v'", err)
return
}
err = json.Unmarshal(jOps, &cmd.Action.Operations)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action operations: '%v'", err)
return
}
err = json.Unmarshal(jSig, &cmd.Action.PGPSignatures)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to unmarshal action signatures: '%v'", err)
return
}
@ -164,6 +162,7 @@ func (db *DB) InsertCommands(cmds []mig.Command) (insertCount int64, err error)
step += 6
}
stmt, err := db.c.Prepare(sql)
defer stmt.Close()
if err != nil {
err = fmt.Errorf("Error while preparing insertion statement: '%v' in '%s'", err, sql)
return

13
database/investigators.go
Просмотреть файл

@ -9,16 +9,19 @@ package database /* import "mig.ninja/mig/database" */
import (
"database/sql"
"fmt"
"time"
_ "github.com/lib/pq"
"mig.ninja/mig"
"time"
)
// ActiveInvestigators returns a slice of investigators keys marked as active
func (db *DB) ActiveInvestigatorsKeys() (keys [][]byte, err error) {
rows, err := db.c.Query("SELECT publickey FROM investigators WHERE status='active'")
if rows != nil {
defer rows.Close()
}
if err != nil && err != sql.ErrNoRows {
rows.Close()
err = fmt.Errorf("Error while listing active investigators keys: '%v'", err)
return
}
@ -29,7 +32,6 @@ func (db *DB) ActiveInvestigatorsKeys() (keys [][]byte, err error) {
var key []byte
err = rows.Scan(&key)
if err != nil {
rows.Close()
err = fmt.Errorf("Error while retrieving investigator key: '%v'", err)
return
}
@ -81,8 +83,10 @@ func (db *DB) InvestigatorByActionID(aid float64) (invs []mig.Investigator, err
FROM investigators, signatures
WHERE signatures.actionid=$1
AND signatures.investigatorid=investigators.id`, aid)
if rows != nil {
defer rows.Close()
}
if err != nil && err != sql.ErrNoRows {
rows.Close()
err = fmt.Errorf("Error while finding investigator: '%v'", err)
return
}
@ -90,7 +94,6 @@ func (db *DB) InvestigatorByActionID(aid float64) (invs []mig.Investigator, err
var inv mig.Investigator
err = rows.Scan(&inv.ID, &inv.Name, &inv.PGPFingerprint, &inv.Status, &inv.CreatedAt, &inv.LastModified)
if err != nil {
rows.Close()
err = fmt.Errorf("Failed to retrieve investigator data: '%v'", err)
return
}

103
database/search/parameters.go Normal file
Просмотреть файл

@ -0,0 +1,103 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package search /* import "mig.ninja/mig/database/search" */
import (
"fmt"
"strings"
"time"
)
// SearchParameters contains fields used to perform database searches
type Parameters struct {
ActionID string `json:"actionid"`
ActionName string `json:"actionname"`
After time.Time `json:"after"`
AgentID string `json:"agentid"`
AgentName string `json:"agentname"`
AgentVersion string `json:"agentversion"`
Before time.Time `json:"before"`
CommandID string `json:"commandid"`
FoundAnything bool `json:"foundanything"`
InvestigatorID string `json:"investigatorid"`
InvestigatorName string `json:"investigatorname"`
Limit float64 `json:"limit"`
Offset float64 `json:"offset"`
Report string `json:"report"`
Status string `json:"status"`
Target string `json:"target"`
ThreatFamily string `json:"threatfamily"`
Type string `json:"type"`
}
// by default, search all records 10 years prior and after today
const DefaultWindow time.Duration = 39600 * time.Hour
// NewParameters initializes search parameters
func NewParameters() (p Parameters) {
p.ActionID = "∞"
p.ActionName = "%"
p.After = time.Now().Add(-DefaultWindow).UTC()
p.AgentID = "∞"
p.AgentName = "%"
p.AgentVersion = "%"
p.Before = time.Now().Add(DefaultWindow).UTC()
p.CommandID = "∞"
p.InvestigatorID = "∞"
p.InvestigatorName = "%"
p.Limit = 100
p.Offset = 0
p.Status = "%"
p.ThreatFamily = "%"
p.Type = "action"
return
}
// String() returns a query string with the current search parameters
func (p Parameters) String() (query string) {
query = fmt.Sprintf("type=%s&after=%s&before=%s", p.Type, p.After.Format(time.RFC3339), p.Before.Format(time.RFC3339))
if p.ActionID != "∞" {
query += fmt.Sprintf("&actionid=%s", p.ActionID)
}
if p.ActionName != "%" {
query += fmt.Sprintf("&actionname=%s", p.ActionName)
}
if p.AgentID != "∞" {
query += fmt.Sprintf("&agentid=%s", p.AgentID)
}
if p.AgentName != "%" {
query += fmt.Sprintf("&agentname=%s", p.AgentName)
}
if p.AgentVersion != "%" {
query += fmt.Sprintf("&agentversion=%s", p.AgentVersion)
}
if p.CommandID != "∞" {
query += fmt.Sprintf("&commandid=%s", p.CommandID)
}
if p.InvestigatorID != "∞" {
query += fmt.Sprintf("&investigatorid=%s", p.InvestigatorID)
}
if p.InvestigatorName != "%" {
query += fmt.Sprintf("&investigatorname=%s", p.InvestigatorName)
}
query += fmt.Sprintf("&limit=%.0f", p.Limit)
if p.Offset != 0 {
query += fmt.Sprintf("&offset=%.0f", p.Offset)
}
if p.Status != "%" {
query += fmt.Sprintf("&status=%s", p.Status)
}
if p.ThreatFamily != "%" {
query += fmt.Sprintf("&threatfamily=%s", p.ThreatFamily)
}
// urlencode % and * wildcard characters
query = strings.Replace(query, "%", "%25", -1)
query = strings.Replace(query, "*", "%25", -1)
// replace + character with a wildcard
query = strings.Replace(query, "+", "%25", -1)
return
}

1167
database/searches.go
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

21
doc/api.rst
Просмотреть файл

@ -944,23 +944,24 @@ GET /api/v1/search
- `actionname`: filter results on string action name, accept `ILIKE` pattern
- `after`: return results recorded after this RFC3339 date, depends on type:
- `after`: return results recorded after this RFC3339 date. If not set,
return results for last 10 years. Impact on search depends on the type:
- `action`: select actions with a `validfrom` date greater than
`after`. Default is last 7 days.
- `action`: select actions with a `validfrom` date greater than `after`.
- `agent`: select agents that have sent a heartbeat since `after`.
Default is last 7 days.
- `command`: select commands with a `starttime` date greated than
`after`. Default is last 7 days.
- `command`: select commands with a `starttime` date greated than `after`.
- `investigator`: select investigators with a `createdat` date greater
than `after`. Default is last 1,000 years.
than `after`.
- `agentid`: filter results on the agent ID
- `agentname`: filter results on string agent name, accept `ILIKE` pattern
- `before`: return results recorded before this RFC3339 date. If not defined,
default is to retrieve results until now.
- `agentversion`: filter results on agent version string, accept `ILIKE` pattern
- `before`: return results recorded before this RFC3339 date. If not set,
return results for the next 10 years. Impact on search depends on the
type:
- `action`: select actions with a `expireafter` date lower than `before`
- `agent`: select agents that have sent a heartbeat priot to `before`
@ -978,7 +979,7 @@ GET /api/v1/search
- `investigatorname`: filter results on string investigator name, accept
`ILIKE` pattern
- `limit`: limit the number of results to 10,000 by default
- `limit`: limit the number of results, default is set to 100
- `offset`: discard the X first results, defaults to 0. Used in conjunction
with `limit`, offset can be used to paginate search results.

13
doc/api.rst.html
Просмотреть файл

@ -931,15 +931,14 @@ investigator's action history.</p></li><li><p>Authentication: X-PGPAUTHORIZATION
<span class="punctuation">}</span>
<span class="punctuation">}</span></code></pre></section><section id="post-api-v1-investigator-create"><header><h3><a href="#id16">2.9   POST /api/v1/investigator/create/</a></h3></header><ul><li><p>Description: create a new investigator in the database</p></li><li><p>Authentication: X-PGPAUTHORIZATION</p></li><li><dl><dt>Parameters: (POST body)</dt><dd><ul><li><p><cite>name</cite>: string that represents the full name</p></li><li><p><cite>publickey</cite>: armored GPG public key</p></li></ul></dd></dl></li><li><p>Response Code: 201 Created</p></li><li><p>Response: Collection+JSON</p></li><li><p>Example: (without authentication)</p></li></ul><pre><code class="code bash"><span class="name variable">$ </span>gpg --export -a --export-options <span class="name builtin">export</span>-minimal bob_kelso@example.net &gt; /tmp/bobpubkey
<span class="name variable">$ </span>curl -iv -F <span class="literal string double">"name=Bob Kelso"</span> -F <span class="name variable">publickey</span><span class="operator">=</span>@/tmp/pubkey https://api.mig.example.net/api/v1/investigator/create/</code></pre></section><section id="post-api-v1-investigator-update"><header><h3><a href="#id17">2.10   POST /api/v1/investigator/update/</a></h3></header><ul><li><p>Description: update an existing investigator in the database</p></li><li><p>Authentication: X-PGPAUTHORIZATION</p></li><li><dl><dt>Parameters: (POST body)</dt><dd><ul><li><p><cite>id</cite>: investigator id, to identify the target investigator</p></li><li><p><cite>status</cite>: new status of the investigator, to be updated</p></li></ul></dd></dl></li><li><p>Response Code: 201 Created</p></li><li><p>Response: Collection+JSON</p></li><li><p>Example: (without authentication)</p></li></ul><pre><code class="code bash"><span class="name variable">$ </span>curl -iv -X POST -d <span class="name variable">id</span><span class="operator">=</span><span class="literal number">1234</span> -d <span class="name variable">status</span><span class="operator">=</span>disabled https://api.mig.example.net/api/v1/investigator/update/</code></pre></section><section id="get-api-v1-search"><header><h3><a href="#id18">2.11   GET /api/v1/search</a></h3></header><ul><li><p>Description: search for actions, commands, agents or investigators.</p></li><li><p>Authentication: X-PGPAUTHORIZATION</p></li><li><p>Response Code: 200 OK</p></li><li><p>Response: Collection+JSON</p></li><li><dl><dt>Parameters:</dt><dd><ul><li><p><cite>type</cite>: define the type of item returned by the search.
Valid types are: <cite>action</cite>, <cite>command</cite>, <cite>agent</cite> or <cite>investigator</cite>.</p><blockquote><ul><li><p><cite>action</cite>: (default) return a list of actions</p></li><li><p><cite>command</cite>: return a list of commands</p></li><li><p><cite>agent</cite>: return a list of agents that have shown activity</p></li><li><p><cite>investigator</cite>: return a list of investigators that have show activity</p></li></ul></blockquote></li><li><p><cite>actionid</cite>: filter results on numeric action ID</p></li><li><p><cite>actionname</cite>: filter results on string action name, accept <cite>ILIKE</cite> pattern</p></li><li><p><cite>after</cite>: return results recorded after this RFC3339 date, depends on type:</p><blockquote><ul><li><p><cite>action</cite>: select actions with a <cite>validfrom</cite> date greater than
<cite>after</cite>. Default is last 7 days.</p></li><li><p><cite>agent</cite>: select agents that have sent a heartbeat since <cite>after</cite>.
Default is last 7 days.</p></li><li><p><cite>command</cite>: select commands with a <cite>starttime</cite> date greated than
<cite>after</cite>. Default is last 7 days.</p></li><li><p><cite>investigator</cite>: select investigators with a <cite>createdat</cite> date greater
than <cite>after</cite>. Default is last 1,000 years.</p></li></ul></blockquote></li><li><p><cite>agentid</cite>: filter results on the agent ID</p></li><li><p><cite>agentname</cite>: filter results on string agent name, accept <cite>ILIKE</cite> pattern</p></li><li><p><cite>before</cite>: return results recorded before this RFC3339 date. If not defined,
default is to retrieve results until now.</p><blockquote><ul><li><p><cite>action</cite>: select actions with a <cite>expireafter</cite> date lower than <cite>before</cite></p></li><li><p><cite>agent</cite>: select agents that have sent a heartbeat priot to <cite>before</cite></p></li><li><p><cite>command</cite>: select commands with a <cite>starttime</cite> date lower than <cite>before</cite></p></li><li><p><cite>investigator</cite>: select investigators with a <cite>lastmodified</cite> date lower
Valid types are: <cite>action</cite>, <cite>command</cite>, <cite>agent</cite> or <cite>investigator</cite>.</p><blockquote><ul><li><p><cite>action</cite>: (default) return a list of actions</p></li><li><p><cite>command</cite>: return a list of commands</p></li><li><p><cite>agent</cite>: return a list of agents that have shown activity</p></li><li><p><cite>investigator</cite>: return a list of investigators that have show activity</p></li></ul></blockquote></li><li><p><cite>actionid</cite>: filter results on numeric action ID</p></li><li><p><cite>actionname</cite>: filter results on string action name, accept <cite>ILIKE</cite> pattern</p></li><li><p><cite>after</cite>: return results recorded after this RFC3339 date. If not set,
return results for last 10 years. Impact on search depends on the type:</p><blockquote><ul><li><p><cite>action</cite>: select actions with a <cite>validfrom</cite> date greater than <cite>after</cite>.</p></li><li><p><cite>agent</cite>: select agents that have sent a heartbeat since <cite>after</cite>.</p></li><li><p><cite>command</cite>: select commands with a <cite>starttime</cite> date greated than <cite>after</cite>.</p></li><li><p><cite>investigator</cite>: select investigators with a <cite>createdat</cite> date greater
than <cite>after</cite>.</p></li></ul></blockquote></li><li><p><cite>agentid</cite>: filter results on the agent ID</p></li><li><p><cite>agentname</cite>: filter results on string agent name, accept <cite>ILIKE</cite> pattern</p></li><li><p><cite>agentversion</cite>: filter results on agent version string, accept <cite>ILIKE</cite> pattern</p></li><li><p><cite>before</cite>: return results recorded before this RFC3339 date. If not set,
return results for the next 10 years. Impact on search depends on the
type:</p><blockquote><ul><li><p><cite>action</cite>: select actions with a <cite>expireafter</cite> date lower than <cite>before</cite></p></li><li><p><cite>agent</cite>: select agents that have sent a heartbeat priot to <cite>before</cite></p></li><li><p><cite>command</cite>: select commands with a <cite>starttime</cite> date lower than <cite>before</cite></p></li><li><p><cite>investigator</cite>: select investigators with a <cite>lastmodified</cite> date lower
than <cite>before</cite></p></li></ul></blockquote></li><li><p><cite>commandid</cite>: filter results on the command ID</p></li><li><p><cite>foundanything</cite>: filter commands on the <cite>foundanything</cite> boolean of their
results (only for type <cite>command</cite>, as it requires looking into results)</p></li><li><p><cite>investigatorid</cite>: filter results on the investigator ID</p></li><li><p><cite>investigatorname</cite>: filter results on string investigator name, accept
<cite>ILIKE</cite> pattern</p></li><li><p><cite>limit</cite>: limit the number of results to 10,000 by default</p></li><li><p><cite>offset</cite>: discard the X first results, defaults to 0. Used in conjunction
<cite>ILIKE</cite> pattern</p></li><li><p><cite>limit</cite>: limit the number of results, default is set to 100</p></li><li><p><cite>offset</cite>: discard the X first results, defaults to 0. Used in conjunction
with <cite>limit</cite>, offset can be used to paginate search results.
ex: <strong>&amp;limit=10&amp;offset=50</strong> will grab 10 results discarding the first 50.</p></li><li><dl><dt><cite>report</cite>: if set, return results in the given report format:</dt><dd><ul><li><p><cite>complianceitems</cite> returns command results as compliance items</p></li><li><p><cite>geolocations</cite> returns command results as geolocation endpoints</p></li></ul></dd></dl></li><li><p><cite>status</cite>: filter on internal status, accept <cite>ILIKE</cite> pattern.
Status depends on the type. Below are the available statuses per type:</p><blockquote><ul><li><p><cite>action</cite>: pending, scheduled, preparing, invalid, inflight, completed</p></li><li><p><cite>agent</cite>: online, upgraded, destroyed, offline, idle</p></li><li><p><cite>command</cite>: prepared, sent, success, timeout, cancelled, expired, failed</p></li><li><p><cite>investigator</cite>: active, disabled</p></li></ul></blockquote></li><li><p><cite>target</cite>: returns agents that match a target query (only for <cite>agent</cite> type)</p></li><li><p><cite>threatfamily</cite>: filter results of the threat family of the action, accept

15
mig-api/search.go
Просмотреть файл

@ -7,14 +7,15 @@ package main
import (
"fmt"
"github.com/jvehent/cljs"
"mig.ninja/mig"
migdb "mig.ninja/mig/database"
"net/http"
"net/url"
"regexp"
"strconv"
"time"
"github.com/jvehent/cljs"
"mig.ninja/mig"
migdbsearch "mig.ninja/mig/database/search"
)
type pagination struct {
@ -27,7 +28,7 @@ type pagination struct {
func search(respWriter http.ResponseWriter, request *http.Request) {
var (
err error
p migdb.SearchParameters
p migdbsearch.Parameters
filterFound bool
)
opid := getOpID(request)
@ -234,13 +235,13 @@ var truere = regexp.MustCompile("(?i)^true$")
var falsere = regexp.MustCompile("(?i)^false$")
// parseSearchParameters transforms a query string into search parameters in the migdb format
func parseSearchParameters(qp url.Values) (p migdb.SearchParameters, filterFound bool, err error) {
func parseSearchParameters(qp url.Values) (p migdbsearch.Parameters, filterFound bool, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("parseSearchParameters()-> %v", e)
}
}()
p = migdb.NewSearchParameters()
p = migdbsearch.NewParameters()
for queryParams, _ := range qp {
switch queryParams {
case "actionname":
@ -256,6 +257,8 @@ func parseSearchParameters(qp url.Values) (p migdb.SearchParameters, filterFound
p.AgentID = qp["agentid"][0]
case "agentname":
p.AgentName = qp["agentname"][0]
case "agentversion":
p.AgentVersion = qp["agentversion"][0]
case "before":
p.Before, err = time.Parse(time.RFC3339, qp["before"][0])
if err != nil {

9
mig-scheduler/scheduler.go
Просмотреть файл

@ -9,13 +9,14 @@ import (
"encoding/json"
"flag"
"fmt"
"os"
"runtime"
"time"
"github.com/streadway/amqp"
"mig.ninja/mig"
"mig.ninja/mig/modules"
"mig.ninja/mig/pgp"
"os"
"runtime"
"time"
)
// build version
@ -182,7 +183,7 @@ func createCommand(ctx Context, action mig.Action, agent mig.Agent, emptyResults
// sendCommand is called when a command file is created in ctx.Directories.Command.Ready
// it read the command, sends it to the agent via AMQP, and update the DB
func sendCommands(cmds []mig.Command, ctx Context) (err error) {
aid := cmds[0].ID
aid := cmds[0].Action.ID
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("sendCommand() -> %v", e)