Implement sending of messages via libcurl

Uses a ring buffer (msg_list) to fill in messages at a max of 512 messages currently.
Then uses select to run curl_multi functions.
This commit is contained in:
Guillaume Destuynder 2014-04-27 09:57:39 -07:00
Родитель 31b10e14e2
Коммит 9f302ea38e
6 изменённых файлов: 296 добавлений и 91 удалений

Просмотреть файл

@ -35,13 +35,13 @@ PREFIX := /usr
all: audisp-json
audisp-json: json-config.o audisp-json.o
${LIBTOOL} --tag=CC --mode=link gcc ${CFLAGS} ${LDFLAGS} ${LIBS} ${DEFINES} -o audisp-json json-config.o audisp-json.o
${LIBTOOL} --tag=CC --mode=link gcc ${CFLAGS} ${LDFLAGS} ${LIBS} -o audisp-json json-config.o audisp-json.o
json-config.o: json-config.c
${GCC} -I. ${CFLAGS} ${LIBS} -c -o json-config.o json-config.c
audisp-json.o: audisp-json.c
${GCC} -I. ${CFLAGS} ${LIBS} -c -o audisp-json.o audisp-json.c
${GCC} -I. ${CFLAGS} ${LIBS} ${DEFINES} -c -o audisp-json.o audisp-json.c
install: audisp-json au-json.conf audisp-json.conf
${INSTALL} -D -m 0644 au-json.conf ${DESTDIR}/${PREFIX}/etc/audisp/plugins.d/au-json.conf

Просмотреть файл

@ -7,15 +7,19 @@ Audisp-json
This program is a plugin for Linux Audit user space programs available at <http://people.redhat.com/sgrubb/audit/>.
It uses the audisp multiplexer.
Audisp-json correlates messages coming from the kernel's audit (and through audisp) into a single JSON message through syslog.
Audisp-json correlates messages coming from the kernel's audit (and through audisp) into a single JSON message that is
sent directly to a log server (it doesn't use syslog).
The JSON format used is MozDef message format.
Regular audit log messages and audisp-json error, info messages still use syslog.
Building
--------
Required dependencies:
- Audit (2.0+)
- libtool
- libcurl
For package building:
- FPM
@ -70,11 +74,13 @@ Example for syslog-ng
destination d_logserver { udp("<SYSLOG_SERVER_IP_HERE>" port(514)); };
log{ source(s_syslog); filter(f_auditd); destination(d_logserver); };
# If you want to "not log" auditd messages, negate the same filter to your other log items
Message handling
----------------
Syscalls are interpreted by audisp-json and transformed into a MozDef JSON message.
This means, for example, all execve() and related calls will be aggregated into a message of type EXECVE.
.. note: MozDef messages are not sent to syslog. They're sent to MozDef directly.
Supported messages are listed in the document messages_format.rst

Просмотреть файл

@ -41,17 +41,26 @@
#include "json-config.h"
#define CONFIG_FILE "/etc/audisp/audisp-json.conf"
#define MAX_JSON_MSG_SIZE 2048
#define CONFIG_FILE_LOCAL "audisp-json.conf"
// after this amount of time for any response (connect, http reply, etc.) just give up
// and lose messages.
// don't set this too high as new curl handles will be created and consume memory while
// waiting for the connection to work again.
#define MAX_CURL_GLOBAL_TIMEOUT 5000L
#define RING_BUF_LEN 512
#define MAX_JSON_MSG_SIZE 4096
#define MAX_ARG_LEN 2048
#define MAX_ATTR_SIZE 1023
#define BUF_SIZE 32
#ifndef PROGRAM_VERSION
#define PROGRAM_VERSION 1
#define PROGRAM_VERSION "1"
#endif
#ifndef PROGRAME_NAME
#define PROGRAM_NAME "audisp-json"
#endif
#define USER_AGENT "PROGRAM_NAME/PROGRAM_VERSION"
/* transform macro int and str value to ... str */
#define _STR(x) #x
#define STR(x) _STR(x)
#define USER_AGENT PROGRAM_NAME"/"STR(PROGRAM_VERSION)
extern int h_errno;
@ -62,8 +71,24 @@ static char *hostname = NULL;
static auparse_state_t *au = NULL;
static int machine = -1;
static long int curl_timeout = -1;
int curl_nr_h = 0;
CURLM *multi_h;
CURL *easy_h;
struct curl_slist *slist1;
typedef struct { char *val; } msg_t;
typedef struct ring_buf_msg {
int size;
int start;
int end;
msg_t *data;
} ring_buf_msg_t;
static ring_buf_msg_t msg_list;
typedef struct ll {
char val[1024];
char val[MAX_ATTR_SIZE];
struct ll *next;
} attr_t;
@ -78,6 +103,127 @@ struct json_msg_type {
struct ll *details;
};
/* ring buffer functions */
int ring_full(ring_buf_msg_t *rb)
{
return (rb->end + 1) % rb->size == rb->start;
}
int ring_empty(ring_buf_msg_t *rb)
{
if ((rb->end-1) == rb->start) {
return 1;
}
return 0;
}
void ring_add(ring_buf_msg_t *rb, char *val)
{
msg_t data = {0};
data.val = val;
rb->data[rb->end] = data;
rb->end = (rb->end + 1) % rb->size;
if (rb->end == rb->start) {
rb->start = (rb->start + 1) % rb->size;
}
}
char *ring_read(ring_buf_msg_t *rb)
{
char *val;
val = rb->data[rb->start].val;
rb->start = (rb->start + 1) % rb->size;
return val;
}
void prepare_curl_handle(void)
{
curl_easy_reset(easy_h);
curl_easy_setopt(easy_h, CURLOPT_URL, config.mozdef_url);
curl_easy_setopt(easy_h, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(easy_h, CURLOPT_USERAGENT, USER_AGENT);
curl_easy_setopt(easy_h, CURLOPT_HTTPHEADER, slist1);
curl_easy_setopt(easy_h, CURLOPT_MAXREDIRS, 10L);
curl_easy_setopt(easy_h, CURLOPT_CUSTOMREQUEST, "POST");
/* keep alive is on by default and only settable in recent libcurl
* keeping this around in case its not actually default in some cases and needs
* to be conditionally enabled
*/
// curl_easy_setopt(easy_h, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(easy_h, CURLOPT_VERBOSE, config.curl_verbose);
curl_easy_setopt(easy_h, CURLOPT_TIMEOUT_MS, MAX_CURL_GLOBAL_TIMEOUT);
curl_easy_setopt(easy_h, CURLOPT_SSL_VERIFYHOST, config.ssl_verify);
curl_easy_setopt(easy_h, CURLOPT_SSL_VERIFYPEER, config.ssl_verify);
}
/* select and fetch urls */
void curl_perform(void)
{
int msgs_left;
int maxfd = -1;
struct timeval timeout;
int rc;
CURLMsg *msg;
CURL *eh;
CURLcode ret;
while (curl_nr_h > 0) {
fd_set r, w, e;
FD_ZERO(&r);
FD_ZERO(&w);
FD_ZERO(&e);
ret = curl_multi_timeout(multi_h, &curl_timeout);
if (ret != CURLM_OK) {
syslog(LOG_ERR, curl_multi_strerror(ret));
}
timeout.tv_sec = 1;
timeout.tv_usec = 0;
if (curl_timeout >= 0) {
timeout.tv_sec = curl_timeout / 1000;
if (timeout.tv_sec > 1)
timeout.tv_sec = 1;
else
timeout.tv_usec = (curl_timeout % 1000) * 1000;
}
ret = curl_multi_fdset(multi_h, &r, &w, &e, &maxfd);
if (ret != CURLM_OK) {
syslog(LOG_ERR, curl_multi_strerror(ret));
return;
}
rc = select(maxfd+1, &r, &w, &e, &timeout);
switch(rc) {
case -1:
syslog(LOG_ERR, strerror(errno));
break;
case 0:
default:
ret = curl_multi_perform(multi_h, &curl_nr_h);
if (ret != CURLM_OK) {
syslog(LOG_ERR, curl_multi_strerror(ret));
}
break;
}
}
/* Cleanup completed handles */
while (msg = curl_multi_info_read(multi_h, &msgs_left)) {
if (msg->msg == CURLMSG_DONE) {
if (!ring_empty(&msg_list)) {
char *msg = ring_read(&msg_list);
prepare_curl_handle();
curl_easy_setopt(easy_h, CURLOPT_POSTFIELDS, msg);
curl_easy_setopt(easy_h, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)strlen(msg));
curl_nr_h++;
}
}
}
}
static void handle_event(auparse_state_t *au,
auparse_cb_event_t cb_event_type, void *user_data);
@ -118,6 +264,7 @@ int main(int argc, char *argv[])
struct sigaction sa;
struct hostent *ht;
char nodename[64];
CURLMcode ret;
sa.sa_flags = 0;
sigemptyset(&sa.sa_mask);
@ -142,18 +289,45 @@ int main(int argc, char *argv[])
}
if (load_config(&config, CONFIG_FILE))
return 1;
if (load_config(&config, CONFIG_FILE_LOCAL))
return 1;
au = auparse_init(AUSOURCE_FEED, 0);
if (au == NULL) {
syslog(LOG_ERR, "could not initialize auparse");
free_config(&config);
return -1;
}
machine = audit_detect_machine();
if (machine < 0)
if (machine < 0) {
return -1;
}
/* libcurl stuff */
msg_list.size = RING_BUF_LEN;
msg_list.start = 0;
msg_list.end = 0;
msg_list.data = (msg_t *)calloc(RING_BUF_LEN, sizeof(msg_t));
if (curl_global_init(CURL_GLOBAL_ALL) != 0) {
syslog(LOG_ERR, "curl_global_init() failed");
return -1;
}
easy_h = curl_easy_init();
multi_h = curl_multi_init();
slist1 = NULL;
slist1 = curl_slist_append(slist1, "Content-Type:application/json");
if (!(easy_h && multi_h && slist1)) {
syslog(LOG_ERR, "cURL handles creation failed, this is fatal.");
return -1;
}
prepare_curl_handle();
ret = curl_multi_add_handle(multi_h, easy_h);
if (ret != CURLM_OK) {
syslog(LOG_ERR, curl_multi_strerror(ret));
return -1;
}
auparse_add_callback(au, handle_event, NULL, NULL);
@ -166,15 +340,23 @@ int main(int argc, char *argv[])
hup==0 && stop==0)
auparse_feed(au, tmp, strnlen(tmp, MAX_AUDIT_MESSAGE_LENGTH));
curl_perform();
if (feof(stdin))
break;
} while (stop == 0);
auparse_flush_feed(au);
while (!ring_empty(&msg_list)) {
curl_perform();
}
auparse_destroy(au);
curl_global_cleanup();
free_config(&config);
syslog(LOG_INFO, "%s unloaded\n", PROGRAM_NAME);
closelog();
auparse_flush_feed(au);
auparse_destroy(au);
free_config(&config);
return 0;
}
@ -198,6 +380,7 @@ static int goto_record_type(auparse_state_t *au, int type)
return -1;
}
/* Removes quotes */
char *unescape(const char *in)
{
char *dst = (char *)in;
@ -206,13 +389,14 @@ char *unescape(const char *in)
char c;
while ((c = *src++) != '\0') {
if (c != '"')
*dst++ = c;
if (c != '"')
*dst++ = c;
}
*dst = '\0';
return s;
}
/* Add a field to the json msg's details={} */
attr_t *json_add_attr(attr_t *list, const char *st, const char *val)
{
attr_t *new;
@ -228,6 +412,17 @@ attr_t *json_add_attr(attr_t *list, const char *st, const char *val)
return new;
}
void json_del_attrs(attr_t *head)
{
attr_t *prev;
while (head) {
prev = head;
head = head->next;
free(prev);
}
}
/* Resolve uid to username */
char *get_username(int uid)
{
size_t bufsize;
@ -254,6 +449,7 @@ char *get_username(int uid)
return name;
}
/* Resolve process name from pid */
char *get_proc_name(int pid)
{
char p[1024];
@ -269,54 +465,13 @@ char *get_proc_name(int pid)
return proc;
}
void json_del_attrs(attr_t *head)
{
attr_t *prev;
while (head) {
prev = head;
head = head->next;
free(prev);
}
}
int syslog_json_msg(struct json_msg_type json_msg)
void syslog_json_msg(struct json_msg_type json_msg)
{
attr_t *head = json_msg.details;
attr_t *prev;
char msg[MAX_JSON_MSG_SIZE];
CURLcode ret;
CURL *hnd;
struct curl_slist *slist1;
char *msg;
slist1 = NULL;
slist1 = curl_slist_append(slist1, "Content-Type:application/json");
hnd = curl_easy_init();
curl_easy_setopt(hnd, CURLOPT_URL, config.mozdef_url);
curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "");
curl_easy_setopt(hnd, CURLOPT_USERAGENT, USER_AGENT);
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);
curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);
/* Here is a list of options the curl code used that cannot get generated
as source easily. You may select to either not use them or implement
them yourself.
CURLOPT_WRITEDATA set to a objectpointer
CURLOPT_WRITEFUNCTION set to a functionpointer
CURLOPT_READDATA set to a objectpointer
CURLOPT_READFUNCTION set to a functionpointer
CURLOPT_SEEKDATA set to a objectpointer
CURLOPT_SEEKFUNCTION set to a functionpointer
CURLOPT_ERRORBUFFER set to a objectpointer
CURLOPT_STDERR set to a objectpointer
CURLOPT_HEADERFUNCTION set to a functionpointer
CURLOPT_HEADERDATA set to a objectpointer
*/
msg = calloc(1, (size_t)MAX_JSON_MSG_SIZE);
snprintf(msg, MAX_JSON_MSG_SIZE,
"{\n\
@ -324,12 +479,12 @@ int syslog_json_msg(struct json_msg_type json_msg)
\"summary\": \"%s\",\n\
\"severity\": \"%s\",\n\
\"hostname\": \"%s\",\n\
\"processid\": \"%u\",\n\
\"processid\": \"%i\",\n\
\"processname\": \"%s\",\n\
\"timestamp\": \"%s\",\n\
\"tags\": [\n\
\"%s\",\n\
\"%u\",\n\
\"%s\",\n\
\"audit\"\n\
],\n\
\"details\": {",
@ -347,22 +502,18 @@ int syslog_json_msg(struct json_msg_type json_msg)
}
}
snprintf(msg+strlen(msg), MAX_JSON_MSG_SIZE, " }\n}");
msg[MAX_JSON_MSG_SIZE] = '\0';
ret = curl_easy_perform(hnd);
curl_easy_cleanup(hnd);
hnd = NULL;
curl_slist_free_all(slist1);
slist1 = NULL;
printf("%s\n", msg);
return (int)ret;
ring_add(&msg_list, msg);
// if you wanna see the json msg...
// printf("%s\n", msg);
}
/* The main event handling, parsing, collerating function */
static void handle_event(auparse_state_t *au,
auparse_cb_event_t cb_event_type, void *user_data)
{
int type, rc, num=0;
time_t au_time;
struct json_msg_type json_msg = {
.category = NULL,
@ -401,7 +552,7 @@ static void handle_event(auparse_state_t *au,
strftime(json_msg.timestamp, 64, "%FT%T%z", tmp);
switch (type) {
case AUDIT_AVC:
case AUDIT_AVC:
argc = auparse_find_field(au, "apparmor");
if (!argc)
return;
@ -425,7 +576,9 @@ static void handle_event(auparse_state_t *au,
goto_record_type(au, type);
if (auparse_find_field(au, "parent"))
json_msg.details = json_add_attr(json_msg.details, "parentprocess", get_proc_name(auparse_get_field_int(au)));
json_msg.details = json_add_attr(json_msg.details, "parentprocess",
get_proc_name(auparse_get_field_int(au)));
goto_record_type(au, type);
if (auparse_find_field(au, "pid"))
@ -536,11 +689,15 @@ static void handle_event(auparse_state_t *au,
goto_record_type(au, type);
if (auparse_find_field(au, "ppid"))
json_msg.details = json_add_attr(json_msg.details, "parentprocess", get_proc_name(auparse_get_field_int(au)));
json_msg.details = json_add_attr(json_msg.details, "parentprocess",
get_proc_name(auparse_get_field_int(au)));
goto_record_type(au, type);
if (auparse_find_field(au, "auid")) {
json_msg.details = json_add_attr(json_msg.details, "originaluser", get_username(auparse_get_field_int(au)));
json_msg.details = json_add_attr(json_msg.details, "originaluser",
get_username(auparse_get_field_int(au)));
json_msg.details = json_add_attr(json_msg.details, "originaluid", auparse_get_field_str(au));
}
goto_record_type(au, type);
@ -587,6 +744,19 @@ static void handle_event(auparse_state_t *au,
}
//This also frees json_msg.details
if (!syslog_json_msg(json_msg))
syslog(LOG_WARNING, "failed to send json message");
syslog_json_msg(json_msg);
/* if we have no traffic going on lets start some
* otherwise, traffic is queued in the select loop at curl_perform()
*/
if (curl_nr_h <= 0) {
char *msg = ring_read(&msg_list);
curl_multi_remove_handle(multi_h, easy_h);
prepare_curl_handle();
curl_easy_setopt(easy_h, CURLOPT_POSTFIELDS, msg);
curl_easy_setopt(easy_h, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)strlen(msg));
curl_nr_h++;
curl_multi_add_handle(multi_h, easy_h);
}
curl_perform();
}

Просмотреть файл

@ -1 +1,3 @@
mozdef_url = https://127.0.0.1:8080/events
ssl_verify = yes
curl_verbose = no

Просмотреть файл

@ -1,5 +1,5 @@
/* json-config.c --
* Copyright (c) 2012 Mozilla Corporation.
* Copyright (c) 2014 Mozilla Corporation.
* Copyright 2008 Red Hat Inc., Durham, North Carolina.
* All Rights Reserved.
*
@ -59,12 +59,16 @@ static int nv_split(char *buf, struct nv_pair *nv);
static const struct kw_pair *kw_lookup(const char *val);
static int server_parser(struct nv_pair *nv, int line,
json_conf_t *config);
static int port_parser(struct nv_pair *nv, int line,
static int ssl_parser(struct nv_pair *nv, int line,
json_conf_t *config);
static int curl_parser(struct nv_pair *nv, int line,
json_conf_t *config);
static const struct kw_pair keywords[] =
{
{"mozdef_url", server_parser, 0 },
{"mozdef_url", server_parser, 0},
{"ssl_verify", ssl_parser, 0},
{"curl_verbose",curl_parser, 0},
};
/*
@ -73,6 +77,7 @@ static const struct kw_pair keywords[] =
void clear_config(json_conf_t *config)
{
config->mozdef_url = NULL;
config->ssl_verify = 1;
}
int load_config(json_conf_t *config, const char *file)
@ -94,8 +99,8 @@ int load_config(json_conf_t *config, const char *file)
return 1;
}
syslog(LOG_WARNING,
"Config file %s doesn't exist, skipping", file);
return 0;
"Config file %s doesn't exist", file);
return 1;
}
fd = rc;
@ -108,12 +113,6 @@ int load_config(json_conf_t *config, const char *file)
close(fd);
return 1;
}
if (st.st_uid != 0) {
syslog(LOG_ERR, "Error - %s isn't owned by root",
file);
close(fd);
return 1;
}
if ((st.st_mode & S_IWOTH) == S_IWOTH) {
syslog(LOG_ERR, "Error - %s is world writable",
file);
@ -279,6 +278,32 @@ static int server_parser(struct nv_pair *nv, int line,
return 0;
}
static int ssl_parser(struct nv_pair *nv, int line,
json_conf_t *config)
{
config->ssl_verify = 1;
if (nv->value) {
if (strncasecmp(nv->value, "no", 2) == 0) {
config->ssl_verify = 0;
}
}
return 0;
}
static int curl_parser(struct nv_pair *nv, int line,
json_conf_t *config)
{
config->curl_verbose = 1;
if (nv->value) {
if (strncasecmp(nv->value, "no", 2) == 0) {
config->curl_verbose = 0;
}
}
return 0;
}
void free_config(json_conf_t *config)
{
free((void *)config->mozdef_url);

Просмотреть файл

@ -1,6 +1,6 @@
/* remote-config.h
* Copyright 2008 Red Hat Inc., Durham, North Carolina.
* Copyright 2012 Mozilla Corporation
* Copyright 2014 Mozilla Corporation
* All Rights Reserved.
*
* This program is free software; you can redistribute it and/or modify
@ -29,6 +29,8 @@
typedef struct json_conf
{
const char *mozdef_url;
int ssl_verify;
int curl_verbose;
} json_conf_t;
void clear_config(json_conf_t *config);