microsoft
/
ModSecurity
зеркало из https://github.com/microsoft/ModSecurity.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

replace protobuf with jsoncons, and print pretty json (#110)

This commit is contained in:
Yanshu 2019-04-18 16:03:48 -07:00 коммит произвёл GitHub
Родитель 7f0eced5bc
Коммит 5f57ea44dc
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
9 изменённых файлов: 84 добавлений и 169 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

13
apache2/Makefile.am
Просмотреть файл

@ -1,17 +1,9 @@
pkglibdir = $(prefix)/lib
pkglib_LTLIBRARIES = mod_security2.la
PROTOS = $(waf_logging/waf_format.proto)
PROTO_OBJS = $(PROTOS:.proto=.pb.o)
PBSRCS = $(waf_logging/waf_format.proto)
PBOBJS = $(PROTOS:.proto=.pb.o)
PBGENS = $(PROTOS:.proto=.pb.cc) $(PROTOS:.proto=.pb.h)
mod_security2_la_SOURCES = acmp.c \
ag_mdb/murmur3.c \
ag_mdb/ag_mdb.cpp \
waf_logging/waf_format.pb.cc \
waf_logging/waf_log_util.cc \
waf_lock/waf_lock.cpp \
apache2_config.c \
@ -203,8 +195,3 @@ install-exec-hook: $(pkglib_LTLIBRARIES)
done
endif
%.pb.cc: %.proto
protoc --cpp_out=. $<
%.pb.o : %.pb.cc
$(CXX) $(CXX_FLAGS) -c -o $@ $<

5
apache2/re_operators.c Normal file → Executable file
Просмотреть файл

@ -1178,8 +1178,11 @@ static int msre_op_rx_execute(modsec_rec *msr, msre_rule *rule, msre_var *var, c
if (rc != PCRE_ERROR_NOMATCH) { /* Match. */
/* We no longer escape the pattern here as it is done when logging */
#ifdef WAF_JSON_LOGGING_ENABLE
char *pattern = apr_pstrdup(msr->mp, regex->pattern ? regex->pattern : "<Unknown Match>");
#else
char *pattern = apr_pstrdup(msr->mp, log_escape(msr->mp, regex->pattern ? regex->pattern : "<Unknown Match>"));
#endif
/* This message will be logged. */
if (strlen(pattern) > 252) {
*error_msg = apr_psprintf(msr->mp, "Pattern match \"%.252s ...\" at %s.",

38
apache2/waf_logging/waf_format.proto
Просмотреть файл

@ -1,38 +0,0 @@
syntax = "proto2";
package waf_format;
message Details {
optional string message = 1;
optional string data = 2;
optional string file = 3;
optional string line = 4;
}
message Properties {
optional string instanceId = 1;
optional string clientIp = 2;
optional string clientPort = 3;
optional string requestUri = 4;
optional string ruleSetType = 5;
optional string ruleSetVersion = 6;
optional string ruleId = 7;
optional string message = 8;
optional string action = 9;
enum SiteType {
Global = 0;
}
optional SiteType site = 10;
optional Details details = 11;
optional string hostname = 12;
optional string transactionId = 13;
}
message Waf_Format {
optional string resourceId = 1;
optional string operationName = 2;
optional string category = 3;
optional Properties properties = 4;
}

104
apache2/waf_logging/waf_log_util.cc
Просмотреть файл

@ -5,52 +5,13 @@ using namespace std;
unordered_map<int, bool> appgw_ruleid_hash;
// This function fills in a waf format message based on modsec input.
void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char* operationName, char* category, char* instanceId, char* clientIP, char* clientPort, const char* requestUri, char* ruleSetType, char* ruleSetVersion, char* ruleId, char* messages, int action, int site, char* details_messages, char* details_data, char* details_file, char* details_line, const char* hostname, char* waf_unique_id) {
waf_format::Properties *properties;
waf_format::Details *details;
string get_json_log_message(char* resourceId, char* operationName, char* category, char* instanceId, char* clientIP, char* clientPort, const char* requestUri, char* ruleSetType, char* ruleSetVersion, char* ruleId, char* messages, int action, int site, char* details_messages, char* details_data, char* details_file, char* details_line, const char* hostname, char* waf_unique_id) {
string action_str;
string message_str;
string site_str;
properties = waf_format->mutable_properties();
details = properties->mutable_details();
if (resourceId != NULL) {
waf_format->set_resourceid(resourceId);
}
if (operationName != NULL) {
waf_format->set_operationname(operationName);
}
if (category != NULL) {
waf_format->set_category(category);
}
if (instanceId != NULL) {
properties->set_instanceid(instanceId);
}
if (clientIP != NULL) {
properties->set_clientip(clientIP);
}
if (clientPort != NULL) {
properties->set_clientport(clientPort);
}
if (requestUri != NULL) {
properties->set_requesturi(requestUri);
}
if (ruleSetType != NULL) {
properties->set_rulesettype(ruleSetType);
}
if (ruleSetVersion != NULL) {
properties->set_rulesetversion(ruleSetVersion);
}
if (ruleId != NULL) {
ruleId[strlen(ruleId) - 1] = '\0';
properties->set_ruleid(ruleId+1);
}
bool is_mandatory = false;
@ -59,14 +20,14 @@ void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char*
is_mandatory = rule_is_mandatory(tmpid);
if (ruleSetVersion[0] != '2' && !is_mandatory)
properties->set_action(WAF_ACTION_MATCHED);
action_str = WAF_ACTION_MATCHED;
else {
switch (action) {
case MODSEC_MODE_DETECT:
properties->set_action(WAF_ACTION_DETECTED);
action_str = WAF_ACTION_DETECTED;
break;
case MODSEC_MODE_PREVENT:
properties->set_action(WAF_ACTION_BLOCKED);
action_str = WAF_ACTION_BLOCKED;
break;
default:
break;
@ -74,7 +35,7 @@ void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char*
}
}
catch (...) {
properties->set_action("");
action_str = "";
}
if (messages != NULL) {
@ -91,77 +52,50 @@ void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char*
}
}
mandatory_message[ind + i] = '\0';
properties->set_message(mandatory_message);
message_str = mandatory_message;
}
else {
messages[strlen(messages) - 1] = '\0';
properties->set_message(messages+1);
message_str = messages+1;
}
}
if (site == 0) {
properties->set_site(waf_format::Properties::Global);
site_str = "Global";
}
if (details_messages != NULL) {
details->set_message(details_messages);
if (details_file != NULL) {
details_file[strlen(details_file) - 1] = '\0';
}
if (details_data != NULL) {
details_data[strlen(details_data) - 1] = '\0';
details->set_data(details_data+1);
}
if (details_file != NULL) {
details->set_file(details_file);
}
if (details_line != NULL) {
details_line[strlen(details_line) - 1] = '\0';
details->set_line(details_line+1);
}
if (hostname != NULL) {
properties->set_hostname(hostname);
}
if (waf_unique_id != NULL) {
waf_unique_id[strlen(waf_unique_id) - 1] = '\0';
properties->set_transactionid(waf_unique_id+1);
}
return to_json_string({resourceId, operationName, category, waf_logging::property{ instanceId, clientIP, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId+1, message_str, action_str, site_str, waf_logging::detail{ details_messages, details_data+1, details_file+1, details_line+1 }, hostname, waf_unique_id+1}});
}
// Main function: get fields from modsec, set the protobuf object and write to file in json.
int generate_json(char** result_json, char* resourceId, char* operationName, char* category, char* instanceId, char* clientIP, char* clientPort, const char* requestUri, char* ruleSetType, char* ruleSetVersion, char* ruleId, char* messages, int action, int site, char* details_messages, char* details_data, char* details_file, char* details_line, const char* hostname, char* waf_unique_id) {
waf_format::Waf_Format waf_format;
string json_string;
google::protobuf::util::JsonPrintOptions options;
google::protobuf::util::Status convert_result;
char* json_str;
try {
// Verify that the version of the library that we linked against is
// compatible with the version of the headers we compiled against.
GOOGLE_PROTOBUF_VERIFY_VERSION;
// Set Waf format.
set_waf_format(&waf_format, resourceId, operationName, category, instanceId, clientIP, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId, messages, action, site, details_messages, details_data, details_file, details_line, hostname, waf_unique_id);
options.add_whitespace = false;
options.always_print_primitive_fields = true;
options.preserve_proto_field_names = true;
convert_result = MessageToJsonString(waf_format, &json_string, options);
if (!convert_result.ok()) {
return WAF_LOG_UTIL_FAILED;
}
json_string = get_json_log_message(resourceId, operationName, category, instanceId, clientIP, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId, messages, action, site, details_messages, details_data, details_file, details_line, hostname, waf_unique_id);
}
catch (...) {
return WAF_LOG_UTIL_FAILED;
}
// Write the waf json string to disk.
*result_json = strdup((json_string + "\n").c_str());
return WAF_LOG_UTIL_SUCCESS;
}

57
apache2/waf_logging/waf_log_util_internal.h Normal file → Executable file
Просмотреть файл

@ -1,9 +1,62 @@
#ifndef _WAF_LOG_UTIL_INTERNAL_HEADER
#define _WAF_LOG_UTIL_INTERNAL_HEADER
#include <google/protobuf/util/json_util.h>
#include"waf_format.pb.h"
#include <fstream>
#include <unordered_map>
#include <exception>
#include <jsoncons/json.hpp>
#include <jsoncons/json_options.hpp>
// For convenience
using jsoncons::ojson;
using jsoncons::json_options;
using jsoncons::line_split_kind;
namespace waf_logging {
struct detail
{
std::string message;
std::string data;
std::string file;
std::string line;
};
struct property
{
std::string instanceId;
std::string clientIp;
std::string clientPort;
std::string requestUri;
std::string ruleSetType;
std::string ruleSetVersion;
std::string ruleId;
std::string message;
std::string action;
std::string site;
detail details;
std::string hostname;
std::string transactionId;
};
struct waf_log
{
std::string resourceId;
std::string operationName;
std::string category;
property properties;
};
}
JSONCONS_TYPE_TRAITS_DECL(waf_logging::detail, message, data, file, line);
JSONCONS_TYPE_TRAITS_DECL(waf_logging::property, instanceId, clientIp, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId, message, action, site, details, hostname, transactionId);
JSONCONS_TYPE_TRAITS_DECL(waf_logging::waf_log, resourceId, operationName, category, properties);
std::string to_json_string(const waf_logging::waf_log& log) {
json_options options;
std::string json_output;
encode_json(ojson{}, log, json_output, options, jsoncons::indenting::indent);
return json_output;
};
#endif

16
configure.ac
Просмотреть файл

@ -580,22 +580,6 @@ AC_ARG_ENABLE(collection-memory-database,
collection_memory_database="-I"`pwd`"/apache2/ag_mdb -DMEMORY_DATABASE_ENABLE"
])
# ProtoBuf compiler.
# First, specify with --with-protoc=/path/of/protoc.
# Or, specify with env variable PROTOC.
# If neither of the above, find it in the path.
#AC_MSG_CHECKING([for ProtoBuf compiler protoc])
AC_ARG_WITH([protoc],
[AS_HELP_STRING([--with-protoc=/path/of/protoc],
[Location of the protocol buffers compiler protoc. Defaults to looking on path.])],
[PROTOC="$withval"],
[ AS_IF([test "x${PROTOC}" == "x"],
[AC_PATH_PROG([PROTOC], [protoc], [no])])
]
)
#AC_MSG_RESULT([${PROTOC}])
AS_IF([test "${PROTOC}" == "no"], [AC_MSG_ERROR([ProtoBuf compiler "protoc" not found.])])
# Enable json waf logging support
AC_ARG_ENABLE(waf_json_logging,
AS_HELP_STRING([--enable-waf_json_logging],

13
docker/Dockerfile Normal file → Executable file
Просмотреть файл

@ -9,14 +9,11 @@ RUN apt-get install -y \
m4 \
libtool
# Install google protobuf 3.5.1
RUN mkdir -p /tmp/protobuf/
RUN curl -o /tmp/protobuf/protobuf-3.5.1.zip -OL https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-cpp-3.5.1.zip
RUN unzip /tmp/protobuf/protobuf-3.5.1.zip -d /tmp/protobuf/
WORKDIR /tmp/protobuf/protobuf-3.5.1
RUN ./autogen.sh
RUN ./configure --prefix=/usr
RUN make && make install
# Install jsoncons
RUN mkdir -p /tmp/jsoncons/
RUN curl -o /tmp/jsoncons/jsoncons-0.122.0.zip -OL https://github.com/danielaparker/jsoncons/archive/v0.122.0.zip
RUN unzip /tmp/jsoncons/jsoncons-0.122.0.zip -d /tmp/jsoncons
RUN cp -r /tmp/jsoncons/jsoncons-0.122.0/include/jsoncons /usr/include/
RUN apt-get install -y \
apache2-dev \

6
nginx/modsecurity/config.in
Просмотреть файл

@ -23,11 +23,7 @@ CORE_LIBS="$CORE_LIBS \
@APXS_LIBS@ \
@YAJL_LIBS@ \
@SSDEEP_LDFLAGS@ \
-lstdc++ \
-Wl,-Bstatic
-lprotobuf
-Wl,-Bdynamic"
-lstdc++"
ngx_addon_name=ngx_http_modsecurity

1
standalone/Makefile.am
Просмотреть файл

@ -5,7 +5,6 @@ pkglib_LTLIBRARIES = standalone.la
standalone_la_SOURCES = ../apache2/acmp.c \
../apache2/ag_mdb/murmur3.c \
../apache2/ag_mdb/ag_mdb.cpp \
../apache2/waf_logging/waf_format.pb.cc \
../apache2/waf_logging/waf_log_util.cc \
../apache2/waf_lock/waf_lock.cpp \
../apache2/apache2_config.c \