From 5f57ea44dcf3d99954d60dbb6f540dbcb8804c3c Mon Sep 17 00:00:00 2001 From: Yanshu Date: Thu, 18 Apr 2019 16:03:48 -0700 Subject: [PATCH] replace protobuf with jsoncons, and print pretty json (#110) --- apache2/Makefile.am | 13 --- apache2/re_operators.c | 5 +- apache2/waf_logging/waf_format.proto | 38 ------- apache2/waf_logging/waf_log_util.cc | 104 ++++---------------- apache2/waf_logging/waf_log_util_internal.h | 57 ++++++++++- configure.ac | 16 --- docker/Dockerfile | 13 +-- nginx/modsecurity/config.in | 6 +- standalone/Makefile.am | 1 - 9 files changed, 84 insertions(+), 169 deletions(-) mode change 100644 => 100755 apache2/re_operators.c delete mode 100755 apache2/waf_logging/waf_format.proto mode change 100644 => 100755 apache2/waf_logging/waf_log_util_internal.h mode change 100644 => 100755 docker/Dockerfile diff --git a/apache2/Makefile.am b/apache2/Makefile.am index 9c3de4c5..cdedd86e 100644 --- a/apache2/Makefile.am +++ b/apache2/Makefile.am @@ -1,17 +1,9 @@ pkglibdir = $(prefix)/lib pkglib_LTLIBRARIES = mod_security2.la -PROTOS = $(waf_logging/waf_format.proto) -PROTO_OBJS = $(PROTOS:.proto=.pb.o) - -PBSRCS = $(waf_logging/waf_format.proto) -PBOBJS = $(PROTOS:.proto=.pb.o) -PBGENS = $(PROTOS:.proto=.pb.cc) $(PROTOS:.proto=.pb.h) - mod_security2_la_SOURCES = acmp.c \ ag_mdb/murmur3.c \ ag_mdb/ag_mdb.cpp \ - waf_logging/waf_format.pb.cc \ waf_logging/waf_log_util.cc \ waf_lock/waf_lock.cpp \ apache2_config.c \ @@ -203,8 +195,3 @@ install-exec-hook: $(pkglib_LTLIBRARIES) done endif -%.pb.cc: %.proto - protoc --cpp_out=. $< - -%.pb.o : %.pb.cc - $(CXX) $(CXX_FLAGS) -c -o $@ $< \ No newline at end of file diff --git a/apache2/re_operators.c b/apache2/re_operators.c old mode 100644 new mode 100755 index b6d1c03c..521dcfc0 --- a/apache2/re_operators.c +++ b/apache2/re_operators.c @@ -1178,8 +1178,11 @@ static int msre_op_rx_execute(modsec_rec *msr, msre_rule *rule, msre_var *var, c if (rc != PCRE_ERROR_NOMATCH) { /* Match. */ /* We no longer escape the pattern here as it is done when logging */ +#ifdef WAF_JSON_LOGGING_ENABLE + char *pattern = apr_pstrdup(msr->mp, regex->pattern ? regex->pattern : ""); +#else char *pattern = apr_pstrdup(msr->mp, log_escape(msr->mp, regex->pattern ? regex->pattern : "")); - +#endif /* This message will be logged. */ if (strlen(pattern) > 252) { *error_msg = apr_psprintf(msr->mp, "Pattern match \"%.252s ...\" at %s.", diff --git a/apache2/waf_logging/waf_format.proto b/apache2/waf_logging/waf_format.proto deleted file mode 100755 index bb01e531..00000000 --- a/apache2/waf_logging/waf_format.proto +++ /dev/null @@ -1,38 +0,0 @@ -syntax = "proto2"; - -package waf_format; - -message Details { - optional string message = 1; - optional string data = 2; - optional string file = 3; - optional string line = 4; -} - - -message Properties { - optional string instanceId = 1; - optional string clientIp = 2; - optional string clientPort = 3; - optional string requestUri = 4; - optional string ruleSetType = 5; - optional string ruleSetVersion = 6; - optional string ruleId = 7; - optional string message = 8; - optional string action = 9; - - enum SiteType { - Global = 0; - } - optional SiteType site = 10; - optional Details details = 11; - optional string hostname = 12; - optional string transactionId = 13; -} - -message Waf_Format { - optional string resourceId = 1; - optional string operationName = 2; - optional string category = 3; - optional Properties properties = 4; -} diff --git a/apache2/waf_logging/waf_log_util.cc b/apache2/waf_logging/waf_log_util.cc index c0a78246..3423c4d2 100755 --- a/apache2/waf_logging/waf_log_util.cc +++ b/apache2/waf_logging/waf_log_util.cc @@ -5,52 +5,13 @@ using namespace std; unordered_map appgw_ruleid_hash; // This function fills in a waf format message based on modsec input. -void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char* operationName, char* category, char* instanceId, char* clientIP, char* clientPort, const char* requestUri, char* ruleSetType, char* ruleSetVersion, char* ruleId, char* messages, int action, int site, char* details_messages, char* details_data, char* details_file, char* details_line, const char* hostname, char* waf_unique_id) { - waf_format::Properties *properties; - waf_format::Details *details; +string get_json_log_message(char* resourceId, char* operationName, char* category, char* instanceId, char* clientIP, char* clientPort, const char* requestUri, char* ruleSetType, char* ruleSetVersion, char* ruleId, char* messages, int action, int site, char* details_messages, char* details_data, char* details_file, char* details_line, const char* hostname, char* waf_unique_id) { + string action_str; + string message_str; + string site_str; - properties = waf_format->mutable_properties(); - details = properties->mutable_details(); - - if (resourceId != NULL) { - waf_format->set_resourceid(resourceId); - } - - if (operationName != NULL) { - waf_format->set_operationname(operationName); - } - - if (category != NULL) { - waf_format->set_category(category); - } - - if (instanceId != NULL) { - properties->set_instanceid(instanceId); - } - - if (clientIP != NULL) { - properties->set_clientip(clientIP); - } - - if (clientPort != NULL) { - properties->set_clientport(clientPort); - } - - if (requestUri != NULL) { - properties->set_requesturi(requestUri); - } - - if (ruleSetType != NULL) { - properties->set_rulesettype(ruleSetType); - } - - if (ruleSetVersion != NULL) { - properties->set_rulesetversion(ruleSetVersion); - } - if (ruleId != NULL) { ruleId[strlen(ruleId) - 1] = '\0'; - properties->set_ruleid(ruleId+1); } bool is_mandatory = false; @@ -59,14 +20,14 @@ void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char* is_mandatory = rule_is_mandatory(tmpid); if (ruleSetVersion[0] != '2' && !is_mandatory) - properties->set_action(WAF_ACTION_MATCHED); + action_str = WAF_ACTION_MATCHED; else { switch (action) { case MODSEC_MODE_DETECT: - properties->set_action(WAF_ACTION_DETECTED); + action_str = WAF_ACTION_DETECTED; break; case MODSEC_MODE_PREVENT: - properties->set_action(WAF_ACTION_BLOCKED); + action_str = WAF_ACTION_BLOCKED; break; default: break; @@ -74,7 +35,7 @@ void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char* } } catch (...) { - properties->set_action(""); + action_str = ""; } if (messages != NULL) { @@ -91,77 +52,50 @@ void set_waf_format(waf_format::Waf_Format* waf_format, char* resourceId, char* } } mandatory_message[ind + i] = '\0'; - properties->set_message(mandatory_message); + message_str = mandatory_message; } else { messages[strlen(messages) - 1] = '\0'; - properties->set_message(messages+1); + message_str = messages+1; } } if (site == 0) { - properties->set_site(waf_format::Properties::Global); + site_str = "Global"; } - - if (details_messages != NULL) { - details->set_message(details_messages); + + if (details_file != NULL) { + details_file[strlen(details_file) - 1] = '\0'; } - + if (details_data != NULL) { details_data[strlen(details_data) - 1] = '\0'; - details->set_data(details_data+1); - } - - if (details_file != NULL) { - details->set_file(details_file); } if (details_line != NULL) { details_line[strlen(details_line) - 1] = '\0'; - details->set_line(details_line+1); } - - if (hostname != NULL) { - properties->set_hostname(hostname); - } if (waf_unique_id != NULL) { waf_unique_id[strlen(waf_unique_id) - 1] = '\0'; - properties->set_transactionid(waf_unique_id+1); } + + return to_json_string({resourceId, operationName, category, waf_logging::property{ instanceId, clientIP, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId+1, message_str, action_str, site_str, waf_logging::detail{ details_messages, details_data+1, details_file+1, details_line+1 }, hostname, waf_unique_id+1}}); } // Main function: get fields from modsec, set the protobuf object and write to file in json. int generate_json(char** result_json, char* resourceId, char* operationName, char* category, char* instanceId, char* clientIP, char* clientPort, const char* requestUri, char* ruleSetType, char* ruleSetVersion, char* ruleId, char* messages, int action, int site, char* details_messages, char* details_data, char* details_file, char* details_line, const char* hostname, char* waf_unique_id) { - waf_format::Waf_Format waf_format; string json_string; - google::protobuf::util::JsonPrintOptions options; - google::protobuf::util::Status convert_result; char* json_str; try { - // Verify that the version of the library that we linked against is - // compatible with the version of the headers we compiled against. - GOOGLE_PROTOBUF_VERIFY_VERSION; - // Set Waf format. - set_waf_format(&waf_format, resourceId, operationName, category, instanceId, clientIP, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId, messages, action, site, details_messages, details_data, details_file, details_line, hostname, waf_unique_id); - - options.add_whitespace = false; - options.always_print_primitive_fields = true; - options.preserve_proto_field_names = true; - - convert_result = MessageToJsonString(waf_format, &json_string, options); - - if (!convert_result.ok()) { - return WAF_LOG_UTIL_FAILED; - } + json_string = get_json_log_message(resourceId, operationName, category, instanceId, clientIP, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId, messages, action, site, details_messages, details_data, details_file, details_line, hostname, waf_unique_id); } catch (...) { return WAF_LOG_UTIL_FAILED; } - - // Write the waf json string to disk. + *result_json = strdup((json_string + "\n").c_str()); return WAF_LOG_UTIL_SUCCESS; } diff --git a/apache2/waf_logging/waf_log_util_internal.h b/apache2/waf_logging/waf_log_util_internal.h old mode 100644 new mode 100755 index e265d2c7..70e475f7 --- a/apache2/waf_logging/waf_log_util_internal.h +++ b/apache2/waf_logging/waf_log_util_internal.h @@ -1,9 +1,62 @@ #ifndef _WAF_LOG_UTIL_INTERNAL_HEADER #define _WAF_LOG_UTIL_INTERNAL_HEADER -#include -#include"waf_format.pb.h" #include +#include #include +#include +#include + +// For convenience +using jsoncons::ojson; +using jsoncons::json_options; +using jsoncons::line_split_kind; + +namespace waf_logging { + + struct detail + { + std::string message; + std::string data; + std::string file; + std::string line; + }; + + struct property + { + std::string instanceId; + std::string clientIp; + std::string clientPort; + std::string requestUri; + std::string ruleSetType; + std::string ruleSetVersion; + std::string ruleId; + std::string message; + std::string action; + std::string site; + detail details; + std::string hostname; + std::string transactionId; + }; + + struct waf_log + { + std::string resourceId; + std::string operationName; + std::string category; + property properties; + }; +} + +JSONCONS_TYPE_TRAITS_DECL(waf_logging::detail, message, data, file, line); +JSONCONS_TYPE_TRAITS_DECL(waf_logging::property, instanceId, clientIp, clientPort, requestUri, ruleSetType, ruleSetVersion, ruleId, message, action, site, details, hostname, transactionId); +JSONCONS_TYPE_TRAITS_DECL(waf_logging::waf_log, resourceId, operationName, category, properties); + +std::string to_json_string(const waf_logging::waf_log& log) { + json_options options; + std::string json_output; + encode_json(ojson{}, log, json_output, options, jsoncons::indenting::indent); + return json_output; +}; #endif diff --git a/configure.ac b/configure.ac index eb3a09e1..e80472e4 100644 --- a/configure.ac +++ b/configure.ac @@ -580,22 +580,6 @@ AC_ARG_ENABLE(collection-memory-database, collection_memory_database="-I"`pwd`"/apache2/ag_mdb -DMEMORY_DATABASE_ENABLE" ]) -# ProtoBuf compiler. -# First, specify with --with-protoc=/path/of/protoc. -# Or, specify with env variable PROTOC. -# If neither of the above, find it in the path. -#AC_MSG_CHECKING([for ProtoBuf compiler protoc]) -AC_ARG_WITH([protoc], - [AS_HELP_STRING([--with-protoc=/path/of/protoc], - [Location of the protocol buffers compiler protoc. Defaults to looking on path.])], - [PROTOC="$withval"], - [ AS_IF([test "x${PROTOC}" == "x"], - [AC_PATH_PROG([PROTOC], [protoc], [no])]) - ] -) -#AC_MSG_RESULT([${PROTOC}]) -AS_IF([test "${PROTOC}" == "no"], [AC_MSG_ERROR([ProtoBuf compiler "protoc" not found.])]) - # Enable json waf logging support AC_ARG_ENABLE(waf_json_logging, AS_HELP_STRING([--enable-waf_json_logging], diff --git a/docker/Dockerfile b/docker/Dockerfile old mode 100644 new mode 100755 index 357e0fd2..dc3350d7 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,14 +9,11 @@ RUN apt-get install -y \ m4 \ libtool -# Install google protobuf 3.5.1 -RUN mkdir -p /tmp/protobuf/ -RUN curl -o /tmp/protobuf/protobuf-3.5.1.zip -OL https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-cpp-3.5.1.zip -RUN unzip /tmp/protobuf/protobuf-3.5.1.zip -d /tmp/protobuf/ -WORKDIR /tmp/protobuf/protobuf-3.5.1 -RUN ./autogen.sh -RUN ./configure --prefix=/usr -RUN make && make install +# Install jsoncons +RUN mkdir -p /tmp/jsoncons/ +RUN curl -o /tmp/jsoncons/jsoncons-0.122.0.zip -OL https://github.com/danielaparker/jsoncons/archive/v0.122.0.zip +RUN unzip /tmp/jsoncons/jsoncons-0.122.0.zip -d /tmp/jsoncons +RUN cp -r /tmp/jsoncons/jsoncons-0.122.0/include/jsoncons /usr/include/ RUN apt-get install -y \ apache2-dev \ diff --git a/nginx/modsecurity/config.in b/nginx/modsecurity/config.in index 1498f95a..6a3926f0 100644 --- a/nginx/modsecurity/config.in +++ b/nginx/modsecurity/config.in @@ -23,11 +23,7 @@ CORE_LIBS="$CORE_LIBS \ @APXS_LIBS@ \ @YAJL_LIBS@ \ @SSDEEP_LDFLAGS@ \ - -lstdc++ \ - -Wl,-Bstatic - -lprotobuf - -Wl,-Bdynamic" - + -lstdc++" ngx_addon_name=ngx_http_modsecurity diff --git a/standalone/Makefile.am b/standalone/Makefile.am index 33afd377..d3623db6 100644 --- a/standalone/Makefile.am +++ b/standalone/Makefile.am @@ -5,7 +5,6 @@ pkglib_LTLIBRARIES = standalone.la standalone_la_SOURCES = ../apache2/acmp.c \ ../apache2/ag_mdb/murmur3.c \ ../apache2/ag_mdb/ag_mdb.cpp \ - ../apache2/waf_logging/waf_format.pb.cc \ ../apache2/waf_logging/waf_log_util.cc \ ../apache2/waf_lock/waf_lock.cpp \ ../apache2/apache2_config.c \