Add event aggregation, fix ftrace usage (#103)
This commit is contained in:
Tad Glines
GitHub
Родитель
99215907e0
Коммит
da2b3d5872
10
CGroups.cpp
10
CGroups.cpp
|
|
@ -38,6 +38,10 @@ void AppendUint64(const std::string& path, uint64_t val) {
|
|||
AppendFile(path, {{std::to_string(val)}});
|
||||
}
|
||||
|
||||
void WriteUint64(const std::string& path, uint64_t val) {
|
||||
WriteFile(path, {{std::to_string(val)}});
|
||||
}
|
||||
|
||||
uint64_t ReadUint64(const std::string& path) {
|
||||
auto lines = ReadFile(path);
|
||||
if (lines.empty()) {
|
||||
|
|
@ -92,7 +96,7 @@ uint64_t CGroupCPU::GetShares() {
|
|||
}
|
||||
|
||||
void CGroupCPU::SetShares(uint64_t val) {
|
||||
AppendUint64(_dir + CGROUP_CPU_SHARES_FILE, val);
|
||||
WriteUint64(_dir + CGROUP_CPU_SHARES_FILE, val);
|
||||
}
|
||||
|
||||
bool CGroupCPU::HasCFSQuotaUS() {
|
||||
|
|
@ -104,7 +108,7 @@ uint64_t CGroupCPU::GetCFSPeriodUS() {
|
|||
}
|
||||
|
||||
void CGroupCPU::SetCFSPeriodUS(uint64_t val) {
|
||||
AppendUint64(_dir + CGROUP_CPU_PERIOD_US_FILE, val);
|
||||
WriteUint64(_dir + CGROUP_CPU_PERIOD_US_FILE, val);
|
||||
}
|
||||
|
||||
uint64_t CGroupCPU::GetCFSQuotaUS() {
|
||||
|
|
@ -112,7 +116,7 @@ uint64_t CGroupCPU::GetCFSQuotaUS() {
|
|||
}
|
||||
|
||||
void CGroupCPU::SetCFSQuotaUS(uint64_t val) {
|
||||
AppendUint64(_dir + CGROUP_CPU_QUOTA_US_FILE, val);
|
||||
WriteUint64(_dir + CGROUP_CPU_QUOTA_US_FILE, val);
|
||||
}
|
||||
|
||||
std::shared_ptr<CGroupCPU> CGroups::OpenCPU(const std::string& name) {
|
||||
|
|
|
|||
|
|
@ -98,11 +98,14 @@ add_executable(auomscollect
|
|||
CGroups.cpp
|
||||
CPULimits.cpp
|
||||
SchedPriority.cpp
|
||||
EventMatcher.cpp
|
||||
EventAggregator.cpp
|
||||
)
|
||||
|
||||
target_compile_options(auomscollect PRIVATE "-Werror")
|
||||
|
||||
target_link_libraries(auomscollect
|
||||
libre2.a
|
||||
dl
|
||||
pthread
|
||||
rt
|
||||
|
|
@ -183,6 +186,8 @@ add_executable(auoms
|
|||
CGroups.cpp
|
||||
CPULimits.cpp
|
||||
CmdlineRedactor.cpp
|
||||
EventMatcher.cpp
|
||||
EventAggregator.cpp
|
||||
)
|
||||
|
||||
target_compile_options(auoms PRIVATE "-Werror")
|
||||
|
|
@ -596,13 +601,17 @@ add_executable(OutputInputTests
|
|||
TranslateSyscall.cpp
|
||||
UserDB.cpp
|
||||
Version.cpp
|
||||
EventMatcher.cpp
|
||||
EventAggregator.cpp
|
||||
)
|
||||
|
||||
if(NOT DO_STATIC_LINK)
|
||||
target_compile_definitions(OutputInputTests PUBLIC BOOST_TEST_DYN_LINK=1)
|
||||
endif()
|
||||
|
||||
target_link_libraries(OutputInputTests ${Boost_LIBRARIES}
|
||||
target_link_libraries(OutputInputTests
|
||||
libre2.a
|
||||
${Boost_LIBRARIES}
|
||||
pthread
|
||||
)
|
||||
|
||||
|
|
@ -630,3 +639,50 @@ target_link_libraries(CmdlineRedactorTests
|
|||
)
|
||||
|
||||
add_test(CmdlineRedactorTests ${CMAKE_BINARY_DIR}/CmdlineRedactorTests --log_sink=CmdlineRedactorTests.log --report_sink=CmdlineRedactorTests.report)
|
||||
|
||||
|
||||
add_executable(EventMatcherTests
|
||||
auoms_version.h
|
||||
EventMatcherTests.cpp
|
||||
EventMatcher.cpp
|
||||
Event.cpp
|
||||
TranslateRecordType.cpp
|
||||
StringUtils.cpp
|
||||
TestEventData.cpp
|
||||
)
|
||||
|
||||
if(NOT DO_STATIC_LINK)
|
||||
target_compile_definitions(EventMatcherTests PUBLIC BOOST_TEST_DYN_LINK=1)
|
||||
endif()
|
||||
|
||||
target_link_libraries(EventMatcherTests
|
||||
libre2.a
|
||||
${Boost_LIBRARIES}
|
||||
pthread
|
||||
)
|
||||
|
||||
add_test(EventMatcherTests ${CMAKE_BINARY_DIR}/EventMatcherTests --log_sink=EventMatcherTests.log --report_sink=EventMatcherTests.report)
|
||||
|
||||
add_executable(EventAggregatorTests
|
||||
auoms_version.h
|
||||
EventAggregatorTests.cpp
|
||||
EventAggregator.cpp
|
||||
EventMatcher.cpp
|
||||
Event.cpp
|
||||
TranslateRecordType.cpp
|
||||
StringUtils.cpp
|
||||
TestEventData.cpp
|
||||
TempFile.cpp
|
||||
)
|
||||
|
||||
if(NOT DO_STATIC_LINK)
|
||||
target_compile_definitions(EventAggregatorTests PUBLIC BOOST_TEST_DYN_LINK=1)
|
||||
endif()
|
||||
|
||||
target_link_libraries(EventAggregatorTests
|
||||
libre2.a
|
||||
${Boost_LIBRARIES}
|
||||
pthread
|
||||
)
|
||||
|
||||
add_test(EventAggregatorTests ${CMAKE_BINARY_DIR}/EventAggregatorTests --log_sink=EventAggregatorTests.log --report_sink=EventAggregatorTests.report)
|
||||
|
|
|
|||
242
Event.cpp
242
Event.cpp
|
|
@ -59,6 +59,14 @@
|
|||
* char[] field_name (null terminated)
|
||||
* char[] raw_value (null terminated)
|
||||
* char[] interp_value (null terminated, only present if interp_value_size > 0)
|
||||
* Extensions:
|
||||
* uint32_t num_extensions
|
||||
* uint32_t[] index
|
||||
* Extension:
|
||||
* uint32_t type
|
||||
* uint32_t size
|
||||
* *data
|
||||
* uint32_t extensions_offset
|
||||
*/
|
||||
|
||||
inline uint32_t& INDEX_VALUE(uint8_t* data, uint32_t offset, uint32_t index) {
|
||||
|
|
@ -245,6 +253,61 @@ constexpr uint32_t FIELD_NAME_OFFSET = FIELD_HEADER_SIZE;
|
|||
constexpr uint32_t FIELD_RAW_VALUE_OFFSET(uint16_t name_size) { return FIELD_NAME_OFFSET + name_size; }
|
||||
constexpr uint32_t FIELD_INTERP_VALUE_OFFSET(uint16_t name_size, uint32_t raw_size) { return FIELD_NAME_OFFSET + name_size + raw_size; }
|
||||
|
||||
constexpr uint32_t EXTENSIONS_HEADER_SIZE = sizeof(uint32_t);
|
||||
constexpr uint32_t EXTENSION_HEADER_SIZE = sizeof(uint32_t)*2;
|
||||
|
||||
inline uint32_t EXTENSIONS_OFFSET(const uint8_t* data) {
|
||||
return *reinterpret_cast<const uint32_t*>(data + (EVENT_SIZE(data) - sizeof(uint32_t)));
|
||||
}
|
||||
|
||||
inline uint32_t& EXTENSIONS_OFFSET(uint8_t* data) {
|
||||
return *reinterpret_cast<uint32_t*>(data + (EVENT_SIZE(data) - sizeof(uint32_t)));
|
||||
}
|
||||
|
||||
constexpr uint32_t EVENT_NUM_EXTENSIONS(const uint8_t* data, uint32_t offset) {
|
||||
return *reinterpret_cast<const uint32_t*>(data + offset + sizeof(uint32_t));
|
||||
}
|
||||
|
||||
inline uint32_t& EVENT_NUM_EXTENSIONS(uint8_t* data, uint32_t offset) {
|
||||
return *reinterpret_cast<uint32_t*>(data + offset + sizeof(uint32_t));
|
||||
}
|
||||
|
||||
constexpr uint32_t EXTENSIONS_INDEX_OFFSET(const uint8_t* data, uint32_t offset) {
|
||||
return *reinterpret_cast<const uint32_t*>(data + offset + (sizeof(uint32_t)*2));
|
||||
}
|
||||
|
||||
inline uint32_t EXTENSION_OFFSET(const uint8_t* data, uint32_t offset, uint32_t index) {
|
||||
return reinterpret_cast<const uint32_t*>(data + offset + (sizeof(uint32_t)*2))[index];
|
||||
}
|
||||
|
||||
inline uint32_t& EXTENSION_OFFSET(uint8_t* data, uint32_t offset, uint32_t index) {
|
||||
return reinterpret_cast<uint32_t*>(data + offset + (sizeof(uint32_t)*2))[index];
|
||||
}
|
||||
|
||||
constexpr uint32_t EXTENSION_TYPE(const uint8_t* data, uint32_t offset) {
|
||||
return *reinterpret_cast<const uint32_t*>(data + offset);
|
||||
}
|
||||
|
||||
inline uint32_t& EXTENSION_TYPE(uint8_t* data, uint32_t offset) {
|
||||
return *reinterpret_cast<uint32_t*>(data + offset);
|
||||
}
|
||||
|
||||
constexpr uint32_t EXTENSION_SIZE(const uint8_t* data, uint32_t offset) {
|
||||
return *reinterpret_cast<const uint32_t*>(data + offset + sizeof(uint32_t));
|
||||
}
|
||||
|
||||
inline uint32_t& EXTENSION_SIZE(uint8_t* data, uint32_t offset) {
|
||||
return *reinterpret_cast<uint32_t*>(data + offset + sizeof(uint32_t));
|
||||
}
|
||||
|
||||
constexpr const void* EXTENSION_DATA(const uint8_t* data, uint32_t offset) {
|
||||
return data + offset + (sizeof(uint32_t)*2);
|
||||
}
|
||||
|
||||
constexpr void* EXTENSION_DATA(uint8_t* data, uint32_t offset) {
|
||||
return data + offset + (sizeof(uint32_t)*2);
|
||||
}
|
||||
|
||||
/*****************************************************************************
|
||||
** EventBuilder
|
||||
*****************************************************************************/
|
||||
|
|
@ -258,6 +321,8 @@ bool EventBuilder::BeginEvent(uint64_t sec, uint32_t msec, uint64_t serial, uint
|
|||
throw std::runtime_error("num_records == 0!");
|
||||
}
|
||||
|
||||
_extensions_offset = 0;
|
||||
_extension_idx = 0;
|
||||
_roffset = EVENT_HEADER_SIZE(num_records);
|
||||
_record_idx = 0;
|
||||
|
||||
|
|
@ -296,12 +361,12 @@ uint16_t EventBuilder::GetEventPriority() {
|
|||
return EVENT_PRIORITY(_data);
|
||||
}
|
||||
|
||||
void EventBuilder::SetEventFlags(uint16_t flags) {
|
||||
void EventBuilder::AddEventFlags(uint16_t flags) {
|
||||
if (_data == nullptr) {
|
||||
throw std::runtime_error("Event not started!");
|
||||
}
|
||||
|
||||
EVENT_FLAGS(_data) = flags;
|
||||
EVENT_FLAGS(_data) |= flags;
|
||||
}
|
||||
|
||||
uint16_t EventBuilder::GetEventFlags() {
|
||||
|
|
@ -337,6 +402,10 @@ int EventBuilder::EndEvent() {
|
|||
throw std::runtime_error("EventRecord ended prematurely: Expected " + std::to_string(EVENT_NUM_RECORDS(_data)) + " records, only " + std::to_string(_record_idx) + " were added");
|
||||
}
|
||||
|
||||
if (_extensions_offset != 0 && _extension_idx != EVENT_NUM_EXTENSIONS(_data, _extensions_offset)) {
|
||||
throw std::runtime_error("Event ended prematurely: Expected " + std::to_string(EVENT_NUM_EXTENSIONS(_data, _extensions_offset)) + " extensions, only " + std::to_string(_extension_idx) + " were added");
|
||||
}
|
||||
|
||||
SET_EVENT_SIZE(_data, static_cast<uint32_t>(_size));
|
||||
|
||||
if (_prioritizer) {
|
||||
|
|
@ -519,6 +588,74 @@ int EventBuilder::GetFieldCount() {
|
|||
return _field_idx;
|
||||
}
|
||||
|
||||
bool EventBuilder::BeginExtensions(uint32_t num_extensions) {
|
||||
if (_data == nullptr) {
|
||||
throw std::runtime_error("Event not started!");
|
||||
}
|
||||
|
||||
if (_record_idx != EVENT_NUM_RECORDS(_data)) {
|
||||
throw std::runtime_error("EventRecord ended prematurely: Expected " + std::to_string(EVENT_NUM_RECORDS(_data)) + " records, only " + std::to_string(_record_idx) + " were added");
|
||||
}
|
||||
|
||||
size_t size = _size + EXTENSIONS_HEADER_SIZE + (sizeof(uint32_t) * num_extensions);
|
||||
if (!_allocator->Allocate(reinterpret_cast<void**>(&_data), size)) {
|
||||
return false;
|
||||
}
|
||||
_extensions_offset = _size;
|
||||
_size = size;
|
||||
_extension_idx = 0;
|
||||
_eoffset = _size;
|
||||
|
||||
EVENT_NUM_EXTENSIONS(_data, _extensions_offset) = num_extensions;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool EventBuilder::AddExtension(uint32_t type, uint32_t data_size, void* data) {
|
||||
if (_data == nullptr) {
|
||||
throw std::runtime_error("Event not started!");
|
||||
}
|
||||
|
||||
if (_extensions_offset == 0) {
|
||||
throw std::runtime_error("Event Extensions not started");
|
||||
}
|
||||
|
||||
size_t size = _size + EXTENSION_HEADER_SIZE + data_size;
|
||||
if (!_allocator->Allocate(reinterpret_cast<void**>(&_data), size)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
EXTENSION_OFFSET(_data, _extensions_offset, _extension_idx) = _eoffset;
|
||||
EXTENSION_TYPE(_data, _eoffset) = type;
|
||||
EXTENSION_SIZE(_data, _eoffset) = data_size;
|
||||
memcpy(_data, data, size);
|
||||
|
||||
_extension_idx += 1;
|
||||
_eoffset = _size;
|
||||
_size = size;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool EventBuilder::EndExtensions() {
|
||||
if (_extension_idx != EVENT_NUM_EXTENSIONS(_data, _extensions_offset)) {
|
||||
throw std::runtime_error("Event ended prematurely: Expected " + std::to_string(EVENT_NUM_EXTENSIONS(_data, _extensions_offset)) + " extensions, only " + std::to_string(_extension_idx) + " were added");
|
||||
}
|
||||
|
||||
size_t size = _size+sizeof(uint32_t);
|
||||
if (!_allocator->Allocate(reinterpret_cast<void**>(&_data), size)) {
|
||||
return false;
|
||||
}
|
||||
_size = size;
|
||||
|
||||
SET_EVENT_SIZE(_data, static_cast<uint32_t>(_size));
|
||||
EXTENSIONS_OFFSET(_data) - _extensions_offset;
|
||||
EVENT_FLAGS(_data) |= EVENT_FLAG_HAS_EXTENSIONS;
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/*****************************************************************************
|
||||
** EventRecordField
|
||||
*****************************************************************************/
|
||||
|
|
@ -765,6 +902,77 @@ void EventRecord::move(int32_t n) {
|
|||
}
|
||||
}
|
||||
|
||||
/*****************************************************************************
|
||||
** EventExtension
|
||||
*****************************************************************************/
|
||||
|
||||
uint32_t EventExtension::Type() const {
|
||||
return EXTENSION_TYPE(_data, _eoffset);
|
||||
}
|
||||
|
||||
uint32_t EventExtension::Size() const {
|
||||
return EXTENSION_SIZE(_data, _eoffset);
|
||||
}
|
||||
|
||||
const void* EventExtension::Data() const {
|
||||
return EXTENSION_DATA(_data, _eoffset);
|
||||
}
|
||||
|
||||
EventExtension::EventExtension(const uint8_t* data, uint32_t offset, uint32_t index) {
|
||||
_data = data;
|
||||
_offset = offset;
|
||||
_index = index;
|
||||
if (_index < EVENT_NUM_EXTENSIONS(_data, _offset)) {
|
||||
_eoffset = EXTENSION_OFFSET(_data, _offset, _index);
|
||||
} else {
|
||||
_eoffset = EVENT_SIZE(_data);
|
||||
}
|
||||
}
|
||||
|
||||
void EventExtension::move(int32_t n) {
|
||||
_index += n;
|
||||
if (_index < EVENT_NUM_EXTENSIONS(_data, _offset)) {
|
||||
_eoffset = EXTENSION_OFFSET(_data, _offset, _index);
|
||||
} else {
|
||||
_eoffset = EVENT_SIZE(_data);
|
||||
}
|
||||
}
|
||||
|
||||
/*****************************************************************************
|
||||
** EventExtensions
|
||||
*****************************************************************************/
|
||||
|
||||
uint32_t EventExtensions::NumExtensions() const {
|
||||
if (_data != nullptr) {
|
||||
return EVENT_NUM_EXTENSIONS(_data, _offset);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
EventExtension EventExtensions::ExtensionAt(uint32_t index) const {
|
||||
if (_data == nullptr || index >= EVENT_NUM_EXTENSIONS(_data, _offset)) {
|
||||
throw std::out_of_range("Extension index out of range for event: " + std::to_string(index));
|
||||
}
|
||||
return EventExtension(_data, _offset, index);
|
||||
}
|
||||
|
||||
EventExtension EventExtensions::begin() const {
|
||||
if (_data != nullptr && EVENT_NUM_EXTENSIONS(_data, _offset) > 0) {
|
||||
return EventExtension(_data, _offset, 0);
|
||||
} else {
|
||||
throw std::out_of_range("Event has no extensions");
|
||||
}
|
||||
}
|
||||
|
||||
EventExtension EventExtensions::end() const {
|
||||
if (_data != nullptr && EVENT_NUM_EXTENSIONS(_data, _offset) > 0) {
|
||||
return EventExtension(_data, _offset, EVENT_NUM_EXTENSIONS(_data, _offset));
|
||||
} else {
|
||||
throw std::out_of_range("Event has no extensions");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*****************************************************************************
|
||||
** Event
|
||||
*****************************************************************************/
|
||||
|
|
@ -806,7 +1014,7 @@ int32_t Event::Pid() const {
|
|||
}
|
||||
|
||||
EventRecord Event::RecordAt(uint32_t index) const {
|
||||
if (index > EVENT_NUM_RECORDS(_data)+1) {
|
||||
if (index >= EVENT_NUM_RECORDS(_data)) {
|
||||
throw std::out_of_range("Record index out of range for event: " + std::to_string(index));
|
||||
}
|
||||
return EventRecord(_data, index);
|
||||
|
|
@ -828,6 +1036,34 @@ EventRecord Event::end() const {
|
|||
}
|
||||
}
|
||||
|
||||
uint32_t Event::NumExtensions() const {
|
||||
if ((EVENT_FLAGS(_data) & EVENT_FLAG_HAS_EXTENSIONS) == 0) {
|
||||
return 0;
|
||||
}
|
||||
return EVENT_NUM_EXTENSIONS(_data, EXTENSIONS_OFFSET(_data));
|
||||
}
|
||||
|
||||
uint32_t Event::ExtensionTypeAt(uint32_t index) const {
|
||||
if ((EVENT_FLAGS(_data) & EVENT_FLAG_HAS_EXTENSIONS) == 0 || index >= EVENT_NUM_EXTENSIONS(_data, EXTENSIONS_OFFSET(_data))) {
|
||||
throw std::out_of_range("Extension index out of range for event: " + std::to_string(index));
|
||||
}
|
||||
return EXTENSION_TYPE(_data, EXTENSION_OFFSET(_data, EXTENSIONS_OFFSET(_data), index));
|
||||
}
|
||||
|
||||
EventExtension Event::ExtensionAt(uint32_t index) const {
|
||||
if ((EVENT_FLAGS(_data) & EVENT_FLAG_HAS_EXTENSIONS) == 0 || index >= EVENT_NUM_EXTENSIONS(_data, EXTENSIONS_OFFSET(_data))) {
|
||||
throw std::out_of_range("Extension index out of range for event: " + std::to_string(index));
|
||||
}
|
||||
return EventExtension(_data, EXTENSIONS_OFFSET(_data), index);
|
||||
}
|
||||
|
||||
EventExtensions Event::Extensions() const {
|
||||
if ((EVENT_FLAGS(_data) & EVENT_FLAG_HAS_EXTENSIONS) == 0) {
|
||||
return EventExtensions(nullptr, 0);
|
||||
}
|
||||
return EventExtensions(_data, EXTENSIONS_OFFSET(_data));
|
||||
}
|
||||
|
||||
int Event::Validate() const {
|
||||
if (_size <= EVENT_RECORD_INDEX_OFFSET) {
|
||||
return 1;
|
||||
|
|
|
|||
155
Event.h
155
Event.h
|
|
@ -22,9 +22,11 @@
|
|||
#include <iterator>
|
||||
#include <memory>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
|
||||
constexpr uint16_t EVENT_FLAG_IS_AUOMS_EVENT = 1;
|
||||
constexpr uint16_t EVENT_FLAG_HAS_EXTENSIONS = 2;
|
||||
|
||||
class EventRecord;
|
||||
|
||||
|
|
@ -196,6 +198,100 @@ private:
|
|||
uint32_t _index;
|
||||
};
|
||||
|
||||
|
||||
class EventExtension {
|
||||
public:
|
||||
typedef std::random_access_iterator_tag iterator_category;
|
||||
typedef EventExtension value_type;
|
||||
typedef int32_t difference_type;
|
||||
typedef EventExtension* pointer;
|
||||
typedef EventExtension& reference;
|
||||
|
||||
|
||||
EventExtension() {
|
||||
_data = nullptr;
|
||||
_offset = 0;
|
||||
_eoffset = 0;
|
||||
_index = 0;
|
||||
}
|
||||
|
||||
EventExtension(const EventExtension& other) = default;
|
||||
EventExtension(EventExtension&& other) = default;
|
||||
EventExtension& operator=(const EventExtension& other) = default;
|
||||
EventExtension& operator=(EventExtension&& other) = default;
|
||||
|
||||
|
||||
uint32_t Type() const;
|
||||
uint32_t Size() const;
|
||||
const void* Data() const;
|
||||
|
||||
operator bool() const {
|
||||
return _data != nullptr;
|
||||
}
|
||||
|
||||
EventExtension& operator+=(int32_t movement) { move(movement); return (*this); }
|
||||
EventExtension& operator-=(int32_t movement) { move(-movement); return (*this); }
|
||||
EventExtension& operator++() { move(1); return (*this); }
|
||||
EventExtension& operator--() { move(-1); return (*this); }
|
||||
EventExtension operator++(int32_t) { auto temp(*this); move(1); return temp;}
|
||||
EventExtension operator--(int32_t) { auto temp(*this); move(-1); return temp;}
|
||||
EventExtension operator+(int32_t movement) const { auto temp(*this); temp.move(movement); return temp; }
|
||||
EventExtension operator-(int32_t movement) const { auto temp(*this); temp.move(-movement); return temp; }
|
||||
|
||||
int32_t operator+(const EventExtension& other) { return other._index + _index; }
|
||||
int32_t operator-(const EventExtension& other) { return other._index - _index; }
|
||||
|
||||
bool operator==(const EventExtension& other) const {
|
||||
return _data == other._data && _eoffset == other._eoffset;
|
||||
}
|
||||
|
||||
bool operator!=(const EventExtension& other) const { return !(*this == other); }
|
||||
|
||||
const EventExtension* operator->() const { return this; }
|
||||
EventExtension* operator->() { return this; }
|
||||
const EventExtension& operator*() const { return *this; }
|
||||
EventExtension& operator*() { return *this; }
|
||||
|
||||
private:
|
||||
friend class Event;
|
||||
friend class EventExtensions;
|
||||
|
||||
EventExtension(const uint8_t* data, uint32_t offset, uint32_t index);
|
||||
void move(int32_t n);
|
||||
|
||||
const uint8_t* _data;
|
||||
uint32_t _offset;
|
||||
uint32_t _eoffset;
|
||||
uint32_t _index;
|
||||
};
|
||||
|
||||
|
||||
class EventExtensions {
|
||||
public:
|
||||
EventExtensions(const EventExtensions& other) = default;
|
||||
EventExtensions(EventExtensions&& other) = default;
|
||||
EventExtensions& operator=(const EventExtensions& other) = default;
|
||||
EventExtensions& operator=(EventExtensions&& other) = default;
|
||||
|
||||
operator bool() const {
|
||||
return _data != nullptr;
|
||||
}
|
||||
|
||||
uint32_t NumExtensions() const;
|
||||
EventExtension ExtensionAt(uint32_t index) const;
|
||||
|
||||
EventExtension begin() const;
|
||||
EventExtension end() const;
|
||||
|
||||
private:
|
||||
friend class Event;
|
||||
|
||||
EventExtensions(const uint8_t* data, uint32_t offset): _data(data), _offset(offset) {};
|
||||
|
||||
const uint8_t* _data;
|
||||
uint32_t _offset;
|
||||
};
|
||||
|
||||
class Event {
|
||||
public:
|
||||
static inline std::pair<uint32_t, uint32_t> GetVersionAndSize(const void* data) {
|
||||
|
|
@ -229,6 +325,12 @@ public:
|
|||
EventRecord begin() const;
|
||||
EventRecord end() const;
|
||||
|
||||
uint32_t NumExtensions() const;
|
||||
uint32_t ExtensionTypeAt(uint32_t index) const;
|
||||
EventExtension ExtensionAt(uint32_t index) const;
|
||||
|
||||
EventExtensions Extensions() const;
|
||||
|
||||
int Validate() const;
|
||||
private:
|
||||
friend class EventRecord;
|
||||
|
|
@ -247,6 +349,49 @@ public:
|
|||
virtual bool Rollback() = 0;
|
||||
};
|
||||
|
||||
class BasicEventBuilderAllocator: public IEventBuilderAllocator {
|
||||
public:
|
||||
explicit BasicEventBuilderAllocator(size_t capacity): _buffer() {
|
||||
_buffer.reserve(capacity);
|
||||
}
|
||||
BasicEventBuilderAllocator(): _buffer() {}
|
||||
|
||||
bool Allocate(void** data, size_t size) override {
|
||||
_buffer.resize(size);
|
||||
*data = _buffer.data();
|
||||
_commited = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
int Commit() override {
|
||||
_commited = true;
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool Rollback() override {
|
||||
_buffer.resize(0);
|
||||
_commited = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
inline void Reserve(size_t capacity) {
|
||||
_buffer.reserve(capacity);
|
||||
}
|
||||
|
||||
inline bool IsCommited() const {
|
||||
return _commited;
|
||||
}
|
||||
|
||||
inline Event GetEvent() const {
|
||||
return Event(_buffer.data(), _buffer.size());
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> _buffer;
|
||||
bool _commited;
|
||||
};
|
||||
|
||||
|
||||
class IEventPrioritizer {
|
||||
public:
|
||||
virtual uint16_t Prioritize(const Event& event) = 0;
|
||||
|
|
@ -270,7 +415,7 @@ private:
|
|||
|
||||
class EventBuilder {
|
||||
public:
|
||||
EventBuilder(std::shared_ptr<IEventBuilderAllocator> allocator, std::shared_ptr<IEventPrioritizer> prioritizer): _allocator(allocator), _prioritizer(prioritizer), _data(nullptr), _size(0)
|
||||
EventBuilder(std::shared_ptr<IEventBuilderAllocator> allocator, std::shared_ptr<IEventPrioritizer> prioritizer): _allocator(allocator), _prioritizer(prioritizer), _data(nullptr), _size(0), _extensions_offset(0)
|
||||
{}
|
||||
|
||||
~EventBuilder() = default;
|
||||
|
|
@ -278,7 +423,7 @@ public:
|
|||
bool BeginEvent(uint64_t sec, uint32_t msec, uint64_t serial, uint16_t num_records);
|
||||
void SetEventPriority(uint16_t flags);
|
||||
uint16_t GetEventPriority();
|
||||
void SetEventFlags(uint16_t flags);
|
||||
void AddEventFlags(uint16_t flags);
|
||||
uint16_t GetEventFlags();
|
||||
void SetEventPid(int32_t pid);
|
||||
int32_t GetEventPid();
|
||||
|
|
@ -290,6 +435,9 @@ public:
|
|||
bool AddField(const char *field_name, const char* raw_value, const char* interp_value, field_type_t field_type);
|
||||
bool AddField(const std::string_view& field_name, const std::string_view& raw_value, const std::string_view& interp_value, field_type_t field_type);
|
||||
int GetFieldCount();
|
||||
bool BeginExtensions(uint32_t num_extensions);
|
||||
bool AddExtension(uint32_t type, uint32_t size, void* data);
|
||||
bool EndExtensions();
|
||||
|
||||
private:
|
||||
std::shared_ptr<IEventBuilderAllocator> _allocator;
|
||||
|
|
@ -304,6 +452,9 @@ private:
|
|||
uint32_t _record_idx;
|
||||
uint16_t _num_fields;
|
||||
uint32_t _field_idx;
|
||||
uint32_t _extensions_offset;
|
||||
uint32_t _eoffset;
|
||||
uint32_t _extension_idx;
|
||||
};
|
||||
|
||||
std::string EventToRawText(const Event& event, bool include_interp);
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,399 @@
|
|||
/*
|
||||
microsoft-oms-auditd-plugin
|
||||
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef AUOMS_EVENTAGGREGATOR_H
|
||||
#define AUOMS_EVENTAGGREGATOR_H
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
#include <queue>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include <string_view>
|
||||
#include <map>
|
||||
|
||||
#include "EventMatcher.h"
|
||||
#include "EventId.h"
|
||||
|
||||
#include <rapidjson/document.h>
|
||||
|
||||
/*
|
||||
|
||||
[
|
||||
{
|
||||
"match_rule": {
|
||||
"record_types": [],
|
||||
"field_rules": [
|
||||
{
|
||||
"field_name": "syscall",
|
||||
"op": "re",
|
||||
"value": "execve"
|
||||
},
|
||||
{
|
||||
"field_name": "cmdline",
|
||||
"op": "re",
|
||||
"value": "/stuff/[0-9]+/foo"
|
||||
},
|
||||
]
|
||||
},
|
||||
"aggregation_fields": {
|
||||
"time": {
|
||||
"mode": "raw" // interp, dynamic
|
||||
"output_name": "aggregated_time"
|
||||
}
|
||||
"pid": {},
|
||||
},
|
||||
"time_field_mode": "drop", (full, delta, drop)
|
||||
"serial_field_mode": "drop",
|
||||
"max_size": 2048,
|
||||
"max_time": 300,
|
||||
"send_first_as_raw": true
|
||||
},
|
||||
{
|
||||
"match_rule": {
|
||||
"record_types": [],
|
||||
"field_rules": [
|
||||
{
|
||||
"field_name": "syscall",
|
||||
"op": "re",
|
||||
"values": ["execve","open"]
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
*/
|
||||
|
||||
enum class AggregationFieldMode: int {
|
||||
NORMAL = -1,
|
||||
DROP = 0,
|
||||
RAW = 1,
|
||||
INTERP = 2,
|
||||
DYNAMIC = 3,
|
||||
DELTA = 4,
|
||||
};
|
||||
|
||||
class AggregationField {
|
||||
public:
|
||||
AggregationField(const std::string& name): _name(name), _mode(AggregationFieldMode::DYNAMIC), _output_name(name) {}
|
||||
AggregationField(const std::string& name, AggregationFieldMode mode): _name(name), _mode(mode), _output_name(name) {}
|
||||
AggregationField(const std::string& name, AggregationFieldMode mode, const std::string& output_name): _name(name), _mode(mode), _output_name(output_name) {}
|
||||
|
||||
inline const std::string& Name() const {
|
||||
return _name;
|
||||
}
|
||||
|
||||
inline AggregationFieldMode Mode() const {
|
||||
return _mode;
|
||||
}
|
||||
|
||||
inline const std::string& OutputName() const {
|
||||
return _output_name;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string _name;
|
||||
AggregationFieldMode _mode;
|
||||
std::string _output_name;
|
||||
};
|
||||
|
||||
class AggregationRule {
|
||||
public:
|
||||
static constexpr uint32_t DEFAULT_MAX_PENDING = 1024;
|
||||
static constexpr uint32_t MIN_MAX_PENDING = 1;
|
||||
static constexpr uint32_t MAX_MAX_PENDING = 10240;
|
||||
static constexpr uint32_t DEFAULT_MAX_SIZE = 8192;
|
||||
static constexpr uint32_t MIN_MAX_SIZE = 128;
|
||||
static constexpr uint32_t MAX_MAX_SIZE = 128*1024;
|
||||
static constexpr uint32_t DEFAULT_MAX_COUNT = 1024;
|
||||
static constexpr uint32_t MIN_MAX_COUNT = 2;
|
||||
static constexpr uint32_t MAX_MAX_COUNT = 128*1024;
|
||||
static constexpr uint32_t DEFAULT_MAX_TIME = 900; // 15 minutes
|
||||
static constexpr uint32_t MIN_MAX_TIME = 1; // 1 second
|
||||
static constexpr uint32_t MAX_MAX_TIME = 3600; // 1 hour
|
||||
static constexpr bool DEFAULT_SEND_FIRST = false;
|
||||
|
||||
|
||||
AggregationRule(const std::shared_ptr<EventMatchRule>& match_rule, const std::vector<AggregationField>& aggregation_fields,
|
||||
AggregationFieldMode time_field_mode, AggregationFieldMode serial_field_mode,
|
||||
uint32_t max_pending, uint32_t max_count, uint32_t max_size, uint32_t max_time, bool send_first)
|
||||
: _match_rule(match_rule), _aggregation_fields(aggregation_fields), _aggregation_fields_map(),
|
||||
_time_field_mode(time_field_mode), _serial_field_mode(serial_field_mode),
|
||||
_max_pending(max_pending), _max_count(max_count), _max_size(max_size), _max_time(max_time), _send_first(send_first)
|
||||
{
|
||||
if (_max_pending < MIN_MAX_PENDING) {
|
||||
_max_pending = MIN_MAX_PENDING;
|
||||
} else if (_max_pending > MAX_MAX_PENDING) {
|
||||
_max_pending = MAX_MAX_PENDING;
|
||||
}
|
||||
|
||||
if (_max_count < MIN_MAX_COUNT) {
|
||||
_max_count = MIN_MAX_COUNT;
|
||||
} else if (_max_count > MAX_MAX_COUNT) {
|
||||
_max_count = MAX_MAX_COUNT;
|
||||
}
|
||||
|
||||
if (_max_size < MIN_MAX_SIZE) {
|
||||
_max_size - MIN_MAX_SIZE;
|
||||
} else if (_max_size > MAX_MAX_SIZE) {
|
||||
_max_size - MAX_MAX_SIZE;
|
||||
}
|
||||
|
||||
if (_max_time < MIN_MAX_TIME) {
|
||||
_max_time = MIN_MAX_TIME;
|
||||
} else if (_max_time > MAX_MAX_TIME) {
|
||||
_max_time = MAX_MAX_TIME;
|
||||
}
|
||||
|
||||
_num_drop_fields = 0;
|
||||
for (int i = 0; i < _aggregation_fields.size(); ++i) {
|
||||
if (_aggregation_fields[i].Mode() == AggregationFieldMode::DROP) {
|
||||
_num_drop_fields += 1;
|
||||
}
|
||||
_aggregation_fields_map.emplace(std::make_pair(std::string_view(_aggregation_fields[i].Name()), i));
|
||||
}
|
||||
}
|
||||
|
||||
static void RulesFromJSON(const rapidjson::Value& value, std::vector<std::shared_ptr<AggregationRule>>& rules);
|
||||
static std::shared_ptr<AggregationRule> FromJSON(const rapidjson::Value& value);
|
||||
static std::shared_ptr<AggregationRule> FromJSON(const std::string& str);
|
||||
void ToJSON(rapidjson::Writer<rapidjson::StringBuffer>& writer) const;
|
||||
std::string ToJSONString() const;
|
||||
|
||||
inline std::shared_ptr<EventMatchRule> MatchRule() const {
|
||||
return _match_rule;
|
||||
}
|
||||
|
||||
inline const std::vector<AggregationField>& AggregationFields() const {
|
||||
return _aggregation_fields;
|
||||
}
|
||||
|
||||
inline int NumDropFields() const {
|
||||
return _num_drop_fields;
|
||||
}
|
||||
|
||||
inline AggregationFieldMode FieldMode(const std::string_view& name) const {
|
||||
auto it = _aggregation_fields_map.find(name);
|
||||
if (it != _aggregation_fields_map.end()) {
|
||||
return _aggregation_fields[it->second].Mode();
|
||||
} else {
|
||||
return AggregationFieldMode::NORMAL;
|
||||
}
|
||||
}
|
||||
|
||||
inline bool HasAggregationField(const std::string_view& name) const {
|
||||
return _aggregation_fields_map.count(name) > 0;
|
||||
}
|
||||
|
||||
inline AggregationFieldMode TimeFieldMode() const {
|
||||
return _time_field_mode;
|
||||
}
|
||||
|
||||
inline AggregationFieldMode SerialFieldMode() const {
|
||||
return _serial_field_mode;
|
||||
}
|
||||
|
||||
inline uint32_t MaxPending() const {
|
||||
return _max_pending;
|
||||
}
|
||||
|
||||
inline uint32_t MaxCount() const {
|
||||
return _max_count;
|
||||
}
|
||||
|
||||
inline uint32_t MaxSize() const {
|
||||
return _max_size;
|
||||
}
|
||||
|
||||
inline uint32_t MaxTime() const {
|
||||
return _max_time;
|
||||
}
|
||||
|
||||
inline bool SendFirst() const {
|
||||
return _send_first;
|
||||
}
|
||||
|
||||
// The aggregation key is the set of non-aggregated fields
|
||||
// The string_views placed in key point to data in event and thus have the
|
||||
// same validity lifespan
|
||||
void CalcAggregationKey(std::vector<std::string_view>& key, const Event& event) const;
|
||||
|
||||
private:
|
||||
std::shared_ptr<EventMatchRule> _match_rule;
|
||||
std::vector<AggregationField> _aggregation_fields;
|
||||
std::unordered_map<std::string_view, int> _aggregation_fields_map;
|
||||
int _num_drop_fields;
|
||||
AggregationFieldMode _time_field_mode;
|
||||
AggregationFieldMode _serial_field_mode;
|
||||
uint32_t _max_pending;
|
||||
uint32_t _max_count;
|
||||
uint32_t _max_size; //bytes
|
||||
uint32_t _max_time; //seconds
|
||||
bool _send_first;
|
||||
};
|
||||
|
||||
class EventAggregator;
|
||||
|
||||
class AggregatedEvent {
|
||||
public:
|
||||
AggregatedEvent(const std::shared_ptr<AggregationRule>& rule): _rule(rule), _last_event(0, 0, 0) {
|
||||
_id = _next_id.fetch_add(1);
|
||||
_count = 0;
|
||||
_expiration_time = std::chrono::steady_clock::now() + std::chrono::seconds(_rule->MaxTime());
|
||||
_data.reserve(AggregationRule::MIN_MAX_SIZE);
|
||||
_aggregated_fields.resize(_rule->AggregationFields().size());
|
||||
_event_times.reserve(AggregationRule::MIN_MAX_COUNT);
|
||||
_event_serials.reserve(AggregationRule::MIN_MAX_COUNT);
|
||||
for (auto& x : _aggregated_fields) {
|
||||
x.reserve(AggregationRule::MIN_MAX_COUNT);
|
||||
}
|
||||
}
|
||||
|
||||
static std::shared_ptr<AggregatedEvent> Read(FILE* file, std::vector<std::shared_ptr<AggregationRule>> rules);
|
||||
void Write(FILE* file, const std::unordered_map<std::shared_ptr<AggregationRule>, int>& rules_map) const;
|
||||
|
||||
inline const std::shared_ptr<AggregationRule>& Rule() const {
|
||||
return _rule;
|
||||
}
|
||||
|
||||
inline bool Empty() const {
|
||||
return _count == 0;
|
||||
}
|
||||
|
||||
inline uint64_t Id() const {
|
||||
return _id;
|
||||
}
|
||||
|
||||
inline std::chrono::steady_clock::time_point ExpirationTime() const {
|
||||
return _expiration_time;
|
||||
}
|
||||
|
||||
inline std::pair<std::chrono::steady_clock::time_point, uint64_t> AgeKey() const {
|
||||
return std::make_pair(_expiration_time, _id);
|
||||
}
|
||||
|
||||
inline const std::vector<std::string_view>& AggregationKey() const {
|
||||
return _agg_key;
|
||||
}
|
||||
|
||||
// Return true of the event was added
|
||||
// Return false if the event was not added (and thus the AggregatedEvent is full)
|
||||
bool AddEvent(const Event& event);
|
||||
|
||||
int BuildEvent(EventBuilder& builder, rapidjson::StringBuffer& buffer) const;
|
||||
|
||||
private:
|
||||
friend class EventAggregator;
|
||||
|
||||
AggregatedEvent() {}
|
||||
|
||||
static std::atomic<uint64_t> _next_id;
|
||||
|
||||
std::shared_ptr<AggregationRule> _rule;
|
||||
std::chrono::steady_clock::time_point _expiration_time;
|
||||
uint64_t _id;
|
||||
EventId _first_event;
|
||||
EventId _last_event;
|
||||
uint32_t _count;
|
||||
std::vector<uint8_t> _origin_event;
|
||||
std::vector<std::string_view> _agg_key;
|
||||
std::string _data;
|
||||
std::vector<std::pair<size_t, size_t>> _event_times;
|
||||
std::vector<std::pair<size_t, size_t>> _event_serials;
|
||||
std::vector<std::vector<std::pair<size_t, size_t>>> _aggregated_fields;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct std::hash<std::vector<std::string_view>>
|
||||
{
|
||||
std::size_t operator()(std::vector<std::string_view> const& v) const noexcept
|
||||
{
|
||||
size_t h = 0;
|
||||
for (auto& i : v) {
|
||||
// This is algorithm is taken from boost:hash_combine();
|
||||
h ^= std::hash<std::string_view>{}(i) + 0x9e3779b9 + (h << 6) + (h >> 2);
|
||||
}
|
||||
return h;
|
||||
}
|
||||
};
|
||||
|
||||
class EventAggregator {
|
||||
public:
|
||||
EventAggregator():
|
||||
_allocator(std::make_shared<BasicEventBuilderAllocator>(256*1024)),
|
||||
_builder(_allocator, DefaultPrioritizer::Create(0)),
|
||||
_matcher(std::make_shared<EventMatcher>())
|
||||
{}
|
||||
|
||||
// Set rules
|
||||
// If existing rules exist, any events associated with old rules that are not in the new set, will be flushed to the _ready_events queue.
|
||||
void SetRules(const std::vector<std::shared_ptr<AggregationRule>>& rules);
|
||||
|
||||
// Load saved aggregation state from file
|
||||
// Any previous state is lost
|
||||
void Load(const std::string& path);
|
||||
|
||||
// Save aggregation state to file
|
||||
void Save(const std::string& path);
|
||||
|
||||
// Check if event is aggregated
|
||||
// Return true if the event was consumed (aggregated)
|
||||
bool AddEvent(const Event& event);
|
||||
|
||||
// Check for complete/ready aggregated events and handle one
|
||||
// If handler_fn returns ret.second == true, then event was consumed
|
||||
// ret.get<0>, true if handler_fn invoked, false if no events ready
|
||||
// ret.get<1>, if get<0> is true, then ret.first from handler_fn, otherwise ret from AggregatedEvent::BuildEvent()
|
||||
// ret.get<2>, if get<0> is true, then ret.second from handler_fn, otherwise false
|
||||
std::tuple<bool, int64_t, bool> HandleEvent(const std::function<std::pair<int64_t, bool> (const Event& event)>& handler_fn);
|
||||
|
||||
size_t NumReadyAggregates() const {
|
||||
return _ready_events.size();
|
||||
}
|
||||
|
||||
size_t NumPendingAggregates() const {
|
||||
size_t count = 0;
|
||||
for (auto& e : _events) {
|
||||
count += e->_events.size();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private:
|
||||
class PerRuleAgg {
|
||||
public:
|
||||
explicit PerRuleAgg(const std::shared_ptr<AggregationRule>& rule): _rule(rule), _events(16) {}
|
||||
|
||||
std::shared_ptr<AggregationRule> _rule;
|
||||
std::unordered_map<std::vector<std::string_view>,std::shared_ptr<AggregatedEvent>> _events;
|
||||
std::map<std::pair<std::chrono::steady_clock::time_point, uint64_t>, std::vector<std::string_view>> _events_age;
|
||||
};
|
||||
|
||||
std::vector<std::shared_ptr<AggregationRule>> _rules;
|
||||
std::shared_ptr<EventMatcher> _matcher;
|
||||
std::vector<std::shared_ptr<PerRuleAgg>> _events;
|
||||
std::map<std::pair<std::chrono::steady_clock::time_point, uint64_t>, std::pair<std::shared_ptr<AggregatedEvent>, int>> _aged_events;
|
||||
std::queue<std::shared_ptr<AggregatedEvent>> _ready_events;
|
||||
std::vector<std::string_view> _tmp_key;
|
||||
rapidjson::StringBuffer _js_buffer;
|
||||
std::shared_ptr<BasicEventBuilderAllocator> _allocator;
|
||||
EventBuilder _builder;
|
||||
};
|
||||
|
||||
#endif //AUOMS_EVENTAGGREGATOR_H
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,440 @@
|
|||
/*
|
||||
microsoft-oms-auditd-plugin
|
||||
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "EventMatcher.h"
|
||||
|
||||
#include "RecordType.h"
|
||||
#include "Translate.h"
|
||||
#include "StringUtils.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <sstream>
|
||||
#include <re2/re2.h>
|
||||
#include <re2/set.h>
|
||||
#include <re2/stringpiece.h>
|
||||
|
||||
#include "rapidjson/document.h"
|
||||
#include "rapidjson/error/en.h"
|
||||
|
||||
static std::unordered_map<std::string, FieldMatchRuleOp> s_opName2op = {
|
||||
{"eq", FIELD_OP_EQ},
|
||||
{"!eq", FIELD_OP_NEQ},
|
||||
{"in", FIELD_OP_IN},
|
||||
{"!in", FIELD_OP_NIN},
|
||||
{"re", FIELD_OP_RE},
|
||||
{"!re", FIELD_OP_NRE},
|
||||
};
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
|
||||
std::shared_ptr<FieldMatchRule> FieldMatchRule::FromJSON(const rapidjson::Value& value) {
|
||||
if (!value.IsObject()) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): value is not a JSON object");
|
||||
}
|
||||
|
||||
std::string name;
|
||||
std::string op_name;
|
||||
FieldMatchRuleOp op;
|
||||
std::vector<std::string> values;
|
||||
|
||||
auto m = value.FindMember("name");
|
||||
if (m == value.MemberEnd()) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): Missing 'name'");
|
||||
}
|
||||
name = m->value.GetString();
|
||||
|
||||
m = value.FindMember("op");
|
||||
if (m == value.MemberEnd()) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): Missing 'op'");
|
||||
}
|
||||
op_name = m->value.GetString();
|
||||
|
||||
m = value.FindMember("value");
|
||||
if (m != value.MemberEnd()) {
|
||||
if (value.HasMember("values")) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): Only one of 'value' or 'values' is allowed");
|
||||
}
|
||||
|
||||
if (!m->value.IsString()) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): Invalid JSON type for 'value', must be a string");
|
||||
}
|
||||
values.emplace_back(m->value.GetString());
|
||||
} else {
|
||||
m = value.FindMember("values");
|
||||
|
||||
if (m == value.MemberEnd()) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): Missing values, one of 'value' or 'values' required");
|
||||
}
|
||||
if (!m->value.IsArray()) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): Invalid JSON type for 'values', must be an array");
|
||||
}
|
||||
if (m->value.Size() == 0) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): 'values' array is empty");
|
||||
}
|
||||
values.reserve(m->value.Size());
|
||||
for (auto it = m->value.Begin(); it != m->value.End(); ++it) {
|
||||
if (!it->IsString()) {
|
||||
throw new std::invalid_argument("FieldMatchRule::FromJSON(): Invalid JSON type for entry in 'values' array");
|
||||
}
|
||||
values.emplace_back(it->GetString());
|
||||
}
|
||||
}
|
||||
|
||||
std::transform(op_name.begin(), op_name.end(), op_name.begin(), [](unsigned char c){ return std::tolower(c); });
|
||||
|
||||
auto opit = s_opName2op.find(op_name);
|
||||
if (opit == s_opName2op.end()) {
|
||||
throw new std::invalid_argument(std::string("FieldMatchRule::FromJSON(): Invalid op value: ") + op_name);
|
||||
}
|
||||
|
||||
return std::make_shared<FieldMatchRule>(name, opit->second, values);
|
||||
}
|
||||
|
||||
std::shared_ptr<FieldMatchRule> FieldMatchRule::FromJSON(const std::string& str) {
|
||||
rapidjson::Document doc;
|
||||
doc.Parse(str.c_str());
|
||||
if (doc.HasParseError()) {
|
||||
throw std::runtime_error(rapidjson::GetParseError_En(doc.GetParseError()));
|
||||
}
|
||||
return FromJSON(doc);
|
||||
}
|
||||
|
||||
void FieldMatchRule::ToJSON(rapidjson::Writer<rapidjson::StringBuffer>& writer) const {
|
||||
writer.StartObject();
|
||||
writer.Key("name");
|
||||
writer.String(_name.data(), _name.size());
|
||||
writer.Key("op");
|
||||
switch(_op) {
|
||||
case FIELD_OP_EQ:
|
||||
writer.String("eq");
|
||||
break;
|
||||
case FIELD_OP_NEQ:
|
||||
writer.String("!eq");
|
||||
break;
|
||||
case FIELD_OP_IN:
|
||||
writer.String("in");
|
||||
break;
|
||||
case FIELD_OP_NIN:
|
||||
writer.String("!in");
|
||||
break;
|
||||
case FIELD_OP_RE:
|
||||
writer.String("re");
|
||||
break;
|
||||
case FIELD_OP_NRE:
|
||||
writer.String("!re");
|
||||
break;
|
||||
default:
|
||||
writer.String("unknown");
|
||||
break;
|
||||
}
|
||||
if (_values.size() < 2) {
|
||||
writer.Key("value");
|
||||
writer.String(_values[0].data(), _values[0].size());
|
||||
} else {
|
||||
writer.StartArray();
|
||||
for (auto& s : _values) {
|
||||
writer.String(s.data(), s.size());
|
||||
}
|
||||
writer.EndArray();
|
||||
}
|
||||
writer.EndObject();
|
||||
}
|
||||
|
||||
std::string FieldMatchRule::ToJSONString() const {
|
||||
rapidjson::StringBuffer buf;
|
||||
rapidjson::Writer<rapidjson::StringBuffer> writer(buf);
|
||||
ToJSON(writer);
|
||||
return std::string(buf.GetString(), buf.GetSize());
|
||||
}
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
|
||||
std::shared_ptr<EventMatchRule> EventMatchRule::FromJSON(const rapidjson::Value& value) {
|
||||
if (!value.IsObject()) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): value is not a JSON object");
|
||||
}
|
||||
|
||||
auto m = value.FindMember("record_types");
|
||||
if (m == value.MemberEnd()) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): Missing 'record_types'");
|
||||
}
|
||||
std::unordered_set<RecordType> rctypes;
|
||||
if (!m->value.IsArray()) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): Invalid JSON type for 'record_types', must be an array");
|
||||
}
|
||||
if (m->value.Size() == 0) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): 'record_types' array is empty");
|
||||
}
|
||||
for (auto it = m->value.Begin(); it != m->value.End(); ++it) {
|
||||
if (!it->IsString()) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): Invalid JSON type for entry in 'record_types' array");
|
||||
}
|
||||
auto rc = RecordNameToType(std::string_view(it->GetString(), it->GetStringLength()));
|
||||
rctypes.emplace(rc);
|
||||
}
|
||||
|
||||
m = value.FindMember("field_rules");
|
||||
if (m == value.MemberEnd()) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): Missing 'field_rules'");
|
||||
}
|
||||
if (!m->value.IsArray()) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): Invalid JSON type for 'field_rules', must be an array");
|
||||
}
|
||||
if (m->value.Size() == 0) {
|
||||
throw new std::invalid_argument("EventMatchRule::FromJSON(): 'field_rules' array is empty");
|
||||
}
|
||||
std::vector<std::shared_ptr<FieldMatchRule>> rules;
|
||||
rules.reserve(m->value.Size());
|
||||
for (auto it = m->value.Begin(); it != m->value.End(); ++it) {
|
||||
rules.emplace_back(FieldMatchRule::FromJSON(*it));
|
||||
}
|
||||
|
||||
return std::make_shared<EventMatchRule>(rctypes, rules);
|
||||
}
|
||||
|
||||
std::shared_ptr<EventMatchRule> EventMatchRule::FromJSON(const std::string& str) {
|
||||
rapidjson::Document doc;
|
||||
doc.Parse(str.c_str());
|
||||
if (doc.HasParseError()) {
|
||||
throw std::runtime_error(rapidjson::GetParseError_En(doc.GetParseError()));
|
||||
}
|
||||
return FromJSON(doc);
|
||||
}
|
||||
|
||||
void EventMatchRule::ToJSON(rapidjson::Writer<rapidjson::StringBuffer>& writer) const {
|
||||
writer.StartObject();
|
||||
|
||||
writer.Key("record_types");
|
||||
writer.StartArray();
|
||||
RecordType rtypes[_record_types.size()];
|
||||
std::copy(_record_types.begin(), _record_types.end(), &rtypes[0]);
|
||||
std::sort(&rtypes[0], &rtypes[_record_types.size()]);
|
||||
for (auto& r : rtypes) {
|
||||
auto name = RecordTypeToName(r);
|
||||
writer.String(name.data(), name.size());
|
||||
}
|
||||
writer.EndArray();
|
||||
|
||||
writer.Key("field_rules");
|
||||
writer.StartArray();
|
||||
for (auto& r : _rules) {
|
||||
r->ToJSON(writer);
|
||||
}
|
||||
writer.EndArray();
|
||||
|
||||
writer.EndObject();
|
||||
}
|
||||
|
||||
std::string EventMatchRule::ToJSONString() const {
|
||||
rapidjson::StringBuffer buf;
|
||||
rapidjson::Writer<rapidjson::StringBuffer> writer(buf);
|
||||
ToJSON(writer);
|
||||
return std::string(buf.GetString(), buf.GetSize());
|
||||
}
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
|
||||
class EventMatcher::FieldMatcher {
|
||||
public:
|
||||
FieldMatcher(const std::string& name, int num_event_rules, int index): _name(name), _num_event_rules(num_event_rules), _index(index), _re_set(RE2::Options(), RE2::UNANCHORED) {
|
||||
RE2::Options opts;
|
||||
opts.set_never_capture(true);
|
||||
|
||||
_re_set = RE2::Set(opts, RE2::UNANCHORED);
|
||||
|
||||
_rules.assign(_num_event_rules, nullptr);
|
||||
_shift_counts.resize(_num_event_rules, 1);
|
||||
_not_mask.resize(_num_event_rules, 0);
|
||||
}
|
||||
|
||||
void AddPatterns(int em_idx, std::shared_ptr<FieldMatchRule> rule) {
|
||||
assert(em_idx < _rules.size());
|
||||
_rules[em_idx] = rule;
|
||||
}
|
||||
|
||||
bool Compile(int num_event_rules, std::vector<std::string>* errors) {
|
||||
for (int i = 0; i < _rules.size(); i++) {
|
||||
if (_rules[i] != nullptr) {
|
||||
_shift_counts[i] = _rules[i]->MinMatch();
|
||||
_not_mask[i] = _rules[i]->Op() & FIELD_OP_NOT;
|
||||
}
|
||||
}
|
||||
|
||||
std::string error;
|
||||
for (int i = 0; i < _rules.size(); i++) {
|
||||
if (_rules[i] != nullptr) {
|
||||
for (auto& p : _rules[i]->Values()) {
|
||||
std::string e;
|
||||
auto idx = _re_set.Add(p, &e);
|
||||
if (idx < 0) {
|
||||
errors->emplace_back("Invalid pattern '" + p + "': " + e);
|
||||
return false;
|
||||
}
|
||||
_to_rule.emplace_back(i);
|
||||
assert(_to_rule.size() == idx+1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!_re_set.Compile()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
_matches.clear();
|
||||
_matches.reserve(_to_rule.size());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Match(const EventRecord& record, uint32_t* ruleMatchedFields) {
|
||||
auto f = record.FieldByName(_name);
|
||||
|
||||
if (!f) {
|
||||
return false;
|
||||
}
|
||||
|
||||
re2::StringPiece val;
|
||||
if (f.InterpValueSize() > 0) {
|
||||
val.set(f.InterpValuePtr(), f.InterpValueSize());
|
||||
} else {
|
||||
val.set(f.RawValuePtr(), f.RawValueSize());
|
||||
}
|
||||
|
||||
if (!_re_set.Match(val, &_matches)) {
|
||||
uint32_t matchCount = 0;
|
||||
for (int i = 0; i < _not_mask.size(); i++) {
|
||||
auto tmp = (0 ^ _not_mask[i]);
|
||||
ruleMatchedFields[i] |= tmp << _index;
|
||||
matchCount += tmp;
|
||||
}
|
||||
return matchCount > 0;
|
||||
}
|
||||
|
||||
int32_t tmp[_shift_counts.size()];
|
||||
std::fill(tmp, tmp+_shift_counts.size(), 0x80000000);
|
||||
|
||||
for (auto m :_matches) {
|
||||
tmp[_to_rule[m]] >>= 1;
|
||||
}
|
||||
|
||||
for (int i = 0; i < _shift_counts.size(); i++) {
|
||||
ruleMatchedFields[i] |= (((static_cast<uint32_t>(tmp[i]) >> 31-_shift_counts[i]) & 1) ^ _not_mask[i]) << _index;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string _name;
|
||||
int _num_event_rules;
|
||||
int _index;
|
||||
std::vector<std::shared_ptr<FieldMatchRule>> _rules;
|
||||
RE2::Set _re_set;
|
||||
std::vector<int> _to_rule;
|
||||
std::vector<int32_t> _shift_counts;
|
||||
std::vector<uint32_t> _not_mask;
|
||||
std::vector<int> _matches;
|
||||
};
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
|
||||
bool EventMatcher::Compile(const std::vector<std::shared_ptr<EventMatchRule>>& rules) {
|
||||
_rules.clear();
|
||||
for (auto& r : rules) {
|
||||
_rules.emplace_back(r);
|
||||
}
|
||||
_fields.clear();
|
||||
_errors.clear();
|
||||
_rules_field_mask.assign(_rules.size(), 0);
|
||||
_record_type_field_mask.clear();
|
||||
_fieldsMap.clear();
|
||||
|
||||
for (int i = 0; i < _rules.size(); i++) {
|
||||
auto& r = _rules[i];
|
||||
for (auto& f : r->Rules()) {
|
||||
std::shared_ptr<FieldMatcher> fm;
|
||||
auto fmi = _fieldsMap.find(f->Name());
|
||||
if (fmi == _fieldsMap.end()) {
|
||||
if (_fields.size() >= 32) {
|
||||
_errors.emplace_back("Number of fields (" + std::to_string(0) + ") exceeds limit of 32");
|
||||
return false;
|
||||
}
|
||||
fm = std::make_shared<FieldMatcher>(f->Name(), _rules.size(), _fields.size());
|
||||
_fieldsMap[f->Name()] = fm;
|
||||
_fields.emplace_back(fm);
|
||||
} else {
|
||||
fm = fmi->second;
|
||||
}
|
||||
std::string error;
|
||||
fm->AddPatterns(i, f);
|
||||
_rules_field_mask[i] |= 1 << fm->_index;
|
||||
for (auto rt : r->RecordTypes()) {
|
||||
_record_type_field_mask[static_cast<uint32_t>(rt)] |= 1 << fm->_index;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool failed = false;
|
||||
for (auto& f : _fields) {
|
||||
if (!f->Compile(_rules.size(), &_errors)) {
|
||||
_errors.emplace_back("Failed to compile RE2::Set for field '" + f->_name + '"');
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
return !failed;
|
||||
}
|
||||
|
||||
int EventMatcher::Match(const Event& event) {
|
||||
if (event.NumRecords() < 1) {
|
||||
return -1;
|
||||
}
|
||||
auto record = event.RecordAt(0);
|
||||
|
||||
auto rmi = _record_type_field_mask.find(record.RecordType());
|
||||
if (rmi == _record_type_field_mask.end()) {
|
||||
return -1;
|
||||
}
|
||||
uint32_t record_field_mask = rmi->second;
|
||||
if (record_field_mask == 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint32_t fieldMatches[_rules.size()];
|
||||
memset(fieldMatches, 0, _rules.size()*sizeof(uint32_t));
|
||||
|
||||
int fcount = 0;
|
||||
for (auto& f : _fields) {
|
||||
if (((1<< f->_index) & record_field_mask) != 0) {
|
||||
f->Match(record, fieldMatches);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < _rules.size(); i++) {
|
||||
if ((fieldMatches[i] & _rules_field_mask[i]) == _rules_field_mask[i] && _rules[i]->RecordTypes().count(static_cast<RecordType>(record.RecordType())) > 0) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
/*
|
||||
microsoft-oms-auditd-plugin
|
||||
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef AUOMS_EVENTMATCHER_H
|
||||
#define AUOMS_EVENTMATCHER_H
|
||||
|
||||
#include <memory>
|
||||
#include <functional>
|
||||
#include <unordered_set>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "Event.h"
|
||||
#include "RecordType.h"
|
||||
|
||||
#include <rapidjson/document.h>
|
||||
#include <rapidjson/stringbuffer.h>
|
||||
#include <rapidjson/writer.h>
|
||||
|
||||
|
||||
enum FieldMatchRuleOp: int {
|
||||
FIELD_OP_NOT = 1,
|
||||
FIELD_OP_EQ = 1<<1,
|
||||
FIELD_OP_NEQ = FIELD_OP_EQ|FIELD_OP_NOT,
|
||||
FIELD_OP_IN = 2<<1,
|
||||
FIELD_OP_NIN = FIELD_OP_IN|FIELD_OP_NOT,
|
||||
FIELD_OP_RE = 3<<1,
|
||||
FIELD_OP_NRE = FIELD_OP_RE|FIELD_OP_NOT,
|
||||
};
|
||||
|
||||
class FieldMatchRule {
|
||||
public:
|
||||
FieldMatchRule(const std::string& name, FieldMatchRuleOp op, const std::string& value): _name(name), _op(op) {
|
||||
switch (op&(~FIELD_OP_NOT)) {
|
||||
case FIELD_OP_EQ:
|
||||
_values.emplace_back("^" + value + "$");
|
||||
break;
|
||||
case FIELD_OP_IN:
|
||||
_values.emplace_back("^" + value + "$");
|
||||
break;
|
||||
case FIELD_OP_RE:
|
||||
_values.emplace_back(value);
|
||||
break;
|
||||
}
|
||||
_min_match = 1;
|
||||
}
|
||||
FieldMatchRule(const std::string& name, FieldMatchRuleOp op, const std::vector<std::string>& values): _name(name), _op(op) {
|
||||
switch (op&(~FIELD_OP_NOT)) {
|
||||
case FIELD_OP_EQ:
|
||||
_values.emplace_back("^" + values[0] + "$");
|
||||
_min_match = 1;
|
||||
break;
|
||||
case FIELD_OP_IN:
|
||||
for (auto& v : values) {
|
||||
_values.emplace_back("^" + v + "$");
|
||||
}
|
||||
_min_match = 1;
|
||||
break;
|
||||
case FIELD_OP_RE:
|
||||
_values = values;
|
||||
_min_match = values.size();
|
||||
break;
|
||||
}
|
||||
std::sort(_values.begin(), _values.end());
|
||||
}
|
||||
|
||||
static std::shared_ptr<FieldMatchRule> FromJSON(const rapidjson::Value& value);
|
||||
static std::shared_ptr<FieldMatchRule> FromJSON(const std::string& str);
|
||||
void ToJSON(rapidjson::Writer<rapidjson::StringBuffer>& writer) const;
|
||||
std::string ToJSONString() const;
|
||||
|
||||
inline const std::string& Name() const {
|
||||
return _name;
|
||||
}
|
||||
|
||||
inline FieldMatchRuleOp Op() const {
|
||||
return _op;
|
||||
}
|
||||
|
||||
inline const std::vector<std::string>& Values() const {
|
||||
return _values;
|
||||
}
|
||||
|
||||
inline int MinMatch() const {
|
||||
return _min_match;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string _name;
|
||||
FieldMatchRuleOp _op;
|
||||
std::vector<std::string> _values;
|
||||
int _min_match;
|
||||
};
|
||||
|
||||
class EventMatchRule {
|
||||
public:
|
||||
EventMatchRule(const std::unordered_set<RecordType>& record_types, const std::vector<std::shared_ptr<FieldMatchRule>>& rules): _record_types(record_types) {
|
||||
_rules.reserve(rules.size());
|
||||
|
||||
for (auto& r : rules) {
|
||||
auto i = _rulesMap.find(r->Name());
|
||||
if (i == _rulesMap.end()) {
|
||||
_rules.emplace_back(r);
|
||||
_rulesMap.emplace(r->Name(), r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static std::shared_ptr<EventMatchRule> FromJSON(const rapidjson::Value& value);
|
||||
static std::shared_ptr<EventMatchRule> FromJSON(const std::string& str);
|
||||
void ToJSON(rapidjson::Writer<rapidjson::StringBuffer>& writer) const;
|
||||
std::string ToJSONString() const;
|
||||
|
||||
inline const std::vector<std::shared_ptr<FieldMatchRule>>& Rules() const {
|
||||
return _rules;
|
||||
}
|
||||
|
||||
inline const std::unordered_set<RecordType>& RecordTypes() const {
|
||||
return _record_types;
|
||||
}
|
||||
|
||||
inline std::shared_ptr<FieldMatchRule> Rule(const std::string& name) const {
|
||||
return _rulesMap.at(name);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unordered_set<RecordType> _record_types;
|
||||
std::vector<std::shared_ptr<FieldMatchRule>> _rules;
|
||||
std::unordered_map<std::string, std::shared_ptr<FieldMatchRule>> _rulesMap;
|
||||
};
|
||||
|
||||
class EventMatcher {
|
||||
public:
|
||||
bool Compile(const std::vector<std::shared_ptr<EventMatchRule>>& rules);
|
||||
|
||||
inline const std::vector<std::string>& Errors() {
|
||||
return _errors;
|
||||
}
|
||||
|
||||
// Return -1 if the event doesn't match any rule
|
||||
// Otherwise return the index of the match rule in the vector passed in via Compile().
|
||||
// If event matches multiple event rules, then the lowest index of all rules that matched is returned.
|
||||
int Match(const Event& event);
|
||||
|
||||
private:
|
||||
class FieldMatcher;
|
||||
|
||||
std::vector<std::shared_ptr<const EventMatchRule>> _rules;
|
||||
std::vector<uint32_t> _rules_field_mask;
|
||||
std::unordered_map<uint32_t, uint32_t> _record_type_field_mask;
|
||||
|
||||
std::vector<std::shared_ptr<FieldMatcher>> _fields;
|
||||
std::unordered_map<std::string, std::shared_ptr<FieldMatcher>> _fieldsMap;
|
||||
std::vector<std::string> _errors;
|
||||
};
|
||||
|
||||
#endif //AUOMS_EVENTMATCHER_H
|
||||
|
|
@ -0,0 +1,286 @@
|
|||
/*
|
||||
microsoft-oms-auditd-plugin
|
||||
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
//#define BOOST_TEST_DYN_LINK
|
||||
#define BOOST_TEST_MODULE "EventMatcherTests"
|
||||
#include <boost/test/unit_test.hpp>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "EventMatcher.h"
|
||||
#include "TestEventData.h"
|
||||
#include "TestEventWriter.h"
|
||||
#include "FieldType.h"
|
||||
#include "RecordType.h"
|
||||
|
||||
class TestRule {
|
||||
public:
|
||||
TestRule(bool should_match, const std::string& rule_json):
|
||||
_should_match(should_match) {
|
||||
_rule = EventMatchRule::FromJSON(rule_json);
|
||||
}
|
||||
|
||||
int _should_match;
|
||||
std::shared_ptr<EventMatchRule> _rule;
|
||||
};
|
||||
|
||||
class FieldMatcherTest {
|
||||
public:
|
||||
FieldMatcherTest(const std::string& name, const std::vector<TestRule>& rules): _name(name), _rules(rules), _error() {}
|
||||
|
||||
bool RunTest(const Event& event) {
|
||||
std::shared_ptr<EventMatchRule> match_rule = nullptr;
|
||||
std::vector<std::shared_ptr<EventMatchRule>> event_rules;
|
||||
for (auto& r : _rules) {
|
||||
event_rules.emplace_back(r._rule);
|
||||
if (!match_rule && r._should_match) {
|
||||
match_rule = r._rule;
|
||||
}
|
||||
}
|
||||
|
||||
EventMatcher matcher;
|
||||
|
||||
if (!matcher.Compile(event_rules)) {
|
||||
_error = "Test (" + _name + ") Failed:";
|
||||
for (auto& e : matcher.Errors()) {
|
||||
_error += "\n";
|
||||
_error += e;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Double compile is intentional to ensure that second compile doesn't break anything
|
||||
if (!matcher.Compile(event_rules)) {
|
||||
_error = "Test (" + _name + ") Failed:";
|
||||
for (auto& e : matcher.Errors()) {
|
||||
_error += "\n";
|
||||
_error += e;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto midx = matcher.Match(event);
|
||||
std::shared_ptr<EventMatchRule> m = nullptr;
|
||||
if (midx > -1) {
|
||||
m = event_rules[midx];
|
||||
}
|
||||
if (match_rule) {
|
||||
if (m != match_rule) {
|
||||
if (m) {
|
||||
_error = "Test (" + _name + ") Failed: Expected to match: " + match_rule->ToJSONString() + "\nBut instead matched: " + m->ToJSONString();
|
||||
} else {
|
||||
_error = "Test (" + _name + ") Failed: Expected to match: " + match_rule->ToJSONString() + "\nBut instead matched nothing";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if (m) {
|
||||
_error = "Test (" + _name + ") Failed: Should have matched nothing, but instead matched: " + m->ToJSONString();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
inline const std::string& Error() {
|
||||
return _error;
|
||||
}
|
||||
private:
|
||||
std::string _name;
|
||||
std::vector<TestRule> _rules;
|
||||
std::string _error;
|
||||
};
|
||||
|
||||
BOOST_AUTO_TEST_CASE( basic_test ) {
|
||||
auto allocator = std::make_shared<BasicEventBuilderAllocator>();
|
||||
auto prioritizer = DefaultPrioritizer::Create(0);
|
||||
auto builder = std::make_shared<EventBuilder>(std::dynamic_pointer_cast<IEventBuilderAllocator>(allocator), prioritizer);
|
||||
|
||||
builder->BeginEvent(0, 0, 0, 1);
|
||||
builder->BeginRecord(static_cast<uint32_t>(RecordType::AUOMS_EXECVE), "", "", 5);
|
||||
builder->AddField("syscall", "59", "execve", field_type_t::SYSCALL);
|
||||
builder->AddField("user", "1000", "test_user", field_type_t::UID);
|
||||
builder->AddField("group", "1000", "test_group", field_type_t::GID);
|
||||
builder->AddField("exe", "\"/usr/local/bin/testcmd\"", nullptr, field_type_t::ESCAPED);
|
||||
builder->AddField("cmdline", "testcmd arg1 arg2 arg3 lastarg", nullptr, field_type_t::UNESCAPED);
|
||||
builder->EndRecord();
|
||||
if (builder->EndEvent() != 1) {
|
||||
BOOST_FAIL("EndEvent failed");
|
||||
}
|
||||
|
||||
BOOST_CHECK_EQUAL(allocator->IsCommited(), true);
|
||||
|
||||
std::vector<std::string> patterns = {std::initializer_list<std::string>({"testcmd","connect"})};
|
||||
|
||||
std::vector<FieldMatcherTest> tests = {
|
||||
{
|
||||
"record type no match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_SYSCALL"], "field_rules": [{"name": "syscall", "op": "eq", "value": "execve"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field eq, match",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_SYSCALL","AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "eq", "value": "execve"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field eq, not match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "eq", "value": "open"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field !eq, match",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "!eq", "value": "open"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field !eq, not match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "!eq", "value": "execve"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"no field eq, not match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "nofield", "op": "eq", "value": "execve"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field in, match",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "in", "values": ["execve","execveat","connect"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field in, not match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "in", "values": ["open","execveat","connect"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field !in, match",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "!in", "values": ["open","execveat","connect"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field !in, not match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "!in", "values": ["execve","execveat","connect"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field re, match",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "cmdline", "op": "re", "values": ["^testcmd.*$", "^.*lastarg$"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field re, not match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "cmdline", "op": "re", "values": ["^bash.*$", "^.*lastarg$"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field !re, match",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "cmdline", "op": "!re", "values": ["^bash.*$", "^.*lastarg$"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"1 field !re, not match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "cmdline", "op": "!re", "values": ["^testcmd.*$", "^.*lastarg$"]}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"2 fields eq, match",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "eq", "value": "execve"},
|
||||
{"name": "user", "op": "eq", "value": "test_user"}
|
||||
]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"2 fields eq, no match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "eq", "value": "execve"},
|
||||
{"name": "user", "op": "eq", "value": "bob"}
|
||||
]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"2 rules, 1 field eq, match first",
|
||||
{{
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "eq", "value": "execve"}]})json"},
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "user", "op": "eq", "value": "test_user"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"3 rules, 1 field eq, match last",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_SYSCALL"], "field_rules": [{"name": "syscall", "op": "eq", "value": "execve"}]})json"},
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "doesnotexist", "op": "eq", "value": "placeholder"}]})json"},
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [{"name": "syscall", "op": "eq", "value": "execve"}]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"2 fields, 3 rules eq, match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "eq", "value": "execve"},
|
||||
{"name": "user", "op": "eq", "value": "bob"}
|
||||
]})json"},
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "eq", "value": "execve"},
|
||||
{"name": "doesnotexist", "op": "eq", "value": "placeholder"}
|
||||
]})json"},
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "eq", "value": "execve"},
|
||||
{"name": "user", "op": "eq", "value": "test_user"}
|
||||
]})json"},
|
||||
}}
|
||||
},
|
||||
{
|
||||
"2 fields, 3 rules eq, match",
|
||||
{{
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "eq", "value": "execve"},
|
||||
{"name": "user", "op": "!re", "value":"^test_.*$"}
|
||||
]})json"},
|
||||
{false, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "in", "values": ["execve","execveat"]},
|
||||
{"name": "doesnotexist", "op": "!re", "value":"placeholder"}
|
||||
]})json"},
|
||||
{true, R"json({"record_types": ["AUOMS_EXECVE"], "field_rules": [
|
||||
{"name": "syscall", "op": "eq", "value": "execve"},
|
||||
{"name": "user", "op": "eq", "value":"test_user"}
|
||||
]})json"},
|
||||
}}
|
||||
},
|
||||
};
|
||||
|
||||
for (auto& test : tests) {
|
||||
if (!test.RunTest(allocator->GetEvent())) {
|
||||
BOOST_FAIL(test.Error());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -75,7 +75,7 @@ BOOST_AUTO_TEST_CASE( test )
|
|||
if (!ret) {
|
||||
BOOST_FAIL("EndRecord failed: " + std::to_string(ret));
|
||||
}
|
||||
builder.SetEventFlags(5);
|
||||
builder.AddEventFlags(5);
|
||||
builder.SetEventPid(12);
|
||||
ret = builder.EndEvent();
|
||||
if (ret != 1) {
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ void WriteFile(const std::string& path, const std::vector<std::string>& lines) {
|
|||
}
|
||||
|
||||
void AppendFile(const std::string& path, const std::vector<std::string>& lines) {
|
||||
std::ofstream out(path, std::ios::binary);
|
||||
std::ofstream out(path, std::ios::binary | std::ios::ate);
|
||||
if (!out.is_open()) {
|
||||
throw std::runtime_error("Failed to open '" + path + "'");
|
||||
}
|
||||
|
|
|
|||
337
Output.cpp
337
Output.cpp
|
|
@ -22,6 +22,9 @@
|
|||
#include "FluentEventWriter.h"
|
||||
#include "RawEventWriter.h"
|
||||
#include "SyslogEventWriter.h"
|
||||
#include "FileUtils.h"
|
||||
|
||||
#include <functional>
|
||||
|
||||
extern "C" {
|
||||
#include <unistd.h>
|
||||
|
|
@ -30,153 +33,66 @@ extern "C" {
|
|||
#include <fcntl.h>
|
||||
}
|
||||
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
|
||||
AckQueue::AckQueue(size_t max_size): _max_size(max_size), _closed(false), _have_auto_cursor(false), _next_seq(0), _auto_cursor_seq(0) {}
|
||||
|
||||
void AckQueue::Init(const std::shared_ptr<PriorityQueue>& queue, const std::shared_ptr<QueueCursorHandle>& cursor_handle) {
|
||||
_queue = queue;
|
||||
_cursor_handle = cursor_handle;
|
||||
_closed = false;
|
||||
void AckReader::Init(std::shared_ptr<IEventWriter> event_writer,
|
||||
std::shared_ptr<IOBase> writer) {
|
||||
_event_writer = event_writer;
|
||||
_writer = writer;
|
||||
_event_ids.clear();
|
||||
_cursors.clear();
|
||||
_auto_cursors.clear();
|
||||
_next_seq = 0;
|
||||
_have_auto_cursor = false;
|
||||
_auto_cursor_seq = 0;
|
||||
}
|
||||
|
||||
void AckQueue::Close() {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
_closed = true;
|
||||
_cond.notify_all();
|
||||
void AckReader::AddPendingAck(const EventId& id) {
|
||||
std::lock_guard<std::mutex> _lock(_mutex);
|
||||
|
||||
auto it = _event_ids.find(id);
|
||||
if (it == _event_ids.end()) {
|
||||
_event_ids.emplace(id, false);
|
||||
}
|
||||
}
|
||||
|
||||
bool AckQueue::IsClosed() {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
return _closed;
|
||||
void AckReader::RemoveAck(const EventId& id) {
|
||||
std::lock_guard<std::mutex> _lock(_mutex);
|
||||
|
||||
_event_ids.erase(id);
|
||||
}
|
||||
|
||||
bool AckQueue::Add(const EventId& event_id, uint32_t priority, uint64_t seq, long timeout) {
|
||||
bool AckReader::WaitForAck(const EventId& id, long timeout) {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
|
||||
if (_cond.wait_for(_lock, std::chrono::milliseconds(timeout), [this]() { return _closed || _event_ids.size() < _max_size; })) {
|
||||
auto qseq = _next_seq++;
|
||||
_event_ids.emplace(event_id, qseq);
|
||||
_cursors.emplace(qseq, _CursorEntry(event_id, priority, seq));
|
||||
return true;
|
||||
if (_event_ids.count(id) == 0) {
|
||||
_event_ids.emplace(id, false);
|
||||
}
|
||||
|
||||
if (!_cond.wait_for(_lock, std::chrono::milliseconds(timeout), [this, &id]{ return _event_ids[id]; })) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void AckQueue::SetAutoCursor(uint32_t priority, uint64_t seq) {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
_event_ids.erase(id);
|
||||
|
||||
_auto_cursor_seq = _next_seq++;
|
||||
_auto_cursors[priority] = seq;
|
||||
_have_auto_cursor = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
void AckQueue::ProcessAutoCursor() {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
void AckReader::handle_ack(const EventId& id) {
|
||||
std::lock_guard<std::mutex> _lock(_mutex);
|
||||
|
||||
if (_have_auto_cursor) {
|
||||
for (auto& c : _auto_cursors) {
|
||||
_queue->Commit(_cursor_handle, c.first, c.second);
|
||||
auto it = _event_ids.find(id);
|
||||
if (it != _event_ids.end()) {
|
||||
it->second = true;
|
||||
_cond.notify_all();
|
||||
}
|
||||
_auto_cursors.clear();
|
||||
_have_auto_cursor = false;
|
||||
}
|
||||
}
|
||||
|
||||
void AckQueue::Remove(const EventId& event_id) {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
|
||||
auto eitr = _event_ids.find(event_id);
|
||||
if (eitr == _event_ids.end()) {
|
||||
return;
|
||||
}
|
||||
auto seq = eitr->second;
|
||||
_event_ids.erase(eitr);
|
||||
|
||||
_cursors.erase(seq);
|
||||
}
|
||||
|
||||
bool AckQueue::Wait(int millis) {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
|
||||
auto now = std::chrono::steady_clock::now();
|
||||
return _cond.wait_until(_lock, now + std::chrono::milliseconds(millis), [this] { return _event_ids.empty(); });
|
||||
}
|
||||
|
||||
void AckQueue::Ack(const EventId& event_id) {
|
||||
std::unique_lock<std::mutex> _lock(_mutex);
|
||||
|
||||
std::unordered_map<uint32_t, uint64_t> found_seq;
|
||||
|
||||
auto eitr = _event_ids.find(event_id);
|
||||
if (eitr != _event_ids.end()) {
|
||||
auto seq = eitr->second;
|
||||
_event_ids.erase(eitr);
|
||||
_cond.notify_all(); // _event_ids was modified, so notify any waiting Add calls
|
||||
|
||||
// Find and remove all from cursors that are <= seq
|
||||
while (!_cursors.empty() && _cursors.begin()->first <= seq) {
|
||||
auto& entry = _cursors.begin()->second;
|
||||
// Make sure to remove any associated event ids from _event_ids.
|
||||
_event_ids.erase(entry._event_id);
|
||||
auto itr = found_seq.find(entry._priority);
|
||||
if (itr == found_seq.end() || itr->second < entry._seq) {
|
||||
found_seq[entry._priority] = entry._seq;
|
||||
}
|
||||
_cursors.erase(_cursors.begin());
|
||||
}
|
||||
}
|
||||
|
||||
if (_have_auto_cursor) {
|
||||
if (_cursors.empty() || _cursors.begin()->first > _auto_cursor_seq) {
|
||||
for (auto& c : _auto_cursors) {
|
||||
auto itr = found_seq.find(c.first);
|
||||
if (itr == found_seq.end() || itr->second < c.second) {
|
||||
found_seq[c.first] = c.second;
|
||||
}
|
||||
}
|
||||
_auto_cursors.clear();
|
||||
_have_auto_cursor = false;
|
||||
}
|
||||
}
|
||||
|
||||
for (auto& c : found_seq) {
|
||||
_queue->Commit(_cursor_handle, c.first, c.second);
|
||||
}
|
||||
}
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
void AckReader::Init(std::shared_ptr<IEventWriter> event_writer,
|
||||
std::shared_ptr<IOBase> writer,
|
||||
std::shared_ptr<AckQueue> ack_queue) {
|
||||
_event_writer = event_writer;
|
||||
_writer = writer;
|
||||
_queue = ack_queue;
|
||||
}
|
||||
|
||||
void AckReader::run() {
|
||||
EventId id;
|
||||
while(_event_writer->ReadAck(id, _writer.get()) == IO::OK) {
|
||||
_queue->Ack(id);
|
||||
handle_ack(id);
|
||||
}
|
||||
|
||||
// The connection is lost, Close writer here so that Output::handle_events will exit
|
||||
_writer->Close();
|
||||
|
||||
_queue->ProcessAutoCursor();
|
||||
|
||||
// Make sure any waiting AckQueue::Add() returns immediately instead of waiting for the timeout.
|
||||
_queue->Close();
|
||||
}
|
||||
|
||||
/****************************************************************************
|
||||
|
|
@ -239,6 +155,10 @@ bool Output::Load(std::unique_ptr<Config>& config) {
|
|||
_event_filter.reset();
|
||||
}
|
||||
|
||||
if (config->HasKey("aggregation_rules")) {
|
||||
AggregationRule::RulesFromJSON(config->GetJSON("aggregation_rules"), _aggregation_rules);
|
||||
}
|
||||
|
||||
if (socket_path != _socket_path || !_writer) {
|
||||
_socket_path = socket_path;
|
||||
_writer = std::unique_ptr<UnixDomainWriter>(new UnixDomainWriter(_socket_path));
|
||||
|
|
@ -259,20 +179,6 @@ bool Output::Load(std::unique_ptr<Config>& config) {
|
|||
}
|
||||
|
||||
if (_ack_mode) {
|
||||
uint64_t ack_queue_size = DEFAULT_ACK_QUEUE_SIZE;
|
||||
if (_config->HasKey("ack_queue_size")) {
|
||||
try {
|
||||
ack_queue_size = _config->GetUint64("ack_queue_size");
|
||||
} catch (std::exception) {
|
||||
Logger::Error("Output(%s): Invalid ack_queue_size parameter value", _name.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (ack_queue_size < 1) {
|
||||
Logger::Error("Output(%s): Invalid ack_queue_size parameter value", _name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (_config->HasKey("ack_timeout")) {
|
||||
try {
|
||||
_ack_timeout = _config->GetInt64("ack_timeout");
|
||||
|
|
@ -285,17 +191,9 @@ bool Output::Load(std::unique_ptr<Config>& config) {
|
|||
Logger::Warn("Output(%s): ack_timeout parameter value to small (%ld), using (%ld)", _name.c_str(), _ack_timeout, MIN_ACK_TIMEOUT);
|
||||
_ack_timeout = MIN_ACK_TIMEOUT;
|
||||
}
|
||||
}
|
||||
|
||||
if (!_ack_queue || _ack_queue->MaxSize() != ack_queue_size) {
|
||||
_ack_queue = std::make_shared<AckQueue>(ack_queue_size);
|
||||
}
|
||||
} else {
|
||||
if (_ack_queue) {
|
||||
_ack_queue.reset();
|
||||
}
|
||||
}
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
// Delete any resources associated with the output
|
||||
|
|
@ -338,65 +236,92 @@ bool Output::check_open()
|
|||
return false;
|
||||
}
|
||||
|
||||
ssize_t Output::send_event(const Event& event) {
|
||||
EventId id(event.Seconds(), event.Milliseconds(), event.Serial());
|
||||
if (_ack_mode) {
|
||||
_ack_reader->AddPendingAck(id);
|
||||
}
|
||||
auto ret = _event_writer->WriteEvent(event, _writer.get());
|
||||
switch (ret) {
|
||||
case IEventWriter::NOOP:
|
||||
_ack_reader->RemoveAck(id);
|
||||
return IWriter::OK;
|
||||
case IWriter::OK:
|
||||
if (_ack_mode) {
|
||||
if (!_ack_reader->WaitForAck(id, _ack_timeout)) {
|
||||
Logger::Warn("Output(%s): Timeout waiting for ack", _name.c_str());
|
||||
return IO::TIMEOUT;
|
||||
}
|
||||
}
|
||||
return IWriter::OK;
|
||||
default:
|
||||
_ack_reader->RemoveAck(id);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
// Return true of the write succeeded
|
||||
bool Output::handle_queue_event(const Event& event, uint32_t priority, uint64_t sequence) {
|
||||
auto ret = send_event(event);
|
||||
if (ret != IWriter::OK) {
|
||||
return false;
|
||||
}
|
||||
|
||||
_queue->Commit(_cursor_handle, priority, sequence);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Return <err,false> of the write failed
|
||||
std::pair<int64_t, bool> Output::handle_agg_event(const Event& event) {
|
||||
auto ret = send_event(event);
|
||||
return std::make_pair(static_cast<int64_t>(ret), ret == IWriter::OK);
|
||||
}
|
||||
|
||||
bool Output::handle_events(bool checkOpen) {
|
||||
_queue->Rollback(_cursor_handle);
|
||||
|
||||
if (_ack_mode) {
|
||||
_ack_queue->Init(_queue, _cursor_handle);
|
||||
_ack_reader->Init(_event_writer, _writer, _ack_queue);
|
||||
_ack_reader->Init(_event_writer, _writer);
|
||||
_ack_reader->Start();
|
||||
}
|
||||
|
||||
while(!IsStopping() && (!checkOpen || _writer->IsOpen())) {
|
||||
std::pair<std::shared_ptr<QueueItem>,bool> get_ret;
|
||||
if (_event_aggregator) {
|
||||
std::tuple<bool, int64_t, bool> agg_ret;
|
||||
do {
|
||||
get_ret = _queue->Get(_cursor_handle, 100, !_ack_mode);
|
||||
} while((!get_ret.first && !get_ret.second) && (!checkOpen || _writer->IsOpen()));
|
||||
agg_ret = _event_aggregator->HandleEvent([this, checkOpen](const Event& event) -> std::pair<int64_t, bool> {
|
||||
if (IsStopping() || !(checkOpen && _writer->IsOpen())) {
|
||||
return std::make_pair(0, false);
|
||||
}
|
||||
return handle_agg_event(event);
|
||||
});
|
||||
} while (std::get<0>(agg_ret));
|
||||
if (std::get<0>(agg_ret) && !std::get<2>(agg_ret)) {
|
||||
// The write failed, so assume the connection is bad
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (get_ret.first && (!checkOpen || _writer->IsOpen()) && !IsStopping()) {
|
||||
std::pair<std::shared_ptr<QueueItem>,bool> get_ret;
|
||||
get_ret = _queue->Get(_cursor_handle, 100, !_ack_mode);
|
||||
|
||||
if(get_ret.first) {
|
||||
Event event(get_ret.first->Data(), get_ret.first->Size());
|
||||
bool filtered = _event_filter && _event_filter->IsEventFiltered(event);
|
||||
if (!filtered) {
|
||||
if (_ack_mode) {
|
||||
// Avoid racing with receiver, add ack before sending event
|
||||
if (!_ack_queue->Add(EventId(event.Seconds(), event.Milliseconds(), event.Serial()),
|
||||
get_ret.first->Priority(), get_ret.first->Sequence(),
|
||||
_ack_timeout)) {
|
||||
if (!_ack_queue->IsClosed()) {
|
||||
Logger::Error("Output(%s): Timeout waiting for Acks", _name.c_str());
|
||||
if (_event_aggregator) {
|
||||
if (_event_aggregator->AddEvent(event)) {
|
||||
// The event was consumed
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (!handle_queue_event(event, get_ret.first->Priority(), get_ret.first->Sequence())) {
|
||||
// The write failed, so assume the connection is bad
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto ret = _event_writer->WriteEvent(event, _writer.get());
|
||||
if (ret == IEventWriter::NOOP) {
|
||||
if (_ack_mode) {
|
||||
// The event was not sent, so remove it's ack
|
||||
_ack_queue->Remove(EventId(event.Seconds(), event.Milliseconds(), event.Serial()));
|
||||
// And update the auto cursor
|
||||
_ack_queue->SetAutoCursor(get_ret.first->Priority(), get_ret.first->Sequence());
|
||||
}
|
||||
} else if (ret != IWriter::OK) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (!_ack_mode) {
|
||||
_queue->Commit(_cursor_handle, get_ret.first->Priority(), get_ret.first->Sequence());
|
||||
}
|
||||
} else {
|
||||
if (_ack_mode) {
|
||||
_ack_queue->SetAutoCursor(get_ret.first->Priority(), get_ret.first->Sequence());
|
||||
} else {
|
||||
_queue->Commit(_cursor_handle, get_ret.first->Priority(), get_ret.first->Sequence());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (_ack_mode) {
|
||||
// Wait a short time for final acks to arrive
|
||||
_ack_queue->Wait(100);
|
||||
}
|
||||
|
||||
// writer must be closed before calling _ack_reader->Stop(), or the stop may hang until the connection is closed remotely.
|
||||
|
|
@ -419,24 +344,68 @@ void Output::on_stopping() {
|
|||
if (_writer) {
|
||||
_writer->CloseWrite();
|
||||
}
|
||||
if (_ack_queue) {
|
||||
_ack_queue->Close();
|
||||
}
|
||||
}
|
||||
|
||||
void Output::on_stop() {
|
||||
if (_ack_reader) {
|
||||
_ack_reader->Stop();
|
||||
}
|
||||
|
||||
if (_writer) {
|
||||
_writer->Close();
|
||||
}
|
||||
|
||||
if (_event_aggregator) {
|
||||
if (!_save_file.empty()) {
|
||||
try {
|
||||
_event_aggregator->Save(_save_file);
|
||||
} catch (const std::exception& ex) {
|
||||
Logger::Error("Output(%s): Failed to save event aggregation state to '%s': %s", _name.c_str(), _save_file.c_str(), ex.what());
|
||||
}
|
||||
} else {
|
||||
Logger::Error("Output(%s): Failed to save event aggregation state: No save file defined", _name.c_str());
|
||||
}
|
||||
_event_aggregator.reset();
|
||||
}
|
||||
|
||||
Logger::Info("Output(%s): Stopped", _name.c_str());
|
||||
}
|
||||
|
||||
void Output::run() {
|
||||
Logger::Info("Output(%s): Started", _name.c_str());
|
||||
|
||||
if (_aggregation_rules.size() > 0) {
|
||||
_event_aggregator = std::make_shared<EventAggregator>();
|
||||
if (!_save_file.empty() && PathExists(_save_file)) {
|
||||
if (!IsOnlyRootWritable(_save_file)) {
|
||||
Logger::Error("Output(%s): Event aggregation state file is non-root writable '%s': It will ignored and removed", _name.c_str(), _save_file.c_str());
|
||||
_event_aggregator = std::make_shared<EventAggregator>();
|
||||
} else {
|
||||
try {
|
||||
_event_aggregator->Load(_save_file);
|
||||
} catch (const std::exception& ex) {
|
||||
Logger::Error("Output(%s): Failed to load event aggregation state from '%s': %s", _name.c_str(), _save_file.c_str(), ex.what());
|
||||
_event_aggregator = std::make_shared<EventAggregator>();
|
||||
}
|
||||
}
|
||||
if (unlink(_save_file.c_str()) != 0) {
|
||||
Logger::Error("Output(%s): Failed to remove aggregation state file '%s': %s", _name.c_str(), _save_file.c_str(), std::strerror(errno));
|
||||
}
|
||||
}
|
||||
try {
|
||||
_event_aggregator->SetRules(_aggregation_rules);
|
||||
} catch (const std::exception& ex) {
|
||||
Logger::Error("Output(%s): Failed to set event aggregation rules: %s", _name.c_str(), ex.what());
|
||||
_event_aggregator.reset();
|
||||
}
|
||||
} else {
|
||||
if (!_save_file.empty() && PathExists(_save_file)) {
|
||||
if (unlink(_save_file.c_str()) != 0) {
|
||||
Logger::Error("Output(%s): Failed to remove aggregation state file '%s': %s", _name.c_str(), _save_file.c_str(), std::strerror(errno));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_cursor_handle = _queue->OpenCursor(_name);
|
||||
if (!_cursor_handle) {
|
||||
Logger::Error("Output(%s): Aborting because cursor is invalid", _name.c_str());
|
||||
|
|
|
|||
86
Output.h
86
Output.h
|
|
@ -24,68 +24,13 @@
|
|||
#include "OMSEventWriter.h"
|
||||
#include "IO.h"
|
||||
#include "IEventFilter.h"
|
||||
#include "EventAggregator.h"
|
||||
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
|
||||
class AckQueue {
|
||||
public:
|
||||
AckQueue(size_t max_size);
|
||||
|
||||
size_t MaxSize() {
|
||||
return _max_size;
|
||||
}
|
||||
|
||||
void Init(const std::shared_ptr<PriorityQueue>& queue, const std::shared_ptr<QueueCursorHandle>& cursor_handle);
|
||||
|
||||
void Close();
|
||||
|
||||
bool IsClosed();
|
||||
|
||||
// Return false if timeout, true if added
|
||||
bool Add(const EventId& event_id, uint32_t priority, uint64_t seq, long timeout);
|
||||
|
||||
// Set (or update) auto cursor
|
||||
void SetAutoCursor(uint32_t priority, uint64_t seq);
|
||||
|
||||
// Get and clear auto cursor
|
||||
void ProcessAutoCursor();
|
||||
|
||||
void Remove(const EventId& event_id);
|
||||
|
||||
// Returns false on timeout, true is queue is empty
|
||||
bool Wait(int millis);
|
||||
|
||||
void Ack(const EventId& event_id);
|
||||
|
||||
private:
|
||||
class _CursorEntry {
|
||||
public:
|
||||
_CursorEntry(EventId event_id, uint32_t priority, uint64_t seq): _event_id(event_id), _priority(priority), _seq(seq) {}
|
||||
EventId _event_id;
|
||||
uint32_t _priority;
|
||||
uint64_t _seq;
|
||||
};
|
||||
std::mutex _mutex;
|
||||
std::condition_variable _cond;
|
||||
std::unordered_map<EventId, uint64_t> _event_ids;
|
||||
std::map<uint64_t, _CursorEntry> _cursors;
|
||||
size_t _max_size;
|
||||
std::shared_ptr<PriorityQueue> _queue;
|
||||
std::shared_ptr<QueueCursorHandle> _cursor_handle;
|
||||
bool _closed;
|
||||
bool _have_auto_cursor;
|
||||
uint64_t _next_seq;
|
||||
uint64_t _auto_cursor_seq;
|
||||
std::unordered_map<uint32_t, uint64_t> _auto_cursors;
|
||||
};
|
||||
|
||||
/****************************************************************************
|
||||
*
|
||||
****************************************************************************/
|
||||
|
|
@ -99,16 +44,22 @@ public:
|
|||
{}
|
||||
|
||||
void Init(std::shared_ptr<IEventWriter> event_writer,
|
||||
std::shared_ptr<IOBase> writer,
|
||||
std::shared_ptr<AckQueue> ack_queue);
|
||||
std::shared_ptr<IOBase> writer);
|
||||
|
||||
void AddPendingAck(const EventId& id);
|
||||
void RemoveAck(const EventId& id);
|
||||
bool WaitForAck(const EventId& id, long timeout);
|
||||
|
||||
protected:
|
||||
void handle_ack(const EventId& id);
|
||||
virtual void run();
|
||||
|
||||
std::mutex _mutex;
|
||||
std::condition_variable _cond;
|
||||
std::string _name;
|
||||
std::shared_ptr<IEventWriter> _event_writer;
|
||||
std::shared_ptr<IOBase> _writer;
|
||||
std::shared_ptr<AckQueue> _queue;
|
||||
std::unordered_map<EventId, bool> _event_ids;
|
||||
};
|
||||
|
||||
/****************************************************************************
|
||||
|
|
@ -147,11 +98,13 @@ public:
|
|||
static constexpr int MAX_SLEEP_PERIOD = 60;
|
||||
static constexpr int DEFAULT_ACK_QUEUE_SIZE = 1000;
|
||||
static constexpr long MIN_ACK_TIMEOUT = 100;
|
||||
static constexpr long DEFAULT_ACK_TIMEOUT = 300*1000; // 5 minutes
|
||||
|
||||
Output(const std::string& name, const std::shared_ptr<PriorityQueue>& queue, const std::shared_ptr<IEventWriterFactory>& writer_factory, const std::shared_ptr<IEventFilterFactory>& filter_factory):
|
||||
_name(name), _queue(queue), _writer_factory(writer_factory), _filter_factory(filter_factory), _ack_mode(false), _ack_timeout(10000)
|
||||
Output(const std::string& name, const std::string& save_dir, const std::shared_ptr<PriorityQueue>& queue, const std::shared_ptr<IEventWriterFactory>& writer_factory, const std::shared_ptr<IEventFilterFactory>& filter_factory):
|
||||
_name(name), _save_dir(save_dir), _queue(queue), _writer_factory(writer_factory), _filter_factory(filter_factory), _ack_mode(false), _ack_timeout(DEFAULT_ACK_TIMEOUT)
|
||||
{
|
||||
_ack_reader = std::unique_ptr<AckReader>(new AckReader(name));
|
||||
_save_file = _save_dir + "/" + name + ".aggsavefile";
|
||||
}
|
||||
|
||||
bool IsConfigDifferent(const Config& config);
|
||||
|
|
@ -172,23 +125,30 @@ protected:
|
|||
// Return true on success, false if Output should stop.
|
||||
bool check_open();
|
||||
|
||||
ssize_t send_event(const Event& event);
|
||||
bool handle_queue_event(const Event& event, uint32_t priority, uint64_t sequence);
|
||||
std::pair<int64_t, bool> handle_agg_event(const Event& event);
|
||||
|
||||
// Return true if writer closed and Output should reconnect, false if Output should stop.
|
||||
bool handle_events(bool checkOpen=true);
|
||||
|
||||
std::mutex _mutex;
|
||||
std::string _name;
|
||||
std::string _save_dir;
|
||||
std::string _save_file;
|
||||
std::string _socket_path;
|
||||
std::shared_ptr<PriorityQueue> _queue;
|
||||
std::shared_ptr<IEventWriterFactory> _writer_factory;
|
||||
std::shared_ptr<IEventFilterFactory> _filter_factory;
|
||||
bool _ack_mode;
|
||||
long _ack_timeout;
|
||||
uint64_t _ack_timeout;
|
||||
std::unique_ptr<Config> _config;
|
||||
std::shared_ptr<QueueCursorHandle> _cursor_handle;
|
||||
std::shared_ptr<IEventWriter> _event_writer;
|
||||
std::shared_ptr<IEventFilter> _event_filter;
|
||||
std::shared_ptr<IOBase> _writer;
|
||||
std::shared_ptr<AckQueue> _ack_queue;
|
||||
std::vector<std::shared_ptr<AggregationRule>> _aggregation_rules;
|
||||
std::shared_ptr<EventAggregator> _event_aggregator;
|
||||
std::unique_ptr<AckReader> _ack_reader;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ bool BuildEvent(std::shared_ptr<EventBuilder>& builder, uint64_t sec, uint32_t m
|
|||
builder->CancelEvent();
|
||||
return false;
|
||||
}
|
||||
if (!builder->AddField("seq", std::to_string(seq), nullptr, field_type_t::UNCLASSIFIED)) {
|
||||
if (!builder->AddField("seq", std::to_string(seq), std::string_view(), field_type_t::UNCLASSIFIED)) {
|
||||
builder->CancelEvent();
|
||||
return false;
|
||||
}
|
||||
|
|
@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE( basic_test ) {
|
|||
{"ack_timeout", "1000"}
|
||||
}));
|
||||
auto writer_factory = std::shared_ptr<IEventWriterFactory>(static_cast<IEventWriterFactory*>(new RawOnlyEventWriterFactory()));
|
||||
Output output("output", queue, writer_factory, nullptr);
|
||||
Output output("output", "", queue, writer_factory, nullptr);
|
||||
output.Load(output_config);
|
||||
|
||||
auto operational_status = std::make_shared<OperationalStatus>("", nullptr);
|
||||
|
|
@ -185,7 +185,7 @@ BOOST_AUTO_TEST_CASE( same_event_id_test ) {
|
|||
{"ack_timeout", "1000"}
|
||||
}));
|
||||
auto writer_factory = std::shared_ptr<IEventWriterFactory>(static_cast<IEventWriterFactory*>(new RawOnlyEventWriterFactory()));
|
||||
Output output("output", queue, writer_factory, nullptr);
|
||||
Output output("output", "", queue, writer_factory, nullptr);
|
||||
output.Load(output_config);
|
||||
|
||||
auto operational_status = std::make_shared<OperationalStatus>("", nullptr);
|
||||
|
|
@ -281,11 +281,10 @@ BOOST_AUTO_TEST_CASE( dropped_acks_test ) {
|
|||
{"output_format","raw"},
|
||||
{"output_socket", socket_path},
|
||||
{"enable_ack_mode", "true"},
|
||||
{"ack_queue_size", "10"},
|
||||
{"ack_timeout", "100"}
|
||||
}));
|
||||
auto writer_factory = std::shared_ptr<IEventWriterFactory>(static_cast<IEventWriterFactory*>(new RawOnlyEventWriterFactory()));
|
||||
Output output("output", queue, writer_factory, nullptr);
|
||||
Output output("output", "", queue, writer_factory, nullptr);
|
||||
output.Load(output_config);
|
||||
|
||||
Gate done_gate;
|
||||
|
|
@ -308,7 +307,6 @@ BOOST_AUTO_TEST_CASE( dropped_acks_test ) {
|
|||
|
||||
bool stop = false;
|
||||
bool drop = true;
|
||||
int num_received = 0;
|
||||
std::array<uint8_t, 1024> data;
|
||||
RawEventReader reader;
|
||||
|
||||
|
|
@ -323,17 +321,19 @@ BOOST_AUTO_TEST_CASE( dropped_acks_test ) {
|
|||
break;
|
||||
}
|
||||
Event event(data.data(), ret);
|
||||
Logger::Info("Input: Recevied %ld", event.Serial());
|
||||
if (event.Serial() == end_serial) {
|
||||
Logger::Info("Input: Recevied End");
|
||||
reader.WriteAck(event, &io);
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
if (!drop) {
|
||||
Logger::Info("Input: Sending Ack");
|
||||
reader.WriteAck(event, &io);
|
||||
_outputs.emplace_back(reinterpret_cast<char *>(data.data()), ret);
|
||||
}
|
||||
drop = !drop;
|
||||
_outputs.emplace_back(reinterpret_cast<char *>(data.data()), ret);
|
||||
num_received += 1;
|
||||
}
|
||||
}
|
||||
done_gate.Open();
|
||||
|
|
@ -359,7 +359,7 @@ BOOST_AUTO_TEST_CASE( dropped_acks_test ) {
|
|||
}
|
||||
}
|
||||
|
||||
if (!done_gate.Wait(Gate::OPEN, 10000)) {
|
||||
if (!done_gate.Wait(Gate::OPEN, 15000)) {
|
||||
BOOST_FAIL("Time out waiting for inputs");
|
||||
}
|
||||
|
||||
|
|
@ -367,12 +367,14 @@ BOOST_AUTO_TEST_CASE( dropped_acks_test ) {
|
|||
queue->Close();
|
||||
input_thread.join();
|
||||
|
||||
int timeout_count = 0;
|
||||
for (auto& msg : log_lines) {
|
||||
if (starts_with(msg, "Output(output): Timeout waiting for Acks")) {
|
||||
BOOST_FAIL("Found 'Timeout waiting for Acks' in log output");
|
||||
if (starts_with(msg, "Output(output): Timeout waiting for ack")) {
|
||||
timeout_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
BOOST_REQUIRE_EQUAL(num_events, timeout_count);
|
||||
BOOST_REQUIRE_EQUAL(num_events, _outputs.size());
|
||||
|
||||
for (int i = 0; i < num_events; i++) {
|
||||
|
|
@ -410,7 +412,7 @@ BOOST_AUTO_TEST_CASE( dropped_conn_test ) {
|
|||
{"ack_timeout", "1000"}
|
||||
}));
|
||||
auto writer_factory = std::shared_ptr<IEventWriterFactory>(static_cast<IEventWriterFactory*>(new RawOnlyEventWriterFactory()));
|
||||
Output output("output", queue, writer_factory, nullptr);
|
||||
Output output("output", "", queue, writer_factory, nullptr);
|
||||
output.Load(output_config);
|
||||
|
||||
Gate done_gate;
|
||||
|
|
|
|||
|
|
@ -221,7 +221,7 @@ void Outputs::do_conf_sync() {
|
|||
load = true;
|
||||
}
|
||||
} else {
|
||||
auto o = std::make_shared<Output>(ent.first, _queue, _writer_factory, _filter_factory);
|
||||
auto o = std::make_shared<Output>(ent.first, _save_dir, _queue, _writer_factory, _filter_factory);
|
||||
it = _outputs.insert(std::make_pair(ent.first, o)).first;
|
||||
load = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,14 +50,14 @@ private:
|
|||
|
||||
class Outputs: public RunBase {
|
||||
public:
|
||||
Outputs(std::shared_ptr<PriorityQueue>& queue, const std::string& conf_dir, std::shared_ptr<UserDB>& user_db, std::shared_ptr<FiltersEngine> filtersEngine, std::shared_ptr<ProcessTree> processTree):
|
||||
_queue(queue), _conf_dir(conf_dir), _do_reload(false) {
|
||||
Outputs(std::shared_ptr<PriorityQueue>& queue, const std::string& conf_dir, const std::string& save_dir, std::shared_ptr<UserDB>& user_db, std::shared_ptr<FiltersEngine> filtersEngine, std::shared_ptr<ProcessTree> processTree):
|
||||
_queue(queue), _conf_dir(conf_dir), _save_dir(save_dir), _do_reload(false) {
|
||||
_writer_factory = std::shared_ptr<IEventWriterFactory>(static_cast<IEventWriterFactory*>(new OutputsEventWriterFactory()));
|
||||
_filter_factory = std::shared_ptr<IEventFilterFactory>(static_cast<IEventFilterFactory*>(new OutputsEventFilterFactory(user_db, filtersEngine, processTree)));
|
||||
}
|
||||
|
||||
Outputs(std::shared_ptr<PriorityQueue>& queue, const std::string& conf_dir, const std::shared_ptr<IEventFilterFactory>& filter_factory):
|
||||
_queue(queue), _conf_dir(conf_dir),
|
||||
Outputs(std::shared_ptr<PriorityQueue>& queue, const std::string& conf_dir, const std::string& save_dir, const std::shared_ptr<IEventFilterFactory>& filter_factory):
|
||||
_queue(queue), _conf_dir(conf_dir), _save_dir(save_dir),
|
||||
_writer_factory(std::shared_ptr<IEventWriterFactory>(static_cast<IEventWriterFactory*>(new OutputsEventWriterFactory()))),
|
||||
_filter_factory(filter_factory),
|
||||
_do_reload(false) {
|
||||
|
|
@ -76,6 +76,7 @@ private:
|
|||
|
||||
std::shared_ptr<PriorityQueue> _queue;
|
||||
std::string _conf_dir;
|
||||
std::string _save_dir;
|
||||
std::shared_ptr<IEventWriterFactory> _writer_factory;
|
||||
std::shared_ptr<IEventFilterFactory> _filter_factory;
|
||||
bool _do_reload;
|
||||
|
|
|
|||
183
ProcessInfo.cpp
183
ProcessInfo.cpp
|
|
@ -19,6 +19,7 @@
|
|||
#include "StringUtils.h"
|
||||
#include "ExecveConverter.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
|
|
@ -43,9 +44,9 @@ time_t boot_time() {
|
|||
return ts.tv_sec - sinfo.uptime;
|
||||
}
|
||||
|
||||
bool read_file(const std::string& path, std::vector<uint8_t>& data, size_t limit, bool& truncated) {
|
||||
bool read_file(const char* path, std::vector<uint8_t>& data, size_t limit, bool& truncated) {
|
||||
errno = 0;
|
||||
int fd = ::open(path.c_str(), O_RDONLY|O_CLOEXEC);
|
||||
int fd = ::open(path, O_RDONLY|O_CLOEXEC);
|
||||
if (fd < 0) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -72,11 +73,11 @@ bool read_file(const std::string& path, std::vector<uint8_t>& data, size_t limit
|
|||
}
|
||||
|
||||
// Return 1 on success, 0 if there is no exe (the case for kernel processes), or -1 on error.
|
||||
int read_link(const std::string& path, std::string& data) {
|
||||
int read_link(const char* path, std::string& data) {
|
||||
char buff[PATH_MAX];
|
||||
data.clear();
|
||||
errno = 0;
|
||||
ssize_t len = ::readlink(path.c_str(), buff, sizeof(buff)-1);
|
||||
ssize_t len = ::readlink(path, buff, sizeof(buff)-1);
|
||||
if (len < 0) {
|
||||
// For kernel processes errno will be ENOENT
|
||||
if (errno == ENOENT) {
|
||||
|
|
@ -88,13 +89,33 @@ int read_link(const std::string& path, std::string& data) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
bool ProcessInfo::parse_stat() {
|
||||
if (_stat.empty()) {
|
||||
return false;
|
||||
int ProcessInfo::read_and_parse_stat(int pid) {
|
||||
std::array<char, 64> path;
|
||||
std::array<char, 2048> data;
|
||||
|
||||
snprintf(path.data(), path.size(), "/proc/%d/stat", pid);
|
||||
|
||||
int fd = ::open(path.data(), O_RDONLY|O_CLOEXEC);
|
||||
if (fd < 0) {
|
||||
if (errno != ENOENT && errno != ESRCH) {
|
||||
Logger::Warn("Failed to open /proc/%d/stat: %s", pid, strerror(errno));
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
char *ptr = reinterpret_cast<char*>(_stat.data());
|
||||
char *end = reinterpret_cast<char*>(ptr+_stat.size());
|
||||
auto nr = ::read(fd, data.data(), data.size());
|
||||
if (nr <= 0) {
|
||||
close(fd);
|
||||
// Only generate a log message if the error was something other than ENOENT (No such file or directory) or ESRCH (No such process)
|
||||
if (nr < 0 && errno != ENOENT && errno != ESRCH) {
|
||||
Logger::Warn("Failed to read /proc/%d/stat: %s", pid, strerror(errno));
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
close(fd);
|
||||
|
||||
char *ptr = reinterpret_cast<char*>(data.data());
|
||||
char *end = reinterpret_cast<char*>(ptr+nr);
|
||||
|
||||
// pid
|
||||
char *f_end = ptr;
|
||||
|
|
@ -102,27 +123,33 @@ bool ProcessInfo::parse_stat() {
|
|||
errno = 0;
|
||||
_pid = static_cast<int>(strtol(ptr, &f_end, 10));
|
||||
if (errno != 0 || *f_end != ' ') {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ptr = f_end+1;
|
||||
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// comm
|
||||
if (*ptr != '(') {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
f_end = strstr(ptr, ") ");
|
||||
if (f_end != nullptr && f_end+1 < end && f_end[1] == ')') {
|
||||
f_end += 1;
|
||||
}
|
||||
if (f_end == nullptr || f_end >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
_comm.assign(ptr+1, f_end-ptr-1);
|
||||
|
||||
_comm_size = f_end-ptr-1;
|
||||
if (_comm_size >= _comm.max_size()) {
|
||||
_comm_size = _comm.max_size()-1;
|
||||
}
|
||||
std::copy_n(ptr+1, _comm_size, _comm.data());
|
||||
_comm[_comm_size] = 0;
|
||||
|
||||
ptr = f_end+1;
|
||||
if (ptr < end && *ptr == ' ') {
|
||||
|
|
@ -130,64 +157,64 @@ bool ProcessInfo::parse_stat() {
|
|||
}
|
||||
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Skip state
|
||||
f_end = strchr(ptr, ' ');
|
||||
if (f_end == nullptr || f_end >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
ptr = f_end+1;
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// ppid
|
||||
errno = 0;
|
||||
_ppid = static_cast<int>(strtol(ptr, &f_end, 10));
|
||||
if (errno != 0 || *f_end != ' ') {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ptr = f_end+1;
|
||||
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Skip pgrp
|
||||
f_end = strchr(ptr, ' ');
|
||||
if (f_end == nullptr || f_end >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
ptr = f_end+1;
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// sid
|
||||
errno = 0;
|
||||
_ses = static_cast<int>(strtol(ptr, &f_end, 10));
|
||||
if (errno != 0 || *f_end != ' ') {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ptr = f_end+1;
|
||||
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Skip to utime
|
||||
for (int i = 0; i < 7; ++i) {
|
||||
f_end = strchr(ptr, ' ');
|
||||
if (f_end == nullptr || f_end >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
ptr = f_end + 1;
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -195,35 +222,35 @@ bool ProcessInfo::parse_stat() {
|
|||
errno = 0;
|
||||
_utime = strtoull(ptr, &f_end, 10);
|
||||
if (errno != 0 || *f_end != ' ') {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ptr = f_end+1;
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// stime
|
||||
errno = 0;
|
||||
_stime = strtoull(ptr, &f_end, 10);
|
||||
if (errno != 0 || *f_end != ' ') {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ptr = f_end+1;
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Skip to starttime
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
f_end = strchr(ptr, ' ');
|
||||
if (f_end == nullptr || f_end >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
ptr = f_end + 1;
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -231,51 +258,71 @@ bool ProcessInfo::parse_stat() {
|
|||
errno = 0;
|
||||
_starttime = strtoull(ptr, &f_end, 10);
|
||||
if (errno != 0 || *f_end != ' ') {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ptr = f_end+1;
|
||||
|
||||
if (ptr >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
return true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool ProcessInfo::parse_status() {
|
||||
if (_status.empty()) {
|
||||
return false;
|
||||
int ProcessInfo::read_and_parse_status(int pid) {
|
||||
std::array<char, 64> path;
|
||||
std::array<char, 8192> data;
|
||||
|
||||
snprintf(path.data(), path.size(), "/proc/%d/status", pid);
|
||||
|
||||
int fd = ::open(path.data(), O_RDONLY|O_CLOEXEC);
|
||||
if (fd < 0) {
|
||||
if (errno != ENOENT && errno != ESRCH) {
|
||||
Logger::Warn("Failed to open /proc/%d/status: %s", pid, strerror(errno));
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
char *ptr = reinterpret_cast<char*>(_status.data());
|
||||
char *end = reinterpret_cast<char*>(ptr+_status.size());
|
||||
auto nr = ::read(fd, data.data(), data.size());
|
||||
if (nr <= 0) {
|
||||
close(fd);
|
||||
// Only generate a log message if the error was something other than ENOENT (No such file or directory) or ESRCH (No such process)
|
||||
if (nr < 0 && errno != ENOENT && errno != ESRCH) {
|
||||
Logger::Warn("Failed to read /proc/%d/status: %s", pid, strerror(errno));
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
close(fd);
|
||||
|
||||
char *ptr = reinterpret_cast<char*>(data.data());
|
||||
char *end = reinterpret_cast<char*>(ptr+nr);
|
||||
|
||||
char *uid_line = strstr(ptr, "Uid:");
|
||||
if (uid_line == nullptr || uid_line >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
char *uid_line_end = strchr(uid_line, '\n');
|
||||
if (uid_line_end == nullptr || uid_line_end >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
char *gid_line = strstr(uid_line_end, "Gid:");
|
||||
if (gid_line == nullptr || gid_line >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
char *gid_line_end = strchr(gid_line, '\n');
|
||||
if (gid_line_end == nullptr || gid_line_end >= end) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ptr = uid_line;
|
||||
ptr += 4; // Skip "Uid:"
|
||||
ptr = strchr(ptr, '\t');
|
||||
if (ptr == nullptr) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int uids[4];
|
||||
|
|
@ -283,7 +330,7 @@ bool ProcessInfo::parse_status() {
|
|||
errno = 0;
|
||||
uids[i] = static_cast<int>(strtol(ptr, &end, 10));
|
||||
if (errno != 0) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
while(*ptr == '\t') {
|
||||
++ptr;
|
||||
|
|
@ -294,7 +341,7 @@ bool ProcessInfo::parse_status() {
|
|||
ptr += 4; // Skip "Gid:"
|
||||
ptr = strchr(ptr, '\t');
|
||||
if (ptr == nullptr) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int gids[4];
|
||||
|
|
@ -302,7 +349,7 @@ bool ProcessInfo::parse_status() {
|
|||
errno = 0;
|
||||
gids[i] = static_cast<int>(strtol(ptr, &end, 10));
|
||||
if (errno != 0) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
while(*ptr == '\t') {
|
||||
++ptr;
|
||||
|
|
@ -319,30 +366,31 @@ bool ProcessInfo::parse_status() {
|
|||
_sgid = gids[2];
|
||||
_fsgid = gids[3];
|
||||
|
||||
return true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool ProcessInfo::read(int pid) {
|
||||
const std::string path = "/proc/" + std::to_string(pid);
|
||||
bool truncated;
|
||||
std::array<char, 64> path;
|
||||
|
||||
if (!read_file(path+"/stat", _stat, 2048, truncated)) {
|
||||
// Only generate a log message if the error was something other than ENOENT (No such file or directory) or ESRCH (No such process)
|
||||
if (errno != ENOENT && errno != ESRCH) {
|
||||
Logger::Warn("Failed to read /proc/%d/stat: %s", pid, strerror(errno));
|
||||
snprintf(path.data(), path.size(), "/proc/%d/exe", pid);
|
||||
|
||||
int pret = read_and_parse_stat(pid);
|
||||
if (pret != 0) {
|
||||
if (pret > 0) {
|
||||
Logger::Warn("Failed to parse /proc/%d/stat", pid);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!read_file(path+"/status", _status, 8192, truncated)) {
|
||||
// Only generate a log message if the error was something other than ENOENT (No such file or directory) or ESRCH (No such process)
|
||||
if (errno != ENOENT && errno != ESRCH) {
|
||||
Logger::Warn("Failed to read /proc/%d/status: %s", pid, strerror(errno));
|
||||
pret = read_and_parse_status(pid);
|
||||
if (pret != 0) {
|
||||
if (pret > 0) {
|
||||
Logger::Warn("Failed to parse /proc/%d/status", pid);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto exe_status = read_link(path+"/exe", _exe);
|
||||
auto exe_status = read_link(path.data(), _exe);
|
||||
if (exe_status < 0) {
|
||||
// EACCES (Permission denied) will be seen occasionally (probably due to racy nature of /proc iteration)
|
||||
// ONly emit error if it wasn't EACCES or ESRCH
|
||||
|
|
@ -354,9 +402,10 @@ bool ProcessInfo::read(int pid) {
|
|||
|
||||
// Only try to read the cmdline file if there was an exe link.
|
||||
// Kernel processes will not have anything in the cmdline file.
|
||||
if (exe_status == 1) {
|
||||
if (exe_status == 1 && _cmdline_size_limit > 0) {
|
||||
// The Event field value size limit is UINT16_MAX (including NULL terminator)
|
||||
if (!read_file(path + "/cmdline", _cmdline, _cmdline_size_limit, _cmdline_truncated)) {
|
||||
snprintf(path.data(), path.size(), "/proc/%d/cmdline", pid);
|
||||
if (!read_file(path.data(), _cmdline, _cmdline_size_limit, _cmdline_truncated)) {
|
||||
// Only generate a log message if the error was something other than ENOENT (No such file or directory) or ESRCH (No such process)
|
||||
if (errno != ENOENT && errno != ESRCH) {
|
||||
Logger::Warn("Failed to read /proc/%d/cmdline: %s", pid, strerror(errno));
|
||||
|
|
@ -366,16 +415,6 @@ bool ProcessInfo::read(int pid) {
|
|||
}
|
||||
|
||||
|
||||
if (!parse_stat()) {
|
||||
Logger::Warn("Failed to parse /proc/%d/stat", pid);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!parse_status()) {
|
||||
Logger::Warn("Failed to parse /proc/%d/status", pid);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -428,7 +467,7 @@ void ProcessInfo::clear() {
|
|||
_sgid = -1;
|
||||
_fsgid = -1;
|
||||
_exe.clear();
|
||||
_comm.clear();
|
||||
_comm.fill(0);
|
||||
_cmdline.clear();
|
||||
_cmdline_truncated = false;
|
||||
_starttime_str.clear();
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ public:
|
|||
inline int sgid() { return _sgid; }
|
||||
inline int fsgid() { return _fsgid; }
|
||||
|
||||
inline std::string comm() { return _comm; }
|
||||
inline std::string_view comm() { return std::string_view(_comm.data(), _comm_size); }
|
||||
inline std::string exe() { return _exe; }
|
||||
|
||||
inline uint64_t utime() { return _utime; }
|
||||
|
|
@ -59,8 +59,8 @@ public:
|
|||
private:
|
||||
explicit ProcessInfo(void* dp, int cmdline_size_limit);
|
||||
|
||||
bool parse_stat();
|
||||
bool parse_status();
|
||||
int read_and_parse_stat(int pid);
|
||||
int read_and_parse_status(int pid);
|
||||
|
||||
bool read(int pid);
|
||||
void clear();
|
||||
|
|
@ -83,11 +83,9 @@ private:
|
|||
int _sgid;
|
||||
int _fsuid;
|
||||
int _fsgid;
|
||||
std::string _comm;
|
||||
size_t _comm_size;
|
||||
std::array<char, 16> _comm;
|
||||
std::string _exe;
|
||||
std::vector<uint8_t> _stat;
|
||||
std::vector<uint8_t> _status;
|
||||
std::vector<uint8_t> _statm;
|
||||
std::vector<uint8_t> _cmdline;
|
||||
std::string _starttime_str;
|
||||
bool _cmdline_truncated;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,11 @@
|
|||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/socket.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <linux/netlink.h>
|
||||
#include <linux/connector.h>
|
||||
#include <linux/cn_proc.h>
|
||||
#include <linux/filter.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#ifndef SOL_NETLINIK
|
||||
|
|
@ -52,7 +56,7 @@ bool ProcessNotify::InitProcSocket()
|
|||
|
||||
Logger::Info("ProcessNotify initialising");
|
||||
|
||||
_proc_socket = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR);
|
||||
_proc_socket = socket(PF_NETLINK, SOCK_DGRAM | SOCK_CLOEXEC, NETLINK_CONNECTOR);
|
||||
if (_proc_socket < 0) {
|
||||
Logger::Error("Cannot create netlink socket for proc monitoring: %s", std::strerror(errno));
|
||||
return false;
|
||||
|
|
@ -68,10 +72,68 @@ bool ProcessNotify::InitProcSocket()
|
|||
return false;
|
||||
}
|
||||
|
||||
// Add BPF filter to exclude proc connector messages related to threads
|
||||
struct sock_filter filter[] = {
|
||||
// If netlink message is multi-part, just pass the message
|
||||
BPF_STMT (BPF_LD|BPF_H|BPF_ABS, offsetof (struct nlmsghdr, nlmsg_type)),
|
||||
BPF_JUMP (BPF_JMP|BPF_JEQ|BPF_K, htons (NLMSG_DONE), 1, 0),
|
||||
// Pass the whole message
|
||||
BPF_STMT (BPF_RET|BPF_K, 0xffffffff),
|
||||
|
||||
// If the netlink message doesn't contain a proc connector message, just pass the message
|
||||
BPF_STMT (BPF_LD|BPF_W|BPF_ABS,
|
||||
NLMSG_LENGTH (0) + offsetof (struct cn_msg, id)
|
||||
+ offsetof (struct cb_id, idx)),
|
||||
BPF_JUMP (BPF_JMP|BPF_JEQ|BPF_K,
|
||||
htonl (CN_IDX_PROC),
|
||||
1, 0),
|
||||
BPF_STMT (BPF_RET|BPF_K, 0xffffffff),
|
||||
|
||||
|
||||
BPF_STMT (BPF_LD|BPF_W|BPF_ABS,
|
||||
NLMSG_LENGTH (0) + offsetof (struct cn_msg, id)
|
||||
+ offsetof (struct cb_id, idx)),
|
||||
BPF_JUMP (BPF_JMP|BPF_JEQ|BPF_K,
|
||||
htonl (CN_VAL_PROC),
|
||||
1, 0),
|
||||
BPF_STMT (BPF_RET|BPF_K, 0xffffffff),
|
||||
|
||||
// If the proc event is a PROC_EVENT_EXIT, pass it
|
||||
BPF_STMT (BPF_LD|BPF_W|BPF_ABS,
|
||||
NLMSG_LENGTH (0) + offsetof (struct cn_msg, data)
|
||||
+ offsetof (struct proc_event, what)),
|
||||
BPF_JUMP (BPF_JMP|BPF_JEQ|BPF_K,
|
||||
htonl (proc_event::what::PROC_EVENT_EXIT),
|
||||
0, 1),
|
||||
BPF_STMT (BPF_RET|BPF_K, 0xffffffff),
|
||||
|
||||
// If the proc event is a PROC_EVENT_EXEC, pass it
|
||||
BPF_STMT (BPF_LD|BPF_W|BPF_ABS,
|
||||
NLMSG_LENGTH (0) + offsetof (struct cn_msg, data)
|
||||
+ offsetof (struct proc_event, what)),
|
||||
BPF_JUMP (BPF_JMP|BPF_JEQ|BPF_K,
|
||||
htonl (proc_event::what::PROC_EVENT_EXEC),
|
||||
0, 1),
|
||||
BPF_STMT (BPF_RET|BPF_K, 0xffffffff),
|
||||
|
||||
// Drop what's left
|
||||
BPF_STMT (BPF_RET|BPF_K, 0),
|
||||
};
|
||||
|
||||
struct sock_fprog fprog;
|
||||
fprog.filter = filter;
|
||||
fprog.len = sizeof filter / sizeof filter[0];
|
||||
|
||||
if (setsockopt (_proc_socket, SOL_SOCKET, SO_ATTACH_FILTER, &fprog, sizeof fprog) != 0) {
|
||||
Logger::Error("Failed set BPF filter socket for proc monitoring: %s", std::strerror(errno));
|
||||
close(_proc_socket);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Prevent ENOBUFS when messages generated faster then can be received.
|
||||
int on = 1;
|
||||
if (setsockopt(_proc_socket, SOL_NETLINK, NETLINK_NO_ENOBUFS, &on, sizeof(on)) != 0) {
|
||||
Logger::Error("Cannot set NETLINK_NO_ENOBUFS option on socket for proc monitoring: %s", std::strerror(errno));
|
||||
Logger::Error("Failed set NETLINK_NO_ENOBUFS option on socket for proc monitoring: %s", std::strerror(errno));
|
||||
close(_proc_socket);
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,12 +31,95 @@
|
|||
#include <iostream>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/socket.h>
|
||||
#include <linux/netlink.h>
|
||||
#include <linux/connector.h>
|
||||
#include <linux/cn_proc.h>
|
||||
#include <bits/stdc++.h>
|
||||
|
||||
/*
|
||||
|
||||
Init
|
||||
Read app procs
|
||||
Apply filter to each proc
|
||||
Propogate filter results to child processes
|
||||
|
||||
Add Pid
|
||||
Read proc into
|
||||
Apply filter to proc
|
||||
Propogate filter results from parent
|
||||
|
||||
ProcInfo
|
||||
Trigger (execve, pnotify_fork, pnotify_exec)
|
||||
Origin (audit, procfs)
|
||||
pid
|
||||
ppid
|
||||
starttime
|
||||
parent_starttime
|
||||
containerid
|
||||
filterflags
|
||||
|
||||
ProcTree
|
||||
std::unordered_map<pid_t, std::shared_ptr<ProcInfo>> proc_tree
|
||||
|
||||
AddForkPid(pid_t pid, pid_t ppid)
|
||||
Lock(new_pids_lock)
|
||||
Add pid to new_pids
|
||||
Notify condition
|
||||
|
||||
AddExecve(pid_t pid, pid_t ppid, exe, cmdline)
|
||||
Lock(mutex)
|
||||
|
||||
addForkPid(pid, ppid)
|
||||
pp = getProc(ppid, 0)
|
||||
p = getProcFromProcFS(pid)
|
||||
p->parent = pp
|
||||
if p->parent
|
||||
copy containerid and filter state from parent
|
||||
if p->notfiltered
|
||||
filterProc(p)
|
||||
|
||||
addExecPid(pid)
|
||||
p = getProc(pid, 0)
|
||||
|
||||
addProc(uid, gid, pid, ppid, exe, cmdline)
|
||||
pi = proc_tree.find(pid)
|
||||
if pi == null
|
||||
pi = new ProcInfo
|
||||
pi->Set(uid, gif, pid, ppid, exe, cmdline)
|
||||
|
||||
getProcFromProcFS(pid)
|
||||
p = new ProcInfo
|
||||
read to p
|
||||
filterProc(p)
|
||||
return p
|
||||
|
||||
getProc(pid, nestcount)
|
||||
if nestcount > 10
|
||||
return nullptr
|
||||
nestcount++;
|
||||
ProcInfo *p
|
||||
pi = proc_tree.find(pid)
|
||||
if pi == null
|
||||
p = getProcFromProcFS(pid)
|
||||
proc_tree[pid] = p
|
||||
else
|
||||
p = pi->second
|
||||
if !(p->parent)
|
||||
p->parent = getProc(pid, nestcount)
|
||||
if p->parent
|
||||
copy containerid and filter state from parent
|
||||
if p->notfiltered
|
||||
filterProc(p)
|
||||
return p
|
||||
|
||||
pnotify_run()
|
||||
read from sock
|
||||
AddForkPid(pid, ppid)
|
||||
AddExecPid(pid)
|
||||
ExitPid(pid)
|
||||
|
||||
run()
|
||||
|
||||
|
||||
*/
|
||||
|
||||
enum ProcessTreeSource { ProcessTreeSource_execve, ProcessTreeSource_pnotify, ProcessTreeSource_procfs };
|
||||
|
||||
class FiltersEngine;
|
||||
|
|
|
|||
|
|
@ -740,7 +740,7 @@ bool RawEventProcessor::process_syscall_event(const Event& event) {
|
|||
|
||||
void RawEventProcessor::end_event()
|
||||
{
|
||||
_builder->SetEventFlags(_event_flags);
|
||||
_builder->AddEventFlags(_event_flags);
|
||||
_event_flags = 0;
|
||||
if (!_builder->EndEvent()) {
|
||||
throw std::runtime_error("Queue closed");
|
||||
|
|
@ -938,7 +938,7 @@ bool RawEventProcessor::generate_proc_event(ProcessInfo* pinfo, uint64_t sec, ui
|
|||
throw std::runtime_error("Queue closed");
|
||||
}
|
||||
|
||||
_builder->SetEventFlags(EVENT_FLAG_IS_AUOMS_EVENT);
|
||||
_builder->AddEventFlags(EVENT_FLAG_IS_AUOMS_EVENT);
|
||||
|
||||
uint16_t num_fields = 17;
|
||||
|
||||
|
|
|
|||
|
|
@ -256,6 +256,7 @@ enum class RecordType: int {
|
|||
AUOMS_DROPPED_RECORDS = 10004,
|
||||
AUOMS_STATUS = 10005,
|
||||
AUOMS_METRIC = 10006,
|
||||
AUOMS_AGGREGATE = 10007,
|
||||
AUOMS_EXECVE = 14688,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@
|
|||
#define FTRACE_SYS_ENTER_TRIGGER "/sys/kernel/debug/tracing/instances/auoms/events/raw_syscalls/sys_enter/trigger"
|
||||
#define FTRACE_SYS_ENTER_HIST "/sys/kernel/debug/tracing/instances/auoms/events/raw_syscalls/sys_enter/hist"
|
||||
#define SYSCALL_HIST_TRIGGER "hist:key=id.syscall:val=hitcount"
|
||||
#define SYSCALL_HIST_TRIGGER_CLEAR "hist:key=id.syscall:val=hitcount:clear"
|
||||
|
||||
// Lines like: { id: sys_recvmsg [ 47] } hitcount: 27076
|
||||
const std::string SyscallMetrics::_hist_line_match_re = R"REGEX(^\{\s*id:\s*(\S+)\s*\[\s*([0-9]+)\s*\]\s*\}\s*hitcount:\s*([0-9]+))REGEX";
|
||||
|
|
@ -137,7 +138,7 @@ bool SyscallMetrics::collect_metrics() {
|
|||
|
||||
// Reset hist
|
||||
try {
|
||||
WriteFile(FTRACE_SYS_ENTER_TRIGGER, {{SYSCALL_HIST_TRIGGER}});
|
||||
AppendFile(FTRACE_SYS_ENTER_TRIGGER, {{SYSCALL_HIST_TRIGGER_CLEAR}});
|
||||
} catch (std::exception &ex) {
|
||||
Logger::Warn("SyscallMetrics: Failed to write sys_enter trigger (%s): %s", FTRACE_SYS_ENTER_TRIGGER, ex.what());
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ struct TestEvent {
|
|||
|
||||
void Write(const std::shared_ptr<EventBuilder>& builder) {
|
||||
builder->BeginEvent(_seconds, _milliseconds, _serial, _records.size());
|
||||
builder->SetEventFlags(_flags);
|
||||
builder->AddEventFlags(_flags);
|
||||
builder->SetEventPid(_pid);
|
||||
for (auto rec : _records) {
|
||||
rec.Write(builder);
|
||||
|
|
|
|||
|
|
@ -223,6 +223,7 @@ static StringTable<RecordType> s_record_type_table(RecordType ::UNKNOWN, {
|
|||
{"AUOMS_DROPPED_RECORDS", RecordType::AUOMS_DROPPED_RECORDS},
|
||||
{"AUOMS_STATUS", RecordType::AUOMS_STATUS},
|
||||
{"AUOMS_METRIC", RecordType::AUOMS_METRIC},
|
||||
{"AUOMS_AGGREGATE", RecordType::AUOMS_AGGREGATE},
|
||||
{"AUOMS_EXECVE", RecordType::AUOMS_EXECVE},
|
||||
});
|
||||
|
||||
|
|
|
|||
16
auoms.cpp
16
auoms.cpp
|
|
@ -48,6 +48,7 @@
|
|||
#include <unistd.h>
|
||||
#include <syslog.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "env_config.h"
|
||||
#include "LockFile.h"
|
||||
|
|
@ -197,6 +198,12 @@ int main(int argc, char**argv) {
|
|||
status_socket_path = config.GetString("status_socket_path");
|
||||
}
|
||||
|
||||
std::string save_dir = data_dir + "/save";
|
||||
|
||||
if (config.HasKey("save_dir")) {
|
||||
save_dir = config.GetString("save_dir");
|
||||
}
|
||||
|
||||
int num_priorities = 8;
|
||||
size_t max_file_data_size = 1024*1024;
|
||||
size_t max_unsaved_files = 128;
|
||||
|
|
@ -321,6 +328,13 @@ int main(int argc, char**argv) {
|
|||
exit(1);
|
||||
}
|
||||
|
||||
if (!PathExists(save_dir)) {
|
||||
if (mkdir(save_dir.c_str(), 0750) != 0) {
|
||||
Logger::Error("Failed to create dir '%s': %s", save_dir.c_str(), std::strerror(errno));
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
Logger::Info("Trying to acquire singleton lock");
|
||||
LockFile singleton_lock(lock_file);
|
||||
switch(singleton_lock.Lock()) {
|
||||
|
|
@ -463,7 +477,7 @@ int main(int argc, char**argv) {
|
|||
outputsFilterFactory = std::shared_ptr<IEventFilterFactory>(static_cast<IEventFilterFactory*>(new OutputsEventFilterFactory(user_db, filtersEngine, processTree)));
|
||||
}
|
||||
|
||||
Outputs outputs(queue, outconf_dir, outputsFilterFactory);
|
||||
Outputs outputs(queue, outconf_dir, save_dir, outputsFilterFactory);
|
||||
|
||||
std::thread autosave_thread([&]() {
|
||||
Signals::InitThread();
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@
|
|||
#--------------------------------------------------------------------------------
|
||||
|
||||
AUOMS_BUILDVERSION_MAJOR=2
|
||||
AUOMS_BUILDVERSION_MINOR=6
|
||||
AUOMS_BUILDVERSION_PATCH=1
|
||||
AUOMS_BUILDVERSION_MINOR=7
|
||||
AUOMS_BUILDVERSION_PATCH=0
|
||||
AUOMS_BUILDVERSION_BUILDNR=0
|
||||
|
|
|
|||
|
|
@ -606,7 +606,7 @@ int main(int argc, char**argv) {
|
|||
{"ack_queue_size", "100"}
|
||||
}));
|
||||
auto writer_factory = std::shared_ptr<IEventWriterFactory>(static_cast<IEventWriterFactory*>(new RawOnlyEventWriterFactory()));
|
||||
Output output("output", queue, writer_factory, nullptr);
|
||||
Output output("output", "", queue, writer_factory, nullptr);
|
||||
output.Load(output_config);
|
||||
|
||||
std::thread autosave_thread([&]() {
|
||||
|
|
|
|||
|
|
@ -132,13 +132,9 @@ if [ -n "$DepsDir" ]; then
|
|||
fi
|
||||
|
||||
if [ -z "$SourceDir" ]; then
|
||||
pushd $(dirname $0)/..
|
||||
SourceDir=$(pwd)
|
||||
popd
|
||||
SourceDir=$(cd $(dirname $0)/.. && pwd)
|
||||
else
|
||||
pushd $SourceDir
|
||||
SourceDir=$(pwd)
|
||||
popd
|
||||
SourceDir=$(cd $SourceDir && pwd)
|
||||
fi
|
||||
|
||||
ToolchainFile=""
|
||||
|
|
@ -203,20 +199,24 @@ if [ $? -ne 0 ]; then
|
|||
fi
|
||||
popd
|
||||
|
||||
if [ ! -d $DestDir/bin ]; then
|
||||
mkdir $DestDir/bin
|
||||
if [ $? -ne 0 ]; then
|
||||
Bail "Failed to create '$DestDir/bin'"
|
||||
fi
|
||||
fi
|
||||
|
||||
cp $BuildDir/release/bin/* $DestDir/bin
|
||||
if [ $? -ne 0 ]; then
|
||||
Bail "Failed to copy binaries to dest dir"
|
||||
fi
|
||||
|
||||
if [ ! -d $DestDir/tests ]; then
|
||||
mkdir $DestDir/tests
|
||||
if [ $? -ne 0 ]; then
|
||||
Bail "Failed to create '$DestDir/tests'"
|
||||
fi
|
||||
fi
|
||||
|
||||
cp $BuildDir/*Tests $DestDir/tests
|
||||
if [ $? -ne 0 ]; then
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ ShowUsage()
|
|||
cat << EOF
|
||||
Error: $1
|
||||
|
||||
Usage: $0 -s <archive dir> -d <dest include dir> -l <dest lib dir> [ -t <toolset> ]
|
||||
Usage: $0 -s <archive dir> -l <dest include dir> -l <dest lib dir> [ -t <toolset> ]
|
||||
|
||||
-s <archive dir> - The dir where source archives reside
|
||||
-i <dest include dir> - Where to place include files
|
||||
|
|
|
|||
|
|
@ -133,17 +133,25 @@ mkdir -p $IntermediateDir
|
|||
|
||||
BUILD_NUMBER=${BuildNumber:-$AUOMS_BUILDVERSION_BUILDNR}
|
||||
|
||||
cp $BinDir/bin/* $TargetDir/bin
|
||||
#cp $BinDir/bin/* $TargetDir/bin
|
||||
|
||||
pushd $DestDir
|
||||
|
||||
PSEUDO_OPTS=""
|
||||
if [ $(id -u) -ne 0 ]; then
|
||||
mkdir -p $PseudoDir
|
||||
PSEUDO_OPTS="PSEUDO_PREFIX=/opt/pseudo PSEUDO_LOCALSTATEDIR=$PseudoDir pseudo"
|
||||
fi
|
||||
|
||||
$PSEUDO_OPTS python $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
PseudoPython()
|
||||
{
|
||||
if [ $(id -u) -ne 0 ]; then
|
||||
PSEUDO_PREFIX=/usr PSEUDO_LOCALSTATEDIR=$PseudoDir pseudo python $*
|
||||
else
|
||||
python $*
|
||||
fi
|
||||
}
|
||||
|
||||
PseudoPython $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
--BASE_DIR=$SourceDir \
|
||||
--TARGET_DIR=$DestDir \
|
||||
--INTERMEDIATE_DIR=$IntermediateDir \
|
||||
|
|
@ -160,7 +168,7 @@ if [ $(id -u) -ne 0 ]; then
|
|||
fi
|
||||
rm -rf $StageDir/*
|
||||
|
||||
$PSEUDO_OPTS python $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
PseudoPython $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
--BASE_DIR=$SourceDir \
|
||||
--TARGET_DIR=$DestDir \
|
||||
--INTERMEDIATE_DIR=$IntermediateDir \
|
||||
|
|
@ -178,7 +186,7 @@ fi
|
|||
rm -rf $StageDir/*
|
||||
|
||||
if [ "$BuildType" == "RelWithDebInfo" ]; then
|
||||
$PSEUDO_OPTS python $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
PseudoPython $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
--BASE_DIR=$SourceDir \
|
||||
--TARGET_DIR=$DestDir \
|
||||
--INTERMEDIATE_DIR=$IntermediateDir \
|
||||
|
|
@ -195,7 +203,7 @@ if [ "$BuildType" == "RelWithDebInfo" ]; then
|
|||
fi
|
||||
rm -rf $StageDir/*
|
||||
|
||||
$PSEUDO_OPTS python $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
PseudoPython $SourceDir/installer/InstallBuilder/installbuilder.py \
|
||||
--BASE_DIR=$SourceDir \
|
||||
--TARGET_DIR=$DestDir \
|
||||
--INTERMEDIATE_DIR=$IntermediateDir \
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче