microsoft
/
ebpf-for-windows
зеркало из https://github.com/microsoft/ebpf-for-windows.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Implement BPF_MAP_TYPE_LPM_TRIE over lock-free hashmap. (#572) 

* Implement BPF_MAP_TYPE_LPM_TRIE over lock-free hashmap.

Signed-off-by: Alan Jowett <alanjo@microsoft.com>
This commit is contained in:
Alan Jowett 2021-09-28 14:27:50 -06:00 коммит произвёл GitHub
Родитель 65783fb64d
Коммит 082d938501
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
14 изменённых файлов: 535 добавлений и 93 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

144
libs/execution_context/ebpf_maps.c
Просмотреть файл

@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation
// SPDX-License-Identifier: MIT
#include "ebpf_bitmap.h"
#include "ebpf_epoch.h"
#include "ebpf_handle.h"
#include "ebpf_maps.h"
@ -35,6 +36,14 @@ typedef struct _ebpf_core_lru_map
ebpf_hash_table_t* key_history;
} ebpf_core_lru_map_t;
typedef struct _ebpf_core_lpm_map
{
ebpf_core_map_t core_map;
uint32_t max_prefix;
// Bitmap of prefix lengths inserted into the map.
uint8_t data[1];
} ebpf_core_lpm_map_t;
_Ret_notnull_ static const ebpf_program_type_t*
_get_map_program_type(_In_ const ebpf_object_t* object)
{
@ -412,8 +421,11 @@ _get_object_from_array_map_entry(_In_ ebpf_core_map_t* map, _In_ const uint8_t*
}
static ebpf_core_map_t*
_create_hash_map_with_map_struct_size(
size_t map_struct_size, _In_ const ebpf_map_definition_in_memory_t* map_definition)
_create_hash_map_internal(
size_t map_struct_size,
_In_ const ebpf_map_definition_in_memory_t* map_definition,
_In_opt_ void (*extract_function)(
_In_ const uint8_t* value, _Outptr_ const uint8_t** data, _Out_ size_t* length_in_bits))
{
ebpf_result_t retval;
ebpf_core_map_t* map = NULL;
@ -437,7 +449,7 @@ _create_hash_map_with_map_struct_size(
map->ebpf_map_definition.key_size,
map->ebpf_map_definition.value_size,
map->ebpf_map_definition.max_entries,
NULL);
extract_function);
if (retval != EBPF_SUCCESS) {
goto Done;
}
@ -458,20 +470,20 @@ Done:
static ebpf_core_map_t*
_create_hash_map(_In_ const ebpf_map_definition_in_memory_t* map_definition)
{
return _create_hash_map_with_map_struct_size(sizeof(ebpf_core_map_t), map_definition);
return _create_hash_map_internal(sizeof(ebpf_core_map_t), map_definition, NULL);
}
static ebpf_core_map_t*
_create_object_hash_map(_In_ const ebpf_map_definition_in_memory_t* map_definition)
{
return _create_hash_map_with_map_struct_size(sizeof(ebpf_core_object_map_t), map_definition);
return _create_hash_map_internal(sizeof(ebpf_core_object_map_t), map_definition, NULL);
}
static ebpf_core_map_t*
_create_lru_hash_map(_In_ const ebpf_map_definition_in_memory_t* map_definition)
{
ebpf_core_lru_map_t* map =
(ebpf_core_lru_map_t*)_create_hash_map_with_map_struct_size(sizeof(ebpf_core_lru_map_t), map_definition);
(ebpf_core_lru_map_t*)_create_hash_map_internal(sizeof(ebpf_core_lru_map_t), map_definition, NULL);
if (map) {
ebpf_result_t retval;
// Note:
@ -866,82 +878,66 @@ _update_entry_per_cpu(
return EBPF_SUCCESS;
}
// Given two keys, compute the number of bits that match.
static size_t
_lpm_trie_matching_key_prefix_length(
_In_reads_((key_length_a + 7) / 8) const uint8_t* key_a,
size_t key_length_a,
_In_reads_((key_length_b + 7) / 8) const uint8_t* key_b,
size_t key_length_b)
static void
_lpm_extract(_In_ const uint8_t* value, _Outptr_ const uint8_t** data, _Out_ size_t* length_in_bits)
{
size_t maximum_matching_length = key_length_a > key_length_b ? key_length_b : key_length_a;
size_t remaining_key_length_in_bits = maximum_matching_length;
uint8_t key_a_last_byte;
uint8_t key_b_last_byte;
size_t index;
uint32_t prefix_length = *(uint32_t*)value;
*data = value;
*length_in_bits = sizeof(uint32_t) * 8 + prefix_length;
}
while (remaining_key_length_in_bits >= 8) {
index = (maximum_matching_length - remaining_key_length_in_bits) / 8;
if (key_a[index] != key_b[index]) {
break;
}
remaining_key_length_in_bits -= 8;
static ebpf_core_map_t*
_create_lpm_map(_In_ const ebpf_map_definition_in_memory_t* map_definition)
{
size_t max_prefix_length = (map_definition->key_size - sizeof(uint32_t)) * 8 + 1;
ebpf_core_lpm_map_t* map = (ebpf_core_lpm_map_t*)_create_hash_map_internal(
EBPF_OFFSET_OF(ebpf_core_lpm_map_t, data) + ebpf_bitmap_size(max_prefix_length), map_definition, _lpm_extract);
if (!map) {
return NULL;
}
if (remaining_key_length_in_bits == 0) {
return maximum_matching_length;
}
index = (maximum_matching_length - remaining_key_length_in_bits) / 8;
key_a_last_byte = key_a[index];
key_b_last_byte = key_b[index];
while (remaining_key_length_in_bits) {
if ((key_a_last_byte & 0x80) != (key_b_last_byte & 0x80)) {
break;
}
remaining_key_length_in_bits--;
key_a_last_byte <<= 1;
key_b_last_byte <<= 1;
}
return maximum_matching_length - remaining_key_length_in_bits;
map->max_prefix = (uint32_t)max_prefix_length;
ebpf_bitmap_initialize((ebpf_bitmap_t*)map->data, max_prefix_length);
return &(map->core_map);
}
static uint8_t*
_find_lpm_trie_map_entry(_In_ ebpf_core_map_t* map, _In_ const uint8_t* key)
_find_lpm_map_entry(_In_ ebpf_core_map_t* map, _In_ const uint8_t* key)
{
uint8_t* best_value = NULL;
uint32_t search_key_length = *(uint32_t*)key;
const uint8_t* search_key = key + sizeof(uint32_t);
uint8_t* current_key = NULL;
uint8_t* current_value = NULL;
size_t longest_key_match = 0;
ebpf_result_t result;
uint32_t* prefix_length = (uint32_t*)key;
uint32_t original_prefix_length = *prefix_length;
uint8_t* value = NULL;
ebpf_core_lpm_map_t* trie_map = EBPF_FROM_FIELD(ebpf_core_lpm_map_t, core_map, map);
if (!map || !key)
return NULL;
do {
result = ebpf_hash_table_next_key_pointer_and_value(
(ebpf_hash_table_t*)map->data, current_key, &current_key, &current_value);
if (result == EBPF_SUCCESS) {
uint32_t compare_key_length = *(uint32_t*)current_key;
uint8_t* compare_key = current_key + sizeof(uint32_t);
size_t compare_key_match =
_lpm_trie_matching_key_prefix_length(compare_key, compare_key_length, search_key, search_key_length);
// All bits in the compare_key must match
if (compare_key_match != compare_key_length) {
continue;
}
// Find the longest comparison that matches.
if (compare_key_match > longest_key_match) {
longest_key_match = compare_key_match;
best_value = current_value;
}
ebpf_bitmap_cursor_t cursor;
ebpf_bitmap_start_reverse_search((ebpf_bitmap_t*)trie_map->data, &cursor);
while (*prefix_length != MAXUINT32) {
*prefix_length = (uint32_t)ebpf_bitmap_reverse_search_next_bit(&cursor);
value = _find_hash_map_entry(map, key);
if (value) {
break;
}
} while (result == EBPF_SUCCESS);
return best_value;
}
*prefix_length = original_prefix_length;
return value;
}
static ebpf_result_t
_update_lpm_map_entry(
_In_ ebpf_core_map_t* map, _In_ const uint8_t* key, _In_opt_ const uint8_t* data, ebpf_map_option_t option)
{
ebpf_core_lpm_map_t* trie_map = EBPF_FROM_FIELD(ebpf_core_lpm_map_t, core_map, map);
uint32_t prefix_length = *(uint32_t*)key;
if (prefix_length > trie_map->max_prefix) {
return EBPF_INVALID_ARGUMENT;
}
ebpf_result_t result = _update_hash_map_entry(map, key, data, option);
if (result == EBPF_SUCCESS) {
ebpf_bitmap_set_bit((ebpf_bitmap_t*)trie_map->data, prefix_length, true);
}
return result;
}
ebpf_map_function_table_t ebpf_map_function_tables[] = {
@ -1037,12 +1033,12 @@ ebpf_map_function_table_t ebpf_map_function_tables[] = {
_next_hash_map_key},
// LPM_TRIE is currently a hash-map with special behavior for find.
{// BPF_MAP_TYPE_LPM_TRIE
_create_hash_map,
_create_lpm_map,
_delete_hash_map,
NULL,
_find_lpm_trie_map_entry,
_find_lpm_map_entry,
NULL,
_update_hash_map_entry,
_update_lpm_map_entry,
NULL,
NULL,
_delete_hash_map_entry,

5
libs/execution_context/unit/execution_context_unit_test.cpp
Просмотреть файл

@ -167,6 +167,7 @@ TEST_CASE("map_crud_operations_lpm_trie_32", "[execution_context]")
{{16, 192, 168, 0, 0}, "192.168.0.0/16"},
{{16, 10, 10, 0, 0}, "10.0.0.0/16"},
{{8, 10, 0, 0, 0}, "10.0.0.0/8"},
{{0, 0, 0, 0, 0}, "0.0.0.0/0"},
};
std::vector<std::pair<lpm_trie_key_t, std::string>> tests{
@ -178,6 +179,7 @@ TEST_CASE("map_crud_operations_lpm_trie_32", "[execution_context]")
{{32, 192, 168, 14, 9}, "192.168.0.0/16"},
{{32, 10, 10, 10, 10}, "10.0.0.0/16"},
{{32, 10, 11, 10, 10}, "10.0.0.0/8"},
{{32, 11, 0, 0, 0}, "0.0.0.0/0"},
};
for (auto& [key, value] : keys) {
@ -247,6 +249,7 @@ TEST_CASE("map_crud_operations_lpm_trie_128", "[execution_context]")
{{64}, "AA/64"},
{{64}, "BB/64"},
{{32}, "BB/32"},
{{0}, "/0"},
};
{
std::vector<uint8_t> values{
@ -272,6 +275,7 @@ TEST_CASE("map_crud_operations_lpm_trie_128", "[execution_context]")
{{64}, "AA/64"},
{{64}, "BB/64"},
{{32}, "BB/32"},
{{128}, "/0"},
};
{
std::vector<uint8_t> values{
@ -283,6 +287,7 @@ TEST_CASE("map_crud_operations_lpm_trie_128", "[execution_context]")
0xAA,
0xBB,
0xBB,
0xFF,
};
for (size_t index = 0; index < values.size(); index++) {
generate_prefix(tests[index].first.prefix_length, values[index], tests[index].first.value);

134
libs/platform/ebpf_bitmap.c Normal file
Просмотреть файл

@ -0,0 +1,134 @@
// Copyright (c) Microsoft Corporation
// SPDX-License-Identifier: MIT
#include "ebpf_bitmap.h"
typedef struct _ebpf_bitmap
{
size_t bit_count;
size_t data[1];
} ebpf_bitmap_t;
typedef struct _ebpf_bitmap_cursor_internal
{
const ebpf_bitmap_t* bitmap;
size_t index; // Current block being searched.
size_t current_block; // Copy of the current block.
} ebpf_bitmap_cursor_internal_t;
C_ASSERT(sizeof(ebpf_bitmap_cursor_internal_t) == sizeof(ebpf_bitmap_cursor_t));
#define BITS_IN_SIZE_T (sizeof(size_t) * 8)
#define BIT_COUNT_TO_BLOCK_COUNT(X) (((X) + BITS_IN_SIZE_T - 1) / BITS_IN_SIZE_T)
size_t
ebpf_bitmap_size(size_t bit_count)
{
return EBPF_OFFSET_OF(ebpf_bitmap_t, data) + BIT_COUNT_TO_BLOCK_COUNT(bit_count) * sizeof(size_t);
}
void
ebpf_bitmap_initialize(_Out_ ebpf_bitmap_t* bitmap, size_t bit_count)
{
bitmap->bit_count = bit_count;
memset(bitmap->data, 0, BIT_COUNT_TO_BLOCK_COUNT(bit_count) * sizeof(size_t));
}
bool
ebpf_bitmap_set_bit(_Inout_ ebpf_bitmap_t* bitmap, size_t index, bool interlocked)
{
volatile int64_t* block = (volatile int64_t*)(bitmap->data + index / BITS_IN_SIZE_T);
uint8_t position_within_block = index % BITS_IN_SIZE_T;
if (interlocked) {
return _interlockedbittestandset64(block, position_within_block);
} else {
return _bittestandset64((int64_t*)block, position_within_block);
}
}
bool
ebpf_bitmap_reset_bit(_Inout_ ebpf_bitmap_t* bitmap, size_t index, bool interlocked)
{
volatile int64_t* block = (volatile int64_t*)(bitmap->data + index / BITS_IN_SIZE_T);
uint8_t position_within_block = index % BITS_IN_SIZE_T;
if (interlocked) {
return _interlockedbittestandreset64(block, position_within_block);
} else {
return _bittestandreset64((int64_t*)block, position_within_block);
}
}
bool
ebpf_bitmap_test_bit(_In_ const ebpf_bitmap_t* bitmap, size_t index)
{
return _bittest64((const int64_t*)&(bitmap->data[index / BITS_IN_SIZE_T]), index % BITS_IN_SIZE_T);
}
void
ebpf_bitmap_start_forward_search(_In_ const ebpf_bitmap_t* bitmap, _Out_ ebpf_bitmap_cursor_t* cursor)
{
ebpf_bitmap_cursor_internal_t* internal_cursor = (ebpf_bitmap_cursor_internal_t*)cursor;
internal_cursor->bitmap = bitmap;
internal_cursor->index = 0;
internal_cursor->current_block = bitmap->data[internal_cursor->index / BITS_IN_SIZE_T];
}
void
ebpf_bitmap_start_reverse_search(_In_ const ebpf_bitmap_t* bitmap, _Out_ ebpf_bitmap_cursor_t* cursor)
{
ebpf_bitmap_cursor_internal_t* internal_cursor = (ebpf_bitmap_cursor_internal_t*)cursor;
internal_cursor->bitmap = bitmap;
internal_cursor->index = bitmap->bit_count - 1;
internal_cursor->current_block = bitmap->data[internal_cursor->index / BITS_IN_SIZE_T];
}
size_t
ebpf_bitmap_forward_search_next_bit(_Inout_ ebpf_bitmap_cursor_t* cursor)
{
ebpf_bitmap_cursor_internal_t* internal_cursor = (ebpf_bitmap_cursor_internal_t*)cursor;
while (internal_cursor->index < internal_cursor->bitmap->bit_count) {
unsigned long next_bit = 0;
if (_BitScanForward64(&next_bit, internal_cursor->current_block)) {
internal_cursor->index = next_bit + (internal_cursor->index / BITS_IN_SIZE_T) * BITS_IN_SIZE_T;
// Clear the bit.
internal_cursor->current_block &= ~((size_t)1 << next_bit);
return internal_cursor->index;
} else {
// Move to next block.
internal_cursor->index /= BITS_IN_SIZE_T;
internal_cursor->index++;
internal_cursor->index *= BITS_IN_SIZE_T;
internal_cursor->current_block = internal_cursor->bitmap->data[internal_cursor->index / BITS_IN_SIZE_T];
}
}
return MAXSIZE_T;
}
size_t
ebpf_bitmap_reverse_search_next_bit(_Inout_ ebpf_bitmap_cursor_t* cursor)
{
ebpf_bitmap_cursor_internal_t* internal_cursor = (ebpf_bitmap_cursor_internal_t*)cursor;
while (internal_cursor->index < internal_cursor->bitmap->bit_count) {
unsigned long next_bit = 0;
if (_BitScanReverse64(&next_bit, internal_cursor->current_block)) {
internal_cursor->index = next_bit + (internal_cursor->index / BITS_IN_SIZE_T) * BITS_IN_SIZE_T;
// Clear the bit.
internal_cursor->current_block &= ~((size_t)1 << next_bit);
return internal_cursor->index;
} else {
// Move to previous block.
internal_cursor->index /= BITS_IN_SIZE_T;
if (internal_cursor->index == 0) {
break;
}
internal_cursor->index--;
internal_cursor->index *= BITS_IN_SIZE_T;
internal_cursor->current_block = internal_cursor->bitmap->data[internal_cursor->index / BITS_IN_SIZE_T];
}
}
return MAXSIZE_T;
}

101
libs/platform/ebpf_bitmap.h Normal file
Просмотреть файл

@ -0,0 +1,101 @@
// Copyright (c) Microsoft Corporation
// SPDX-License-Identifier: MIT
#pragma once
#include <ebpf_platform.h>
#ifdef __cplusplus
extern "C"
{
#endif
typedef struct _ebpf_bitmap ebpf_bitmap_t;
typedef uintptr_t ebpf_bitmap_cursor_t[3];
/**
* @brief Compute the size in bytes required to hold a ebpf_bitmap_t.
*
* @param[in] bit_count Count of bits the bitmap will hold.
* @return Size in bytes required.
*/
size_t
ebpf_bitmap_size(size_t bit_count);
/**
* @brief Initialize an already allocated bitmap.
*
* @param[out] bitmap Pointer to the bitmap.
* @param[in] bit_count Count of bits to be stored.
*/
void
ebpf_bitmap_initialize(_Out_ ebpf_bitmap_t* bitmap, size_t bit_count);
/**
* @brief Set bit at index to true.
*
* @param[in,out] bitmap Pointer to the bitmap.
* @param[in] index Index to modify.
* @param[in] interlocked Perform the operation using interlocked.
* @return true if the bit was set, false otherwise.
*/
bool
ebpf_bitmap_set_bit(_Inout_ ebpf_bitmap_t* bitmap, size_t index, bool interlocked);
/**
* @brief Set bit at index to false.
*
* @param[in,out] bitmap Pointer to the bitmap.
* @param[in] index Index to modify.
* @param[in] interlocked Perform the operation using interlocked.
* @return true if the bit was set, false otherwise.
*/
bool
ebpf_bitmap_reset_bit(_Inout_ ebpf_bitmap_t* bitmap, size_t index, bool interlocked);
/**
* @brief Get the value of the bit at index.
*
* @param[in] bitmap Pointer to the bitmap.
* @param[in] index Index to modify.
* @return true if the bit was set, false otherwise.
*/
bool
ebpf_bitmap_test_bit(_In_ const ebpf_bitmap_t* bitmap, size_t index);
/**
* @brief Initialize a cursor to perform a forward scan of bits.
*
* @param[in] bitmap Pointer to the bitmap.
* @param[out] cursor Pointer to cursor.
*/
void
ebpf_bitmap_start_forward_search(_In_ const ebpf_bitmap_t* bitmap, _Out_ ebpf_bitmap_cursor_t* cursor);
/**
* @brief Initialize a cursor to perform a reverse scan of bits.
*
* @param[in] bitmap Pointer to the bitmap.
* @param[out] cursor Pointer to cursor.
*/
void
ebpf_bitmap_start_reverse_search(_In_ const ebpf_bitmap_t* bitmap, _Out_ ebpf_bitmap_cursor_t* cursor);
/**
* @brief Find the next set bit in the bitmap via forward search.
*
* @param[in,out] cursor Pointer to cursor.
* @return Offset of the next set bit.
*/
size_t
ebpf_bitmap_forward_search_next_bit(_Inout_ ebpf_bitmap_cursor_t* cursor);
/**
* @brief Find the next set bit in the bitmap via reverse search.
*
* @param[in,out] cursor Pointer to cursor.
* @return Offset of the next set bit.
*/
size_t
ebpf_bitmap_reverse_search_next_bit(_Inout_ ebpf_bitmap_cursor_t* cursor);
#ifdef __cplusplus
}
#endif

60
libs/platform/ebpf_hash_table.c
Просмотреть файл

@ -73,7 +73,7 @@ _ebpf_rol(uint32_t value, size_t count)
* @return Hash of key.
*/
unsigned long
_ebpf_murmur3_32(_In_ const uint8_t* key, size_t length, uint32_t seed)
_ebpf_murmur3_32(_In_ const uint8_t* key, size_t length_in_bits, uint32_t seed)
{
uint32_t c1 = 0xcc9e2d51;
uint32_t c2 = 0x1b873593;
@ -82,8 +82,10 @@ _ebpf_murmur3_32(_In_ const uint8_t* key, size_t length, uint32_t seed)
uint32_t m = 5;
uint32_t n = 0xe6546b64;
uint32_t hash = seed;
uint32_t length_in_bytes = ((uint32_t)length_in_bits / 8);
uint32_t remaining_bits = length_in_bits % 8;
for (size_t index = 0; (length - index) > 3; index += 4) {
for (size_t index = 0; (length_in_bytes - index) > 3; index += 4) {
uint32_t k = *(uint32_t*)(key + index);
k *= c1;
k = _ebpf_rol(k, r1);
@ -95,16 +97,23 @@ _ebpf_murmur3_32(_In_ const uint8_t* key, size_t length, uint32_t seed)
hash += n;
}
unsigned long remainder = 0;
for (size_t index = length & (~3); index < length; index++) {
for (size_t index = length_in_bytes & (~3); index < length_in_bytes; index++) {
remainder <<= 8;
remainder |= key[index];
}
if (remaining_bits) {
uint8_t bits = key[length_in_bytes];
bits >>= (8 - remaining_bits);
remainder <<= 8;
remainder |= bits;
}
remainder *= c1;
remainder = _ebpf_rol(remainder, r1);
remainder *= c2;
hash ^= remainder;
hash ^= (uint32_t)length;
hash ^= (uint32_t)length_in_bytes;
hash *= 0x85ebca6b;
hash ^= (hash >> r2);
hash *= 0xc2b2ae35;
@ -126,26 +135,44 @@ _ebpf_murmur3_32(_In_ const uint8_t* key, size_t length, uint32_t seed)
static int
_ebpf_hash_table_compare(_In_ const ebpf_hash_table_t* hash_table, _In_ const uint8_t* key_a, _In_ const uint8_t* key_b)
{
size_t length_a;
size_t length_b;
size_t length_a_in_bits;
size_t length_b_in_bits;
const uint8_t* data_a;
const uint8_t* data_b;
uint8_t remainder_a;
uint8_t remainder_b;
if (hash_table->extract) {
hash_table->extract(key_a, &data_a, &length_a);
hash_table->extract(key_b, &data_b, &length_b);
hash_table->extract(key_a, &data_a, &length_a_in_bits);
hash_table->extract(key_b, &data_b, &length_b_in_bits);
} else {
length_a = hash_table->key_size;
length_a_in_bits = hash_table->key_size * 8;
data_a = key_a;
length_b = hash_table->key_size;
length_b_in_bits = hash_table->key_size * 8;
data_b = key_b;
}
if (length_a < length_b) {
if (length_a_in_bits < length_b_in_bits) {
return -1;
}
if (length_a > length_b) {
if (length_a_in_bits > length_b_in_bits) {
return 1;
}
return memcmp(data_a, data_b, length_a);
int cmp_result = memcmp(data_a, data_b, length_a_in_bits / 8);
// No match or length ends on a byte boundary.
if (cmp_result != 0 || length_a_in_bits % 8 == 0) {
return cmp_result;
}
// Check remaining high-order bits.
remainder_a = data_a[length_a_in_bits / 8];
remainder_b = data_b[length_b_in_bits / 8];
remainder_a >>= 8 - (length_a_in_bits % 8);
remainder_b >>= 8 - (length_b_in_bits % 8);
if (remainder_a < remainder_b) {
return -1;
} else if (remainder_a > remainder_b) {
return 1;
} else {
return 0;
}
}
/**
@ -164,7 +191,7 @@ _ebpf_hash_table_compute_hash(_In_ const ebpf_hash_table_t* hash_table, _In_ con
if (hash_table->extract) {
hash_table->extract(key, &data, &length);
} else {
length = hash_table->key_size;
length = hash_table->key_size * 8;
data = key;
}
return _ebpf_murmur3_32(data, length, hash_table->seed);
@ -386,7 +413,10 @@ ebpf_hash_table_create(
size_t key_size,
size_t value_size,
size_t bucket_count,
_In_opt_ void (*extract)(_In_ const uint8_t* value, _Outptr_ const uint8_t** data, _Out_ size_t* num))
_In_opt_ void (*extract)(
_In_ const uint8_t* value,
_Outptr_result_buffer_((*length_in_bits + 7) / 8) const uint8_t** data,
_Out_ size_t* length_in_bits))
{
ebpf_result_t retval;
ebpf_hash_table_t* table = NULL;

2
libs/platform/ebpf_pinning_table.c
Просмотреть файл

@ -31,7 +31,7 @@ _ebpf_pinning_table_extract(_In_ const uint8_t* value, _Outptr_ const uint8_t**
{
const ebpf_utf8_string_t* key = *(ebpf_utf8_string_t**)value;
*data = key->value;
*length = key->length;
*length = key->length * 8;
}
static void

4
libs/platform/ebpf_platform.h
Просмотреть файл

@ -422,7 +422,9 @@ extern "C"
size_t value_size,
size_t bucket_count,
_In_opt_ void (*extract_function)(
_In_ const uint8_t* value, _Outptr_ const uint8_t** data, _Out_ size_t* length));
_In_ const uint8_t* value,
_Outptr_result_buffer_((*length_in_bits + 7) / 8) const uint8_t** data,
_Out_ size_t* length_in_bits));
/**
* @brief Remove all items from the hash table and release memory.

2
libs/platform/kernel/platform_kernel.vcxproj
Просмотреть файл

@ -39,6 +39,7 @@
</ProjectConfiguration>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\ebpf_bitmap.c" />
<ClCompile Include="..\ebpf_epoch.c" />
<ClCompile Include="..\ebpf_hash_table.c" />
<ClCompile Include="..\ebpf_object.c" />
@ -58,6 +59,7 @@
<ClInclude Include="..\..\..\include\ebpf_result.h" />
<ClInclude Include="..\..\..\include\ebpf_structs.h" />
<ClInclude Include="..\..\..\include\ebpf_windows.h" />
<ClInclude Include="..\ebpf_bitmap.h" />
<ClInclude Include="..\ebpf_epoch.h" />
<ClInclude Include="..\ebpf_handle.h" />
<ClInclude Include="..\ebpf_object.h" />

6
libs/platform/kernel/platform_kernel.vcxproj.filters
Просмотреть файл

@ -52,6 +52,9 @@
<ClCompile Include="..\ebpf_state.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\ebpf_bitmap.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\ebpf_epoch.h">
@ -102,6 +105,9 @@
<ClInclude Include="..\..\..\include\ebpf_windows.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\ebpf_bitmap.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\ebpf_program_types.acf">

46
libs/platform/unit/platform_unit_test.cpp
Просмотреть файл

@ -12,6 +12,7 @@
#include "catch_wrapper.hpp"
#include "ebpf_bind_program_data.h"
#include "ebpf_bitmap.h"
#include "ebpf_epoch.h"
#include "ebpf_nethooks.h"
#include "ebpf_platform.h"
@ -656,4 +657,47 @@ TEST_CASE("state_test", "[state]")
REQUIRE(ebpf_state_load(allocated_index_1, &retreived_value) == EBPF_SUCCESS);
REQUIRE(retreived_value == reinterpret_cast<uintptr_t>(&foo));
ebpf_state_terminate();
}
}
template <size_t bit_count>
void
bitmap_test()
{
std::vector<uint8_t> data(ebpf_bitmap_size(bit_count));
ebpf_bitmap_t* bitmap = reinterpret_cast<ebpf_bitmap_t*>(data.data());
ebpf_bitmap_initialize(bitmap, bit_count);
// Set every even bit interlocked.
for (size_t i = 0; i < bit_count; i += 2) {
ebpf_bitmap_set_bit(bitmap, i, true);
}
// Verify every even bit is set via ebpf_bitmap_test_bit.
for (size_t i = 0; i < bit_count; i += 2) {
REQUIRE(ebpf_bitmap_test_bit(bitmap, i));
}
// Verify every even bit is set via ebpf_bitmap_forward_search_next_bit.
ebpf_bitmap_cursor_t cursor;
ebpf_bitmap_start_forward_search(bitmap, &cursor);
for (size_t i = 0; i < bit_count; i += 2) {
REQUIRE(ebpf_bitmap_forward_search_next_bit(&cursor) == i);
}
REQUIRE(ebpf_bitmap_forward_search_next_bit(&cursor) == MAXSIZE_T);
ebpf_bitmap_start_reverse_search(bitmap, &cursor);
for (size_t i = 0; i < bit_count; i += 2) {
REQUIRE(ebpf_bitmap_reverse_search_next_bit(&cursor) == bit_count - i - 1);
}
REQUIRE(ebpf_bitmap_reverse_search_next_bit(&cursor) == MAXSIZE_T);
}
#define BIT_MASK_TEST(X) \
TEST_CASE("bitmap_test:" #X, "[platform]") { bitmap_test<X>(); }
BIT_MASK_TEST(33);
BIT_MASK_TEST(65);
BIT_MASK_TEST(129);
BIT_MASK_TEST(1025);

2
libs/platform/user/platform_user.vcxproj
Просмотреть файл

@ -23,6 +23,7 @@
</ProjectConfiguration>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\ebpf_bitmap.c" />
<ClCompile Include="..\ebpf_epoch.c" />
<ClCompile Include="..\ebpf_hash_table.c" />
<ClCompile Include="..\ebpf_object.c" />
@ -34,6 +35,7 @@
<ClCompile Include="ebpf_extension_user.c" />
<ClCompile Include="ebpf_handle_user.c" />
<ClCompile Include="ebpf_platform_user.cpp" />
<ClInclude Include="..\ebpf_bitmap.h" />
<ClInclude Include="..\ebpf_epoch.h" />
<ClInclude Include="..\ebpf_handle.h" />
<ClInclude Include="..\ebpf_object.h" />

6
libs/platform/user/platform_user.vcxproj.filters
Просмотреть файл

@ -55,6 +55,9 @@
<ClCompile Include="..\ebpf_state.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\ebpf_bitmap.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<Midl Include="..\ebpf_program_types.idl">
@ -88,5 +91,8 @@
<ClInclude Include="..\ebpf_state.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\ebpf_bitmap.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
</Project>

113
tests/performance/ExecutionContext.cpp
Просмотреть файл

@ -3,6 +3,8 @@
#define TEST_AREA "ExecutionContext"
#include <numeric>
#include "performance.h"
extern "C"
@ -138,8 +140,89 @@ typedef class _ebpf_map_test_state
ebpf_map_t* map;
} ebpf_map_test_state_t;
typedef class _ebpf_map_lpm_trie_test_state
{
public:
_ebpf_map_lpm_trie_test_state() : map(nullptr) { REQUIRE(ebpf_core_initiate() == EBPF_SUCCESS); }
void
populate_ipv4_routes(size_t route_count)
{
ebpf_utf8_string_t name{(uint8_t*)"ipv4_route_table", 11};
ebpf_map_definition_in_memory_t definition{
sizeof(ebpf_map_definition_in_memory_t),
BPF_MAP_TYPE_LPM_TRIE,
sizeof(uint32_t) * 2,
sizeof(uint64_t),
static_cast<uint32_t>(route_count)};
REQUIRE(ebpf_map_create(&name, &definition, ebpf_handle_invalid, &map) == EBPF_SUCCESS);
// Prefix Length Distributions from https://bgp.potaroo.net/as2.0/bgp-active.html
std::vector<size_t> ipv4_prefix_length_distribution{
0, 0, 0, 0, 0, 0, 0, 16, 13, 41, 102, 306, 596, 1215, 2090, 13647,
8391, 14216, 25741, 43665, 53098, 109281, 97781, 523876, 1459, 0, 0, 1, 0, 1, 0, 1,
};
size_t total = 0;
total = std::accumulate(ipv4_prefix_length_distribution.begin(), ipv4_prefix_length_distribution.end(), total);
for (size_t prefix_length = 0; prefix_length < ipv4_prefix_length_distribution.size(); prefix_length++) {
size_t scaled_size = ipv4_prefix_length_distribution[prefix_length] * route_count / total;
for (size_t count = 0; count < scaled_size; count++) {
ipv4_routes.push_back({static_cast<uint32_t>(prefix_length + 1), ebpf_random_uint32()});
}
}
for (auto& [prefix_length, prefix] : ipv4_routes) {
std::vector<uint8_t> prefix_bytes(sizeof(uint32_t));
*reinterpret_cast<uint32_t*>(prefix_bytes.data()) = prefix;
populate_route(prefix_bytes, prefix_length);
}
}
void
populate_route(const std::vector<uint8_t>& prefix, uint32_t length)
{
std::vector<uint8_t> value(sizeof(uint64_t));
std::vector<uint8_t> key(prefix.size() + sizeof(length));
memcpy(key.data(), &length, sizeof(length));
std::copy(prefix.begin(), prefix.end(), key.begin() + sizeof(length));
ebpf_epoch_enter();
REQUIRE(
ebpf_map_update_entry(map, key.size(), key.data(), value.size(), value.data(), EBPF_ANY, 0) ==
EBPF_SUCCESS);
ebpf_epoch_exit();
}
void
test_find_ipv4_route()
{
struct _key
{
uint32_t prefix_length;
uint32_t prefix;
} ipv4_key = {32, ipv4_routes[ebpf_random_uint32() % ipv4_routes.size()].second};
volatile uint64_t* value = nullptr;
ebpf_epoch_enter();
ebpf_map_find_entry(map, sizeof(ipv4_key), (uint8_t*)&ipv4_key, sizeof(value), (uint8_t*)&value, 0);
UNREFERENCED_PARAMETER(value);
ebpf_epoch_exit();
}
~_ebpf_map_lpm_trie_test_state()
{
ebpf_object_release_reference((ebpf_object_t*)map);
ebpf_core_terminate();
}
private:
ebpf_map_t* map;
std::vector<std::pair<uint32_t, uint32_t>> ipv4_routes;
} ebpf_map_lpm_trie_test_state_t;
static ebpf_program_test_state_t* _ebpf_program_test_state_instance = nullptr;
static ebpf_map_test_state_t* _ebpf_map_test_state_instance = nullptr;
static ebpf_map_lpm_trie_test_state_t* _ebpf_map_lpm_trie_test_state_instance = nullptr;
static void
_ebpf_program_invoke()
@ -165,6 +248,12 @@ _map_update_test(uint32_t cpu_id)
_ebpf_map_test_state_instance->test_update(cpu_id);
}
static void
_lpm_trie_ipv4_find()
{
_ebpf_map_lpm_trie_test_state_instance->test_find_ipv4_route();
}
static const char*
_ebpf_map_type_t_to_string(ebpf_map_type_t type)
{
@ -262,6 +351,23 @@ test_program_invoke_interpret(bool preemptible)
measure.run_test();
}
template <size_t route_count>
void
test_lpm_trie_ipv4(bool preemptible)
{
size_t iterations = PERFORMANCE_MEASURE_ITERATION_COUNT;
_ebpf_map_lpm_trie_test_state lpm_trie_state;
lpm_trie_state.populate_ipv4_routes(route_count);
_ebpf_map_lpm_trie_test_state_instance = &lpm_trie_state;
std::string name = __FUNCTION__;
name += "<";
name += std::to_string(route_count);
name += ">";
_performance_measure measure(name.c_str(), preemptible, _lpm_trie_ipv4_find, iterations);
measure.run_test();
}
PERF_TEST(test_program_invoke_jit);
PERF_TEST(test_program_invoke_interpret);
@ -278,4 +384,9 @@ PERF_TEST(test_bpf_map_lookup_elem_write<BPF_MAP_TYPE_PERCPU_ARRAY>);
PERF_TEST(test_bpf_map_update_elem<BPF_MAP_TYPE_HASH>);
PERF_TEST(test_bpf_map_update_elem<BPF_MAP_TYPE_ARRAY>);
PERF_TEST(test_bpf_map_update_elem<BPF_MAP_TYPE_PERCPU_HASH>);
PERF_TEST(test_bpf_map_update_elem<BPF_MAP_TYPE_PERCPU_ARRAY>);
PERF_TEST(test_bpf_map_update_elem<BPF_MAP_TYPE_PERCPU_ARRAY>);
PERF_TEST(test_lpm_trie_ipv4<1024>);
PERF_TEST(test_lpm_trie_ipv4<1024 * 16>);
PERF_TEST(test_lpm_trie_ipv4<1024 * 256>);
PERF_TEST(test_lpm_trie_ipv4<1024 * 1024>);

3
tests/sample/sample.vcxproj.filters
Просмотреть файл

@ -63,5 +63,8 @@
<CustomBuild Include="test_utility_helpers.c">
<Filter>Source Files</Filter>
</CustomBuild>
<CustomBuild Include="encap_reflect_packet.c">
<Filter>Source Files</Filter>
</CustomBuild>
</ItemGroup>
</Project>