diff --git a/src/mem/slowalloc.h b/src/mem/slowalloc.h new file mode 100644 index 00000000..c726cfce --- /dev/null +++ b/src/mem/slowalloc.h @@ -0,0 +1,68 @@ +#include "globalalloc.h" + +namespace snmalloc +{ + /** + * RAII wrapper around an `Alloc`. This class gets an allocator from the + * global pool and wraps it so that `Alloc` methods can be called + * directly via the `->` operator on this class. When this object is + * destroyed, it returns the allocator to the global pool. + * + * This does not depend on thread-local storage working, so can be used for + * bootstrapping. + */ + struct SlowAllocator + { + /** + * The allocator that this wrapper will use. + */ + Alloc* alloc; + /** + * Constructor. Claims an allocator from the global pool + */ + SlowAllocator() : alloc(current_alloc_pool()->acquire()) {} + /** + * Copying is not supported, it could easily lead to accidental sharing of + * allocators. + */ + SlowAllocator(const SlowAllocator&) = delete; + /** + * Moving is not supported, though it would be easy to add if there's a use + * case for it. + */ + SlowAllocator(SlowAllocator&&) = delete; + /** + * Copying is not supported, it could easily lead to accidental sharing of + * allocators. + */ + SlowAllocator& operator=(const SlowAllocator&) = delete; + /** + * Moving is not supported, though it would be easy to add if there's a use + * case for it. + */ + SlowAllocator& operator=(SlowAllocator&&) = delete; + /** + * Destructor. Returns the allocator to the pool. + */ + ~SlowAllocator() + { + current_alloc_pool()->release(alloc); + } + /** + * Arrow operator, allows methods exposed by `Alloc` to be called on the + * wrapper. + */ + Alloc* operator->() + { + return alloc; + } + }; + /** + * Returns a new slow allocator. When the `SlowAllocator` goes out of scope, + * the underlying `Alloc` will be returned to the pool. + */ + inline SlowAllocator get_slow_allocator() + { + return SlowAllocator{}; + } +} diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 2bdd5525..c879ab6e 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -1,21 +1,26 @@ +#include "../mem/slowalloc.h" #include "../snmalloc.h" #include using namespace snmalloc; +#ifndef SNMALLOC_EXPORT +# define SNMALLOC_EXPORT +#endif + #ifndef SNMALLOC_NAME_MANGLE # define SNMALLOC_NAME_MANGLE(a) a #endif extern "C" { - void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) { return Alloc::external_pointer(ptr); } - void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { // Include size 0 in the first sizeclass. size = ((size - 1) >> (bits::BITS - 1)) + size; @@ -23,7 +28,7 @@ extern "C" return ThreadAlloc::get()->alloc(size); } - void SNMALLOC_NAME_MANGLE(free)(void* ptr) + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) { if (ptr == nullptr) return; @@ -31,7 +36,7 @@ extern "C" ThreadAlloc::get()->dealloc(ptr); } - void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -45,12 +50,12 @@ extern "C" return ThreadAlloc::get()->alloc(sz); } - size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(void* ptr) + SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(void* ptr) { return Alloc::alloc_size(ptr); } - void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) { if (size == (size_t)-1) { @@ -88,7 +93,8 @@ extern "C" } #ifndef __FreeBSD__ - void* SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -101,14 +107,16 @@ extern "C" } #endif - void* SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size) + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size) { assert((size % alignment) == 0); (void)alignment; return SNMALLOC_NAME_MANGLE(malloc)(size); } - void* SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) { if ( (alignment == 0) || (alignment == size_t(-1)) || @@ -141,7 +149,7 @@ extern "C" return nullptr; } - int SNMALLOC_NAME_MANGLE(posix_memalign)( + SNMALLOC_EXPORT int SNMALLOC_NAME_MANGLE(posix_memalign)( void** memptr, size_t alignment, size_t size) { if ( @@ -161,13 +169,13 @@ extern "C" } #ifndef __FreeBSD__ - void* SNMALLOC_NAME_MANGLE(valloc)(size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(valloc)(size_t size) { return SNMALLOC_NAME_MANGLE(memalign)(OS_PAGE_SIZE, size); } #endif - void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size) + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size) { if (size == size_t(-1)) { @@ -178,11 +186,46 @@ extern "C" OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1)); } - void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {} - void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {} - void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {} - int SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t) + // Stub implementations for jemalloc compatibility. + // These are called by FreeBSD's libthr (pthreads) to notify malloc of + // various events. They are currently unused, though we may wish to reset + // statistics on fork if built with statistics. + + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {} + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {} + SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {} + + SNMALLOC_EXPORT int + SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t) { return ENOENT; } + +#ifndef __PIC__ + // The following functions are required to work before TLS is set up, in + // statically-linked programs. These temporarily grab an allocator from the + // pool and return it. + + void* __je_bootstrap_malloc(size_t size) + { + return get_slow_allocator()->alloc(size); + } + void* __je_bootstrap_calloc(size_t nmemb, size_t size) + { + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (overflow) + { + errno = ENOMEM; + return 0; + } + // Include size 0 in the first sizeclass. + sz = ((sz - 1) >> (bits::BITS - 1)) + sz; + return get_slow_allocator()->alloc(sz); + } + void __je_bootstrap_free(void* ptr) + { + get_slow_allocator()->dealloc(ptr); + } +#endif } diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index efa89786..90a2f5b7 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -4,6 +4,9 @@ # include "../ds/bits.h" # include "../mem/allocconfig.h" +# include +# include +# include # include namespace snmalloc