diff --git a/memory/build/malloc_decls.h b/memory/build/malloc_decls.h index d5e1c296287f..ddd7fee86293 100644 --- a/memory/build/malloc_decls.h +++ b/memory/build/malloc_decls.h @@ -38,7 +38,7 @@ #ifdef MALLOC_DECL #if MALLOC_FUNCS & MALLOC_FUNCS_INIT -MALLOC_DECL(init, void, const malloc_table_t*) +MALLOC_DECL(init, void, malloc_table_t*) #endif #if MALLOC_FUNCS & MALLOC_FUNCS_BRIDGE MALLOC_DECL(get_bridge, struct ReplaceMallocBridge*) diff --git a/memory/build/mozjemalloc.cpp b/memory/build/mozjemalloc.cpp index 31f0682d5c23..385688258b20 100644 --- a/memory/build/mozjemalloc.cpp +++ b/memory/build/mozjemalloc.cpp @@ -4822,12 +4822,10 @@ static #define MALLOC_DECL(name, return_type, ...) MozJemalloc::name, -static const malloc_table_t malloc_table = { +static malloc_table_t gReplaceMallocTable = { #include "malloc_decls.h" }; -static malloc_table_t replace_malloc_table; - #ifdef MOZ_NO_REPLACE_FUNC_DECL #define MALLOC_DECL(name, return_type, ...) \ typedef return_type(name##_impl_t)(__VA_ARGS__); \ @@ -4872,18 +4870,6 @@ replace_malloc_handle() #define REPLACE_MALLOC_GET_FUNC(handle, name) \ (name##_impl_t*)dlsym(handle, "replace_" #name) -#else - -typedef bool replace_malloc_handle_t; - -static replace_malloc_handle_t -replace_malloc_handle() -{ - return true; -} - -#define REPLACE_MALLOC_GET_FUNC(handle, name) replace_##name - #endif static void @@ -4891,17 +4877,28 @@ replace_malloc_init_funcs(); // Below is the malloc implementation overriding jemalloc and calling the // replacement functions if they exist. -static int replace_malloc_initialized = 0; +static bool gReplaceMallocInitialized = false; static void init() { - replace_malloc_init_funcs(); +#ifdef MOZ_NO_REPLACE_FUNC_DECL + replace_malloc_handle_t handle = replace_malloc_handle(); + if (handle) { +#define MALLOC_DECL(name, ...) \ + replace_##name = REPLACE_MALLOC_GET_FUNC(handle, name); + +#define MALLOC_FUNCS (MALLOC_FUNCS_INIT | MALLOC_FUNCS_BRIDGE) +#include "malloc_decls.h" + } +#endif + // Set this *before* calling replace_init, otherwise if replace_init calls // malloc() we'll get an infinite loop. - replace_malloc_initialized = 1; + gReplaceMallocInitialized = true; if (replace_init) { - replace_init(&malloc_table); + replace_init(&gReplaceMallocTable); } + replace_malloc_init_funcs(); } #define MALLOC_DECL(name, return_type, ...) \ @@ -4909,17 +4906,17 @@ init() inline return_type ReplaceMalloc::name( \ ARGS_HELPER(TYPED_ARGS, ##__VA_ARGS__)) \ { \ - if (MOZ_UNLIKELY(!replace_malloc_initialized)) { \ + if (MOZ_UNLIKELY(!gReplaceMallocInitialized)) { \ init(); \ } \ - return replace_malloc_table.name(ARGS_HELPER(ARGS, ##__VA_ARGS__)); \ + return gReplaceMallocTable.name(ARGS_HELPER(ARGS, ##__VA_ARGS__)); \ } #include "malloc_decls.h" MOZ_JEMALLOC_API struct ReplaceMallocBridge* get_bridge(void) { - if (MOZ_UNLIKELY(!replace_malloc_initialized)) { + if (MOZ_UNLIKELY(!gReplaceMallocInitialized)) { init(); } if (MOZ_LIKELY(!replace_get_bridge)) { @@ -4936,46 +4933,29 @@ get_bridge(void) static void replace_malloc_init_funcs() { - replace_malloc_handle_t handle = replace_malloc_handle(); - if (handle) { -#ifdef MOZ_NO_REPLACE_FUNC_DECL -#define MALLOC_DECL(name, ...) \ - replace_##name = REPLACE_MALLOC_GET_FUNC(handle, name); - -#define MALLOC_FUNCS (MALLOC_FUNCS_INIT | MALLOC_FUNCS_BRIDGE) -#include "malloc_decls.h" -#endif - -#define MALLOC_DECL(name, ...) \ - replace_malloc_table.name = REPLACE_MALLOC_GET_FUNC(handle, name); -#include "malloc_decls.h" - } - - if (!replace_malloc_table.posix_memalign && replace_malloc_table.memalign) { - replace_malloc_table.posix_memalign = + if (gReplaceMallocTable.posix_memalign == MozJemalloc::posix_memalign && + gReplaceMallocTable.memalign != MozJemalloc::memalign) { + gReplaceMallocTable.posix_memalign = AlignedAllocator::posix_memalign; } - if (!replace_malloc_table.aligned_alloc && replace_malloc_table.memalign) { - replace_malloc_table.aligned_alloc = + if (gReplaceMallocTable.aligned_alloc == MozJemalloc::aligned_alloc && + gReplaceMallocTable.memalign != MozJemalloc::memalign) { + gReplaceMallocTable.aligned_alloc = AlignedAllocator::aligned_alloc; } - if (!replace_malloc_table.valloc && replace_malloc_table.memalign) { - replace_malloc_table.valloc = + if (gReplaceMallocTable.valloc == MozJemalloc::valloc && + gReplaceMallocTable.memalign != MozJemalloc::memalign) { + gReplaceMallocTable.valloc = AlignedAllocator::valloc; } - if (!replace_malloc_table.moz_create_arena_with_params && - replace_malloc_table.malloc) { + if (gReplaceMallocTable.moz_create_arena_with_params == + MozJemalloc::moz_create_arena_with_params && + gReplaceMallocTable.malloc != MozJemalloc::malloc) { #define MALLOC_DECL(name, ...) \ - replace_malloc_table.name = DummyArenaAllocator::name; + gReplaceMallocTable.name = DummyArenaAllocator::name; #define MALLOC_FUNCS MALLOC_FUNCS_ARENA #include "malloc_decls.h" } - -#define MALLOC_DECL(name, ...) \ - if (!replace_malloc_table.name) { \ - replace_malloc_table.name = MozJemalloc::name; \ - } -#include "malloc_decls.h" } #endif // MOZ_REPLACE_MALLOC diff --git a/memory/build/replace_malloc.h b/memory/build/replace_malloc.h index 2f6b85c352cb..8f697309773c 100644 --- a/memory/build/replace_malloc.h +++ b/memory/build/replace_malloc.h @@ -19,12 +19,14 @@ // An initialization function is called before any malloc replacement // function, and has the following declaration: // -// void replace_init(const malloc_table_t *) +// void replace_init(malloc_table_t *) // -// The const malloc_table_t pointer given to that function is a table -// containing pointers to the original jemalloc implementation, so that -// replacement functions can call them back if they need to. The pointer -// itself can safely be kept around (no need to copy the table itself). +// The malloc_table_t pointer given to that function is a table containing +// pointers to the original allocator implementation, so that replacement +// functions can call them back if they need to. The initialization function +// needs to alter that table to replace the function it wants to replace. +// If it needs the original implementation, it thus needs a copy of the +// original table. // // The functions to be implemented in the external library are of the form: // @@ -39,7 +41,7 @@ // return ptr; // } // -// where "orig" is the pointer obtained from replace_init. +// where "orig" is a pointer to a copy of the table replace_init got. // // See malloc_decls.h for a list of functions that can be replaced this // way. The implementations are all in the form: @@ -83,10 +85,16 @@ MOZ_BEGIN_EXTERN_C #define MOZ_REPLACE_WEAK #endif +// Export replace_init and replace_get_bridge. #define MALLOC_DECL(name, return_type, ...) \ MOZ_EXPORT return_type replace_##name(__VA_ARGS__) MOZ_REPLACE_WEAK; -#define MALLOC_FUNCS MALLOC_FUNCS_ALL +#define MALLOC_FUNCS (MALLOC_FUNCS_INIT | MALLOC_FUNCS_BRIDGE) +#include "malloc_decls.h" + +// Define the remaining replace_* functions as not exported. +#define MALLOC_DECL(name, return_type, ...) \ + return_type replace_##name(__VA_ARGS__); #include "malloc_decls.h" #endif // MOZ_NO_REPLACE_FUNC_DECL diff --git a/memory/replace/dmd/DMD.cpp b/memory/replace/dmd/DMD.cpp index 593a14055214..ec62559bca27 100644 --- a/memory/replace/dmd/DMD.cpp +++ b/memory/replace/dmd/DMD.cpp @@ -89,7 +89,7 @@ StatusMsg(const char* aFmt, ...) void operator=(const T&) #endif -static const malloc_table_t* gMallocTable = nullptr; +static malloc_table_t gMallocTable; // Whether DMD finished initializing. static bool gIsDMDInitialized = false; @@ -118,13 +118,13 @@ public: { if (aNumElems & mozilla::tl::MulOverflowMask::value) return nullptr; - return (T*)gMallocTable->malloc(aNumElems * sizeof(T)); + return (T*)gMallocTable.malloc(aNumElems * sizeof(T)); } template static T* maybe_pod_calloc(size_t aNumElems) { - return (T*)gMallocTable->calloc(aNumElems, sizeof(T)); + return (T*)gMallocTable.calloc(aNumElems, sizeof(T)); } template @@ -132,12 +132,12 @@ public: { if (aNewSize & mozilla::tl::MulOverflowMask::value) return nullptr; - return (T*)gMallocTable->realloc(aPtr, aNewSize * sizeof(T)); + return (T*)gMallocTable.realloc(aPtr, aNewSize * sizeof(T)); } static void* malloc_(size_t aSize) { - void* p = gMallocTable->malloc(aSize); + void* p = gMallocTable.malloc(aSize); ExitOnFailure(p); return p; } @@ -152,7 +152,7 @@ public: static void* calloc_(size_t aSize) { - void* p = gMallocTable->calloc(1, aSize); + void* p = gMallocTable.calloc(1, aSize); ExitOnFailure(p); return p; } @@ -168,7 +168,7 @@ public: // This realloc_ is the one we use for direct reallocs within DMD. static void* realloc_(void* aPtr, size_t aNewSize) { - void* p = gMallocTable->realloc(aPtr, aNewSize); + void* p = gMallocTable.realloc(aPtr, aNewSize); ExitOnFailure(p); return p; } @@ -184,12 +184,12 @@ public: static void* memalign_(size_t aAlignment, size_t aSize) { - void* p = gMallocTable->memalign(aAlignment, aSize); + void* p = gMallocTable.memalign(aAlignment, aSize); ExitOnFailure(p); return p; } - static void free_(void* aPtr) { gMallocTable->free(aPtr); } + static void free_(void* aPtr) { gMallocTable.free(aPtr); } static char* strdup_(const char* aStr) { @@ -229,7 +229,7 @@ public: static size_t MallocSizeOf(const void* aPtr) { - return gMallocTable->malloc_usable_size(const_cast(aPtr)); + return gMallocTable.malloc_usable_size(const_cast(aPtr)); } void @@ -1220,7 +1220,7 @@ AllocCallback(void* aPtr, size_t aReqSize, Thread* aT) AutoLockState lock; AutoBlockIntercepts block(aT); - size_t actualSize = gMallocTable->malloc_usable_size(aPtr); + size_t actualSize = gMallocTable.malloc_usable_size(aPtr); // We may or may not record the allocation stack trace, depending on the // options and the outcome of a Bernoulli trial. @@ -1259,15 +1259,18 @@ FreeCallback(void* aPtr, Thread* aT, DeadBlock* aDeadBlock) // malloc/free interception //--------------------------------------------------------------------------- -static void Init(const malloc_table_t* aMallocTable); +static void Init(malloc_table_t* aMallocTable); } // namespace dmd } // namespace mozilla void -replace_init(const malloc_table_t* aMallocTable) +replace_init(malloc_table_t* aMallocTable) { mozilla::dmd::Init(aMallocTable); +#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC_BASE +#define MALLOC_DECL(name, ...) aMallocTable->name = replace_ ## name; +#include "malloc_decls.h" } ReplaceMallocBridge* @@ -1286,7 +1289,7 @@ replace_malloc(size_t aSize) // we're still in Init() and something has indirectly called malloc. Do a // vanilla malloc. (In the latter case, if it fails we'll crash. But // OOM is highly unlikely so early on.) - return gMallocTable->malloc(aSize); + return gMallocTable.malloc(aSize); } Thread* t = Thread::Fetch(); @@ -1297,7 +1300,7 @@ replace_malloc(size_t aSize) } // This must be a call to malloc from outside DMD. Intercept it. - void* ptr = gMallocTable->malloc(aSize); + void* ptr = gMallocTable.malloc(aSize); AllocCallback(ptr, aSize, t); return ptr; } @@ -1308,7 +1311,7 @@ replace_calloc(size_t aCount, size_t aSize) using namespace mozilla::dmd; if (!gIsDMDInitialized) { - return gMallocTable->calloc(aCount, aSize); + return gMallocTable.calloc(aCount, aSize); } Thread* t = Thread::Fetch(); @@ -1316,7 +1319,7 @@ replace_calloc(size_t aCount, size_t aSize) return InfallibleAllocPolicy::calloc_(aCount * aSize); } - void* ptr = gMallocTable->calloc(aCount, aSize); + void* ptr = gMallocTable.calloc(aCount, aSize); AllocCallback(ptr, aCount * aSize, t); return ptr; } @@ -1327,7 +1330,7 @@ replace_realloc(void* aOldPtr, size_t aSize) using namespace mozilla::dmd; if (!gIsDMDInitialized) { - return gMallocTable->realloc(aOldPtr, aSize); + return gMallocTable.realloc(aOldPtr, aSize); } Thread* t = Thread::Fetch(); @@ -1346,7 +1349,7 @@ replace_realloc(void* aOldPtr, size_t aSize) // move, but doing better isn't worth the effort. DeadBlock db; FreeCallback(aOldPtr, t, &db); - void* ptr = gMallocTable->realloc(aOldPtr, aSize); + void* ptr = gMallocTable.realloc(aOldPtr, aSize); if (ptr) { AllocCallback(ptr, aSize, t); MaybeAddToDeadBlockTable(db); @@ -1357,7 +1360,7 @@ replace_realloc(void* aOldPtr, size_t aSize) // block will end up looking like it was allocated for the first time here, // which is untrue, and the slop bytes will be zero, which may be untrue. // But this case is rare and doing better isn't worth the effort. - AllocCallback(aOldPtr, gMallocTable->malloc_usable_size(aOldPtr), t); + AllocCallback(aOldPtr, gMallocTable.malloc_usable_size(aOldPtr), t); } return ptr; } @@ -1368,7 +1371,7 @@ replace_memalign(size_t aAlignment, size_t aSize) using namespace mozilla::dmd; if (!gIsDMDInitialized) { - return gMallocTable->memalign(aAlignment, aSize); + return gMallocTable.memalign(aAlignment, aSize); } Thread* t = Thread::Fetch(); @@ -1376,7 +1379,7 @@ replace_memalign(size_t aAlignment, size_t aSize) return InfallibleAllocPolicy::memalign_(aAlignment, aSize); } - void* ptr = gMallocTable->memalign(aAlignment, aSize); + void* ptr = gMallocTable.memalign(aAlignment, aSize); AllocCallback(ptr, aSize, t); return ptr; } @@ -1387,7 +1390,7 @@ replace_free(void* aPtr) using namespace mozilla::dmd; if (!gIsDMDInitialized) { - gMallocTable->free(aPtr); + gMallocTable.free(aPtr); return; } @@ -1402,7 +1405,7 @@ replace_free(void* aPtr) DeadBlock db; FreeCallback(aPtr, t, &db); MaybeAddToDeadBlockTable(db); - gMallocTable->free(aPtr); + gMallocTable.free(aPtr); } namespace mozilla { @@ -1582,9 +1585,9 @@ postfork() // gStackTraceTable are allocated dynamically (so we can guarantee their // construction in this function) rather than statically. static void -Init(const malloc_table_t* aMallocTable) +Init(malloc_table_t* aMallocTable) { - gMallocTable = aMallocTable; + gMallocTable = *aMallocTable; gDMDBridge = InfallibleAllocPolicy::new_(); #ifndef XP_WIN diff --git a/memory/replace/logalloc/LogAlloc.cpp b/memory/replace/logalloc/LogAlloc.cpp index df4f253aad7e..84517fec789a 100644 --- a/memory/replace/logalloc/LogAlloc.cpp +++ b/memory/replace/logalloc/LogAlloc.cpp @@ -22,7 +22,7 @@ #include "base/lock.h" -static const malloc_table_t* sFuncs = nullptr; +static malloc_table_t sFuncs; static intptr_t sFd = 0; static bool sStdoutOrStderr = false; @@ -74,9 +74,18 @@ class LogAllocBridge : public ReplaceMallocBridge }; void -replace_init(const malloc_table_t* aTable) +replace_init(malloc_table_t* aTable) { - sFuncs = aTable; + sFuncs = *aTable; +#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC_BASE +#define MALLOC_DECL(name, ...) aTable->name = replace_ ## name; +#include "malloc_decls.h" + aTable->jemalloc_stats = replace_jemalloc_stats; +#ifndef LOGALLOC_MINIMAL + aTable->posix_memalign = replace_posix_memalign; + aTable->aligned_alloc = replace_aligned_alloc; + aTable->valloc = replace_valloc; +#endif #ifndef _WIN32 /* When another thread has acquired a lock before forking, the child @@ -107,7 +116,7 @@ replace_init(const malloc_table_t* aTable) * called after ours, which means it needs to be registered before ours. * So trick the real allocator into initializing itself without more side * effects by calling malloc with a size it can't possibly allocate. */ - sFuncs->malloc(-1); + sFuncs.malloc(-1); pthread_atfork(prefork, postfork, postfork); #endif @@ -175,7 +184,7 @@ void* replace_malloc(size_t aSize) { AutoLock lock(sLock); - void* ptr = sFuncs->malloc(aSize); + void* ptr = sFuncs.malloc(aSize); if (ptr) { FdPrintf(sFd, "%zu %zu malloc(%zu)=%p\n", GetPid(), GetTid(), aSize, ptr); } @@ -187,7 +196,7 @@ int replace_posix_memalign(void** aPtr, size_t aAlignment, size_t aSize) { AutoLock lock(sLock); - int ret = sFuncs->posix_memalign(aPtr, aAlignment, aSize); + int ret = sFuncs.posix_memalign(aPtr, aAlignment, aSize); if (ret == 0) { FdPrintf(sFd, "%zu %zu posix_memalign(%zu,%zu)=%p\n", GetPid(), GetTid(), aAlignment, aSize, *aPtr); @@ -199,7 +208,7 @@ void* replace_aligned_alloc(size_t aAlignment, size_t aSize) { AutoLock lock(sLock); - void* ptr = sFuncs->aligned_alloc(aAlignment, aSize); + void* ptr = sFuncs.aligned_alloc(aAlignment, aSize); if (ptr) { FdPrintf(sFd, "%zu %zu aligned_alloc(%zu,%zu)=%p\n", GetPid(), GetTid(), aAlignment, aSize, ptr); @@ -212,7 +221,7 @@ void* replace_calloc(size_t aNum, size_t aSize) { AutoLock lock(sLock); - void* ptr = sFuncs->calloc(aNum, aSize); + void* ptr = sFuncs.calloc(aNum, aSize); if (ptr) { FdPrintf(sFd, "%zu %zu calloc(%zu,%zu)=%p\n", GetPid(), GetTid(), aNum, aSize, ptr); @@ -224,7 +233,7 @@ void* replace_realloc(void* aPtr, size_t aSize) { AutoLock lock(sLock); - void* new_ptr = sFuncs->realloc(aPtr, aSize); + void* new_ptr = sFuncs.realloc(aPtr, aSize); if (new_ptr || !aSize) { FdPrintf(sFd, "%zu %zu realloc(%p,%zu)=%p\n", GetPid(), GetTid(), aPtr, aSize, new_ptr); @@ -239,14 +248,14 @@ replace_free(void* aPtr) if (aPtr) { FdPrintf(sFd, "%zu %zu free(%p)\n", GetPid(), GetTid(), aPtr); } - sFuncs->free(aPtr); + sFuncs.free(aPtr); } void* replace_memalign(size_t aAlignment, size_t aSize) { AutoLock lock(sLock); - void* ptr = sFuncs->memalign(aAlignment, aSize); + void* ptr = sFuncs.memalign(aAlignment, aSize); if (ptr) { FdPrintf(sFd, "%zu %zu memalign(%zu,%zu)=%p\n", GetPid(), GetTid(), aAlignment, aSize, ptr); @@ -259,7 +268,7 @@ void* replace_valloc(size_t aSize) { AutoLock lock(sLock); - void* ptr = sFuncs->valloc(aSize); + void* ptr = sFuncs.valloc(aSize); if (ptr) { FdPrintf(sFd, "%zu %zu valloc(%zu)=%p\n", GetPid(), GetTid(), aSize, ptr); } @@ -271,6 +280,6 @@ void replace_jemalloc_stats(jemalloc_stats_t* aStats) { AutoLock lock(sLock); - sFuncs->jemalloc_stats(aStats); + sFuncs.jemalloc_stats(aStats); FdPrintf(sFd, "%zu %zu jemalloc_stats()\n", GetPid(), GetTid()); } diff --git a/mozglue/build/replace_malloc.mk b/mozglue/build/replace_malloc.mk index 320fb59321b1..472d51d88017 100644 --- a/mozglue/build/replace_malloc.mk +++ b/mozglue/build/replace_malloc.mk @@ -3,11 +3,5 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. ifeq (Darwin_1,$(OS_TARGET)_$(MOZ_REPLACE_MALLOC)) -# Technically, ) is not a necessary delimiter in the awk call, but make -# doesn't like the opening ( there being alone... -MK_LDFLAGS = \ - $(shell awk -F'[(),]' '/^MALLOC_DECL/{print "-Wl,-U,_replace_" $$2}' $(topsrcdir)/memory/build/malloc_decls.h) \ - $(NULL) - -EXTRA_DEPS += $(topsrcdir)/memory/build/malloc_decls.h +MK_LDFLAGS = -Wl,-U,_replace_init -Wl,-U,_replace_get_bridge endif