From 4ce8d0124cc0e761c77b1b890573d85618e9f159 Mon Sep 17 00:00:00 2001 From: Csoregi Natalia Date: Wed, 29 Nov 2017 03:08:46 +0200 Subject: [PATCH] Backed out 7 changesets (bug 1420355) for mass failures on OS X and Android. r=backout on a CLOSED TREE Backed out changeset a7ed89e13a4c (bug 1420355) Backed out changeset fd6702e6e0a0 (bug 1420355) Backed out changeset 0479dda078a2 (bug 1420355) Backed out changeset e69357ccca9e (bug 1420355) Backed out changeset 3742a4b69ba2 (bug 1420355) Backed out changeset 451cd087922f (bug 1420355) Backed out changeset d80b5c4e1dd0 (bug 1420355) --- browser/installer/package-manifest.in | 3 + build/automation.py.in | 24 +- build/mobile/remoteautomation.py | 5 +- build/moz.configure/memory.configure | 11 - memory/build/moz.build | 3 - memory/build/mozjemalloc.cpp | 28 -- memory/build/replace_malloc.h | 18 +- memory/replace/dmd/DMD.cpp | 89 +++-- memory/replace/dmd/moz.build | 14 +- .../test/script-diff-dark-matter-expected.txt | 2 +- .../dmd/test/script-diff-dark-matter2.json | 2 +- .../test/script-ignore-alloc-fns-expected.txt | 2 +- .../dmd/test/script-ignore-alloc-fns.json | 2 +- memory/replace/dmd/test/test_dmd.js | 2 +- memory/replace/logalloc/LogAlloc.cpp | 315 +++++++++--------- memory/replace/logalloc/README | 16 +- memory/replace/logalloc/logalloc.mozbuild | 3 +- memory/replace/logalloc/moz.build | 2 +- memory/replace/logalloc/replay/Makefile.in | 2 - memory/replace/logalloc/replay/moz.build | 14 +- memory/replace/moz.build | 8 - python/mozbuild/mozbuild/mach_commands.py | 29 +- testing/awsy/mach_commands.py | 28 +- testing/mochitest/mochitest_options.py | 16 + testing/mochitest/runtests.py | 4 +- testing/mochitest/runtestsremote.py | 9 +- testing/mozbase/mozrunner/mozrunner/utils.py | 26 +- testing/xpcshell/runxpcshelltests.py | 15 + 28 files changed, 393 insertions(+), 299 deletions(-) diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index 155ef2657935..eb9d47b2b688 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -68,6 +68,9 @@ #ifndef MOZ_STATIC_JS @BINPATH@/@DLL_PREFIX@mozjs@DLL_SUFFIX@ #endif +#ifdef MOZ_DMD +@BINPATH@/@DLL_PREFIX@dmd@DLL_SUFFIX@ +#endif #ifndef MOZ_SYSTEM_NSPR #ifndef MOZ_FOLD_LIBS @BINPATH@/@DLL_PREFIX@nspr4@DLL_SUFFIX@ diff --git a/build/automation.py.in b/build/automation.py.in index 079bf7db5896..c669da01795e 100644 --- a/build/automation.py.in +++ b/build/automation.py.in @@ -183,12 +183,34 @@ class Automation(object): else: os.kill(self.pid, signal.SIGKILL) - def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, lsanPath=None, ubsanPath=None): + def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None, ubsanPath=None): if xrePath == None: xrePath = self.DIST_BIN if env == None: env = dict(os.environ) + ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath)) + dmdLibrary = None + preloadEnvVar = None + if self.UNIXISH or self.IS_MAC: + envVar = "LD_LIBRARY_PATH" + preloadEnvVar = "LD_PRELOAD" + if self.IS_MAC: + envVar = "DYLD_LIBRARY_PATH" + dmdLibrary = "libdmd.dylib" + else: # unixish + dmdLibrary = "libdmd.so" + if envVar in env: + ldLibraryPath = ldLibraryPath + ":" + env[envVar] + env[envVar] = ldLibraryPath + elif self.IS_WIN32: + env["PATH"] = env["PATH"] + ";" + str(ldLibraryPath) + dmdLibrary = "dmd.dll" + preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" + + if dmdPath and dmdLibrary and preloadEnvVar: + env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) + if crashreporter and not debugger: env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['MOZ_CRASHREPORTER'] = '1' diff --git a/build/mobile/remoteautomation.py b/build/mobile/remoteautomation.py index cde9406983c3..00ae2a8ec5a5 100644 --- a/build/mobile/remoteautomation.py +++ b/build/mobile/remoteautomation.py @@ -60,12 +60,15 @@ class RemoteAutomation(Automation): self._remoteLog = logfile # Set up what we need for the remote environment - def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, lsanPath=None, ubsanPath=None): + def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None, ubsanPath=None): # Because we are running remote, we don't want to mimic the local env # so no copying of os.environ if env is None: env = {} + if dmdPath: + env['MOZ_REPLACE_MALLOC_LIB'] = os.path.join(dmdPath, 'libdmd.so') + # Except for the mochitest results table hiding option, which isn't # passed to runtestsremote.py as an actual option, but through the # MOZ_HIDE_RESULTS_TABLE environment variable. diff --git a/build/moz.configure/memory.configure b/build/moz.configure/memory.configure index 2889819c988b..f15e2bdbb775 100644 --- a/build/moz.configure/memory.configure +++ b/build/moz.configure/memory.configure @@ -62,14 +62,3 @@ def replace_malloc(value, jemalloc, milestone, build_project): set_config('MOZ_REPLACE_MALLOC', replace_malloc) set_define('MOZ_REPLACE_MALLOC', replace_malloc) add_old_configure_assignment('MOZ_REPLACE_MALLOC', replace_malloc) - - -@depends(replace_malloc, build_project) -def replace_malloc_static(replace_malloc, build_project): - # Default to statically linking replace-malloc libraries that can be - # statically linked, except when building with --enable-project=memory. - if replace_malloc and build_project != 'memory': - return True - - -set_config('MOZ_REPLACE_MALLOC_STATIC', replace_malloc_static) diff --git a/memory/build/moz.build b/memory/build/moz.build index 4cf347d165ea..93fe3e5c373c 100644 --- a/memory/build/moz.build +++ b/memory/build/moz.build @@ -49,7 +49,4 @@ if CONFIG['MOZ_BUILD_APP'] != 'memory': if CONFIG['_MSC_VER']: CXXFLAGS += ['-wd4273'] # inconsistent dll linkage (bug 558163) -if CONFIG['MOZ_REPLACE_MALLOC_STATIC']: - DEFINES['MOZ_REPLACE_MALLOC_STATIC'] = True - DisableStlWrapping() diff --git a/memory/build/mozjemalloc.cpp b/memory/build/mozjemalloc.cpp index b1e02a167425..51d48088ed6e 100644 --- a/memory/build/mozjemalloc.cpp +++ b/memory/build/mozjemalloc.cpp @@ -4874,20 +4874,6 @@ replace_malloc_handle() static void replace_malloc_init_funcs(); -#ifdef MOZ_REPLACE_MALLOC_STATIC -extern "C" void -logalloc_init(malloc_table_t*, ReplaceMallocBridge**); - -extern "C" void -dmd_init(malloc_table_t*, ReplaceMallocBridge**); -#endif - -bool -Equals(malloc_table_t& aTable1, malloc_table_t& aTable2) -{ - return memcmp(&aTable1, &aTable2, sizeof(malloc_table_t)) == 0; -} - // Below is the malloc implementation overriding jemalloc and calling the // replacement functions if they exist. static bool gReplaceMallocInitialized = false; @@ -4895,10 +4881,6 @@ static ReplaceMallocBridge* gReplaceMallocBridge = nullptr; static void init() { -#ifdef MOZ_REPLACE_MALLOC_STATIC - malloc_table_t initialTable = gReplaceMallocTable; -#endif - #ifdef MOZ_DYNAMIC_REPLACE_INIT replace_malloc_handle_t handle = replace_malloc_handle(); if (handle) { @@ -4912,16 +4894,6 @@ init() if (replace_init) { replace_init(&gReplaceMallocTable, &gReplaceMallocBridge); } -#ifdef MOZ_REPLACE_MALLOC_STATIC - if (Equals(initialTable, gReplaceMallocTable)) { - logalloc_init(&gReplaceMallocTable, &gReplaceMallocBridge); - } -#ifdef MOZ_DMD - if (Equals(initialTable, gReplaceMallocTable)) { - dmd_init(&gReplaceMallocTable, &gReplaceMallocBridge); - } -#endif -#endif replace_malloc_init_funcs(); } diff --git a/memory/build/replace_malloc.h b/memory/build/replace_malloc.h index 13c6abea6143..014ddfd7be08 100644 --- a/memory/build/replace_malloc.h +++ b/memory/build/replace_malloc.h @@ -76,7 +76,6 @@ // Implementing a replace-malloc library is incompatible with using mozalloc. #define MOZ_NO_MOZALLOC 1 -#include "mozilla/MacroArgs.h" #include "mozilla/Types.h" MOZ_BEGIN_EXTERN_C @@ -87,20 +86,15 @@ MOZ_BEGIN_EXTERN_C #define MOZ_REPLACE_WEAK #endif -// When building a replace-malloc library for static linking, we want -// each to have a different name for their "public" functions. -// The build system defines MOZ_REPLACE_MALLOC_PREFIX in that case. -#ifdef MOZ_REPLACE_MALLOC_PREFIX -#define replace_init MOZ_CONCAT(MOZ_REPLACE_MALLOC_PREFIX, _init) -#define MOZ_REPLACE_PUBLIC -#else -#define MOZ_REPLACE_PUBLIC MOZ_EXPORT -#endif - // Replace-malloc library initialization function. See top of this file -MOZ_REPLACE_PUBLIC void +MOZ_EXPORT void replace_init(malloc_table_t*, struct ReplaceMallocBridge**) MOZ_REPLACE_WEAK; +// Define the replace_* functions as not exported. +#define MALLOC_DECL(name, return_type, ...) \ + return_type replace_##name(__VA_ARGS__); +#include "malloc_decls.h" + MOZ_END_EXTERN_C #endif // replace_malloc_h diff --git a/memory/replace/dmd/DMD.cpp b/memory/replace/dmd/DMD.cpp index 4a9b8ba80e92..3a8cc9aabc40 100644 --- a/memory/replace/dmd/DMD.cpp +++ b/memory/replace/dmd/DMD.cpp @@ -91,6 +91,9 @@ StatusMsg(const char* aFmt, ...) static malloc_table_t gMallocTable; +// Whether DMD finished initializing. +static bool gIsDMDInitialized = false; + // This provides infallible allocations (they abort on OOM). We use it for all // of DMD's own allocations, which fall into the following three cases. // @@ -1256,16 +1259,34 @@ FreeCallback(void* aPtr, Thread* aT, DeadBlock* aDeadBlock) // malloc/free interception //--------------------------------------------------------------------------- -static bool Init(malloc_table_t* aMallocTable); +static void Init(malloc_table_t* aMallocTable); } // namespace dmd } // namespace mozilla -static void* +void +replace_init(malloc_table_t* aMallocTable, ReplaceMallocBridge** aBridge) +{ + mozilla::dmd::Init(aMallocTable); +#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC_BASE +#define MALLOC_DECL(name, ...) aMallocTable->name = replace_ ## name; +#include "malloc_decls.h" + *aBridge = mozilla::dmd::gDMDBridge; +} + +void* replace_malloc(size_t aSize) { using namespace mozilla::dmd; + if (!gIsDMDInitialized) { + // DMD hasn't started up, either because it wasn't enabled by the user, or + // we're still in Init() and something has indirectly called malloc. Do a + // vanilla malloc. (In the latter case, if it fails we'll crash. But + // OOM is highly unlikely so early on.) + return gMallocTable.malloc(aSize); + } + Thread* t = Thread::Fetch(); if (t->InterceptsAreBlocked()) { // Intercepts are blocked, which means this must be a call to malloc @@ -1279,11 +1300,15 @@ replace_malloc(size_t aSize) return ptr; } -static void* +void* replace_calloc(size_t aCount, size_t aSize) { using namespace mozilla::dmd; + if (!gIsDMDInitialized) { + return gMallocTable.calloc(aCount, aSize); + } + Thread* t = Thread::Fetch(); if (t->InterceptsAreBlocked()) { return InfallibleAllocPolicy::calloc_(aCount * aSize); @@ -1294,11 +1319,15 @@ replace_calloc(size_t aCount, size_t aSize) return ptr; } -static void* +void* replace_realloc(void* aOldPtr, size_t aSize) { using namespace mozilla::dmd; + if (!gIsDMDInitialized) { + return gMallocTable.realloc(aOldPtr, aSize); + } + Thread* t = Thread::Fetch(); if (t->InterceptsAreBlocked()) { return InfallibleAllocPolicy::realloc_(aOldPtr, aSize); @@ -1331,11 +1360,15 @@ replace_realloc(void* aOldPtr, size_t aSize) return ptr; } -static void* +void* replace_memalign(size_t aAlignment, size_t aSize) { using namespace mozilla::dmd; + if (!gIsDMDInitialized) { + return gMallocTable.memalign(aAlignment, aSize); + } + Thread* t = Thread::Fetch(); if (t->InterceptsAreBlocked()) { return InfallibleAllocPolicy::memalign_(aAlignment, aSize); @@ -1346,11 +1379,16 @@ replace_memalign(size_t aAlignment, size_t aSize) return ptr; } -static void +void replace_free(void* aPtr) { using namespace mozilla::dmd; + if (!gIsDMDInitialized) { + gMallocTable.free(aPtr); + return; + } + Thread* t = Thread::Fetch(); if (t->InterceptsAreBlocked()) { return InfallibleAllocPolicy::free_(aPtr); @@ -1365,17 +1403,6 @@ replace_free(void* aPtr) gMallocTable.free(aPtr); } -void -replace_init(malloc_table_t* aMallocTable, ReplaceMallocBridge** aBridge) -{ - if (mozilla::dmd::Init(aMallocTable)) { -#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC_BASE -#define MALLOC_DECL(name, ...) aMallocTable->name = replace_ ## name; -#include "malloc_decls.h" - *aBridge = mozilla::dmd::gDMDBridge; - } -} - namespace mozilla { namespace dmd { @@ -1440,6 +1467,9 @@ Options::Options(const char* aDMDEnvVar) , mStacks(Stacks::Partial) , mShowDumpStats(false) { + // It's no longer necessary to set the DMD env var to "1" if you want default + // options (you can leave it undefined) but we still accept "1" for + // backwards compatibility. char* e = mDMDEnvVar; if (e && strcmp(e, "1") != 0) { bool isEnd = false; @@ -1549,21 +1579,10 @@ postfork() // have run. For this reason, non-scalar globals such as gStateLock and // gStackTraceTable are allocated dynamically (so we can guarantee their // construction in this function) rather than statically. -static bool +static void Init(malloc_table_t* aMallocTable) { - // DMD is controlled by the |DMD| environment variable. - const char* e = getenv("DMD"); - - if (!e) { - return false; - } - // Initialize the function table first, because StatusMsg uses - // InfallibleAllocPolicy::malloc_, which uses it. gMallocTable = *aMallocTable; - - StatusMsg("$DMD = '%s'\n", e); - gDMDBridge = InfallibleAllocPolicy::new_(); #ifndef XP_WIN @@ -1575,6 +1594,16 @@ Init(malloc_table_t* aMallocTable) // system malloc a chance to insert its own atfork handler. pthread_atfork(prefork, postfork, postfork); #endif + + // DMD is controlled by the |DMD| environment variable. + const char* e = getenv("DMD"); + + if (e) { + StatusMsg("$DMD = '%s'\n", e); + } else { + StatusMsg("$DMD is undefined\n"); + } + // Parse $DMD env var. gOptions = InfallibleAllocPolicy::new_(e); @@ -1603,7 +1632,7 @@ Init(malloc_table_t* aMallocTable) MOZ_ALWAYS_TRUE(gDeadBlockTable->init(tableSize)); } - return true; + gIsDMDInitialized = true; } //--------------------------------------------------------------------------- diff --git a/memory/replace/dmd/moz.build b/memory/replace/dmd/moz.build index 2df4e4751079..a1492debd633 100644 --- a/memory/replace/dmd/moz.build +++ b/memory/replace/dmd/moz.build @@ -9,18 +9,14 @@ EXPORTS += [ ] UNIFIED_SOURCES += [ + '../../../mfbt/HashFunctions.cpp', + '../../../mfbt/JSONWriter.cpp', + '../../../mfbt/Poison.cpp', + '../../../mozglue/misc/StackWalk.cpp', 'DMD.cpp', ] -if not CONFIG['MOZ_REPLACE_MALLOC_STATIC']: - UNIFIED_SOURCES += [ - '/mfbt/HashFunctions.cpp', - '/mfbt/JSONWriter.cpp', - '/mfbt/Poison.cpp', - '/mozglue/misc/StackWalk.cpp', - ] - -ReplaceMalloc('dmd') +SharedLibrary('dmd') DEFINES['MOZ_NO_MOZALLOC'] = True DEFINES['IMPL_MFBT'] = True diff --git a/memory/replace/dmd/test/script-diff-dark-matter-expected.txt b/memory/replace/dmd/test/script-diff-dark-matter-expected.txt index b1fc28bac542..382f4eee593b 100644 --- a/memory/replace/dmd/test/script-diff-dark-matter-expected.txt +++ b/memory/replace/dmd/test/script-diff-dark-matter-expected.txt @@ -7,7 +7,7 @@ Invocation 1 { } Invocation 2 { - $DMD = '1' + $DMD is undefined Mode = 'dark-matter' } diff --git a/memory/replace/dmd/test/script-diff-dark-matter2.json b/memory/replace/dmd/test/script-diff-dark-matter2.json index 94b8888b89f6..a001040c024a 100644 --- a/memory/replace/dmd/test/script-diff-dark-matter2.json +++ b/memory/replace/dmd/test/script-diff-dark-matter2.json @@ -1,7 +1,7 @@ { "version": 5, "invocation": { - "dmdEnvVar": "1", + "dmdEnvVar": null, "mode": "dark-matter" }, "blockList": [ diff --git a/memory/replace/dmd/test/script-ignore-alloc-fns-expected.txt b/memory/replace/dmd/test/script-ignore-alloc-fns-expected.txt index 9428ef45fb8a..af9a0f6e92d5 100644 --- a/memory/replace/dmd/test/script-ignore-alloc-fns-expected.txt +++ b/memory/replace/dmd/test/script-ignore-alloc-fns-expected.txt @@ -2,7 +2,7 @@ # dmd.py --filter-stacks-for-testing -o script-ignore-alloc-fns-actual.txt --ignore-alloc-fns script-ignore-alloc-fns.json Invocation { - $DMD = '1' + $DMD is undefined Mode = 'dark-matter' } diff --git a/memory/replace/dmd/test/script-ignore-alloc-fns.json b/memory/replace/dmd/test/script-ignore-alloc-fns.json index 900d33e7590f..7e9446a7891b 100644 --- a/memory/replace/dmd/test/script-ignore-alloc-fns.json +++ b/memory/replace/dmd/test/script-ignore-alloc-fns.json @@ -1,7 +1,7 @@ { "version": 5, "invocation": { - "dmdEnvVar": "1", + "dmdEnvVar": null, "mode": "dark-matter" }, "blockList": [ diff --git a/memory/replace/dmd/test/test_dmd.js b/memory/replace/dmd/test/test_dmd.js index 1efdb7333374..8113e7749321 100644 --- a/memory/replace/dmd/test/test_dmd.js +++ b/memory/replace/dmd/test/test_dmd.js @@ -136,7 +136,7 @@ function run_test() { // in-place (to fix stacks) when it runs dmd.py, and that's not safe to do // asynchronously. - gEnv.set('DMD', '1'); + gEnv.set(gEnv.get("DMD_PRELOAD_VAR"), gEnv.get("DMD_PRELOAD_VALUE")); runProcess(gDmdTestFile, []); diff --git a/memory/replace/logalloc/LogAlloc.cpp b/memory/replace/logalloc/LogAlloc.cpp index c126588b7cf7..15eb1e0b2276 100644 --- a/memory/replace/logalloc/LogAlloc.cpp +++ b/memory/replace/logalloc/LogAlloc.cpp @@ -73,118 +73,55 @@ class LogAllocBridge : public ReplaceMallocBridge } }; -/* Do a simple, text-form, log of all calls to replace-malloc functions. - * Use locking to guarantee that an allocation that did happen is logged - * before any other allocation/free happens. - */ - -static void* -replace_malloc(size_t aSize) -{ - AutoLock lock(sLock); - void* ptr = sFuncs.malloc(aSize); - if (ptr) { - FdPrintf(sFd, "%zu %zu malloc(%zu)=%p\n", GetPid(), GetTid(), aSize, ptr); - } - return ptr; -} - -#ifndef LOGALLOC_MINIMAL -static int -replace_posix_memalign(void** aPtr, size_t aAlignment, size_t aSize) -{ - AutoLock lock(sLock); - int ret = sFuncs.posix_memalign(aPtr, aAlignment, aSize); - if (ret == 0) { - FdPrintf(sFd, "%zu %zu posix_memalign(%zu,%zu)=%p\n", GetPid(), GetTid(), - aAlignment, aSize, *aPtr); - } - return ret; -} - -static void* -replace_aligned_alloc(size_t aAlignment, size_t aSize) -{ - AutoLock lock(sLock); - void* ptr = sFuncs.aligned_alloc(aAlignment, aSize); - if (ptr) { - FdPrintf(sFd, "%zu %zu aligned_alloc(%zu,%zu)=%p\n", GetPid(), GetTid(), - aAlignment, aSize, ptr); - } - return ptr; -} -#endif - -static void* -replace_calloc(size_t aNum, size_t aSize) -{ - AutoLock lock(sLock); - void* ptr = sFuncs.calloc(aNum, aSize); - if (ptr) { - FdPrintf(sFd, "%zu %zu calloc(%zu,%zu)=%p\n", GetPid(), GetTid(), aNum, - aSize, ptr); - } - return ptr; -} - -static void* -replace_realloc(void* aPtr, size_t aSize) -{ - AutoLock lock(sLock); - void* new_ptr = sFuncs.realloc(aPtr, aSize); - if (new_ptr || !aSize) { - FdPrintf(sFd, "%zu %zu realloc(%p,%zu)=%p\n", GetPid(), GetTid(), aPtr, - aSize, new_ptr); - } - return new_ptr; -} - -static void -replace_free(void* aPtr) -{ - AutoLock lock(sLock); - if (aPtr) { - FdPrintf(sFd, "%zu %zu free(%p)\n", GetPid(), GetTid(), aPtr); - } - sFuncs.free(aPtr); -} - -static void* -replace_memalign(size_t aAlignment, size_t aSize) -{ - AutoLock lock(sLock); - void* ptr = sFuncs.memalign(aAlignment, aSize); - if (ptr) { - FdPrintf(sFd, "%zu %zu memalign(%zu,%zu)=%p\n", GetPid(), GetTid(), - aAlignment, aSize, ptr); - } - return ptr; -} - -#ifndef LOGALLOC_MINIMAL -static void* -replace_valloc(size_t aSize) -{ - AutoLock lock(sLock); - void* ptr = sFuncs.valloc(aSize); - if (ptr) { - FdPrintf(sFd, "%zu %zu valloc(%zu)=%p\n", GetPid(), GetTid(), aSize, ptr); - } - return ptr; -} -#endif - -static void -replace_jemalloc_stats(jemalloc_stats_t* aStats) -{ - AutoLock lock(sLock); - sFuncs.jemalloc_stats(aStats); - FdPrintf(sFd, "%zu %zu jemalloc_stats()\n", GetPid(), GetTid()); -} - void replace_init(malloc_table_t* aTable, ReplaceMallocBridge** aBridge) { + static LogAllocBridge bridge; + sFuncs = *aTable; +#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC_BASE +#define MALLOC_DECL(name, ...) aTable->name = replace_ ## name; +#include "malloc_decls.h" + aTable->jemalloc_stats = replace_jemalloc_stats; +#ifndef LOGALLOC_MINIMAL + aTable->posix_memalign = replace_posix_memalign; + aTable->aligned_alloc = replace_aligned_alloc; + aTable->valloc = replace_valloc; +#endif + *aBridge = &bridge; + +#ifndef _WIN32 + /* When another thread has acquired a lock before forking, the child + * process will inherit the lock state but the thread, being nonexistent + * in the child process, will never release it, leading to a dead-lock + * whenever the child process gets the lock. We thus need to ensure no + * other thread is holding the lock before forking, by acquiring it + * ourselves, and releasing it after forking, both in the parent and child + * processes. + * Windows doesn't have this problem since there is no fork(). + * The real allocator, however, might be doing the same thing (jemalloc + * does). But pthread_atfork `prepare` handlers (first argument) are + * processed in reverse order they were established. But replace_init + * runs before the real allocator has had any chance to initialize and + * call pthread_atfork itself. This leads to its prefork running before + * ours. This leads to a race condition that can lead to a deadlock like + * the following: + * - thread A forks. + * - libc calls real allocator's prefork, so thread A holds the real + * allocator lock. + * - thread B calls malloc, which calls our replace_malloc. + * - consequently, thread B holds our lock. + * - thread B then proceeds to call the real allocator's malloc, and + * waits for the real allocator's lock, which thread A holds. + * - libc calls our prefork, so thread A waits for our lock, which + * thread B holds. + * To avoid this race condition, the real allocator's prefork must be + * called after ours, which means it needs to be registered before ours. + * So trick the real allocator into initializing itself without more side + * effects by calling malloc with a size it can't possibly allocate. */ + sFuncs.malloc(-1); + pthread_atfork(prefork, postfork, postfork); +#endif + /* Initialize output file descriptor from the MALLOC_LOG environment * variable. Numbers up to 9999 are considered as a preopened file * descriptor number. Other values are considered as a file name. */ @@ -231,55 +168,113 @@ replace_init(malloc_table_t* aTable, ReplaceMallocBridge** aBridge) } #endif } - - // Don't initialize if we weren't passed a valid MALLOC_LOG. - if (sFd == 0) { - return; - } - - static LogAllocBridge bridge; - sFuncs = *aTable; -#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC_BASE -#define MALLOC_DECL(name, ...) aTable->name = replace_ ## name; -#include "malloc_decls.h" - aTable->jemalloc_stats = replace_jemalloc_stats; -#ifndef LOGALLOC_MINIMAL - aTable->posix_memalign = replace_posix_memalign; - aTable->aligned_alloc = replace_aligned_alloc; - aTable->valloc = replace_valloc; -#endif - *aBridge = &bridge; - -#ifndef _WIN32 - /* When another thread has acquired a lock before forking, the child - * process will inherit the lock state but the thread, being nonexistent - * in the child process, will never release it, leading to a dead-lock - * whenever the child process gets the lock. We thus need to ensure no - * other thread is holding the lock before forking, by acquiring it - * ourselves, and releasing it after forking, both in the parent and child - * processes. - * Windows doesn't have this problem since there is no fork(). - * The real allocator, however, might be doing the same thing (jemalloc - * does). But pthread_atfork `prepare` handlers (first argument) are - * processed in reverse order they were established. But replace_init - * runs before the real allocator has had any chance to initialize and - * call pthread_atfork itself. This leads to its prefork running before - * ours. This leads to a race condition that can lead to a deadlock like - * the following: - * - thread A forks. - * - libc calls real allocator's prefork, so thread A holds the real - * allocator lock. - * - thread B calls malloc, which calls our replace_malloc. - * - consequently, thread B holds our lock. - * - thread B then proceeds to call the real allocator's malloc, and - * waits for the real allocator's lock, which thread A holds. - * - libc calls our prefork, so thread A waits for our lock, which - * thread B holds. - * To avoid this race condition, the real allocator's prefork must be - * called after ours, which means it needs to be registered before ours. - * So trick the real allocator into initializing itself without more side - * effects by calling malloc with a size it can't possibly allocate. */ - sFuncs.malloc(-1); - pthread_atfork(prefork, postfork, postfork); -#endif +} + +/* Do a simple, text-form, log of all calls to replace-malloc functions. + * Use locking to guarantee that an allocation that did happen is logged + * before any other allocation/free happens. + */ + +void* +replace_malloc(size_t aSize) +{ + AutoLock lock(sLock); + void* ptr = sFuncs.malloc(aSize); + if (ptr) { + FdPrintf(sFd, "%zu %zu malloc(%zu)=%p\n", GetPid(), GetTid(), aSize, ptr); + } + return ptr; +} + +#ifndef LOGALLOC_MINIMAL +int +replace_posix_memalign(void** aPtr, size_t aAlignment, size_t aSize) +{ + AutoLock lock(sLock); + int ret = sFuncs.posix_memalign(aPtr, aAlignment, aSize); + if (ret == 0) { + FdPrintf(sFd, "%zu %zu posix_memalign(%zu,%zu)=%p\n", GetPid(), GetTid(), + aAlignment, aSize, *aPtr); + } + return ret; +} + +void* +replace_aligned_alloc(size_t aAlignment, size_t aSize) +{ + AutoLock lock(sLock); + void* ptr = sFuncs.aligned_alloc(aAlignment, aSize); + if (ptr) { + FdPrintf(sFd, "%zu %zu aligned_alloc(%zu,%zu)=%p\n", GetPid(), GetTid(), + aAlignment, aSize, ptr); + } + return ptr; +} +#endif + +void* +replace_calloc(size_t aNum, size_t aSize) +{ + AutoLock lock(sLock); + void* ptr = sFuncs.calloc(aNum, aSize); + if (ptr) { + FdPrintf(sFd, "%zu %zu calloc(%zu,%zu)=%p\n", GetPid(), GetTid(), aNum, + aSize, ptr); + } + return ptr; +} + +void* +replace_realloc(void* aPtr, size_t aSize) +{ + AutoLock lock(sLock); + void* new_ptr = sFuncs.realloc(aPtr, aSize); + if (new_ptr || !aSize) { + FdPrintf(sFd, "%zu %zu realloc(%p,%zu)=%p\n", GetPid(), GetTid(), aPtr, + aSize, new_ptr); + } + return new_ptr; +} + +void +replace_free(void* aPtr) +{ + AutoLock lock(sLock); + if (aPtr) { + FdPrintf(sFd, "%zu %zu free(%p)\n", GetPid(), GetTid(), aPtr); + } + sFuncs.free(aPtr); +} + +void* +replace_memalign(size_t aAlignment, size_t aSize) +{ + AutoLock lock(sLock); + void* ptr = sFuncs.memalign(aAlignment, aSize); + if (ptr) { + FdPrintf(sFd, "%zu %zu memalign(%zu,%zu)=%p\n", GetPid(), GetTid(), + aAlignment, aSize, ptr); + } + return ptr; +} + +#ifndef LOGALLOC_MINIMAL +void* +replace_valloc(size_t aSize) +{ + AutoLock lock(sLock); + void* ptr = sFuncs.valloc(aSize); + if (ptr) { + FdPrintf(sFd, "%zu %zu valloc(%zu)=%p\n", GetPid(), GetTid(), aSize, ptr); + } + return ptr; +} +#endif + +void +replace_jemalloc_stats(jemalloc_stats_t* aStats) +{ + AutoLock lock(sLock); + sFuncs.jemalloc_stats(aStats); + FdPrintf(sFd, "%zu %zu jemalloc_stats()\n", GetPid(), GetTid()); } diff --git a/memory/replace/logalloc/README b/memory/replace/logalloc/README index c2e8cf66cef0..ce0d82be8930 100644 --- a/memory/replace/logalloc/README +++ b/memory/replace/logalloc/README @@ -5,8 +5,20 @@ Firefox's default memory allocator independently or through another replace-malloc library, allowing the testing of other allocators under the exact same workload. -To get an allocation log the following environment variable when starting -Firefox: +To get an allocation log the following environment variables need to be set +when starting Firefox: +- on Linux: + LD_PRELOAD=/path/to/liblogalloc.so +- on Mac OSX: + DYLD_INSERT_LIBRARIES=/path/to/liblogalloc.dylib +- on Windows: + MOZ_REPLACE_MALLOC_LIB=/path/to/logalloc.dll +- on Android: + MOZ_REPLACE_MALLOC_LIB=/path/to/liblogalloc.so + (see https://wiki.mozilla.org/Mobile/Fennec/Android#Arguments_and_Environment_Variables + for how to pass environment variables to Firefox for Android) + +- on all platforms: MALLOC_LOG=/path/to/log-file or MALLOC_LOG=number diff --git a/memory/replace/logalloc/logalloc.mozbuild b/memory/replace/logalloc/logalloc.mozbuild index 7af682eb8308..15d15556aaed 100644 --- a/memory/replace/logalloc/logalloc.mozbuild +++ b/memory/replace/logalloc/logalloc.mozbuild @@ -10,6 +10,7 @@ SOURCES += [ ] DisableStlWrapping() +USE_STATIC_LIBS = True NO_PGO = True DEFINES['MOZ_NO_MOZALLOC'] = True # Avoid Lock_impl code depending on mozilla::Logger. @@ -29,7 +30,7 @@ else: include('/ipc/chromium/chromium-config.mozbuild') # Android doesn't have pthread_atfork, but we have our own in mozglue. -if CONFIG['OS_TARGET'] == 'Android' and FORCE_SHARED_LIB: +if CONFIG['OS_TARGET'] == 'Android': USE_LIBS += [ 'mozglue', ] diff --git a/memory/replace/logalloc/moz.build b/memory/replace/logalloc/moz.build index 08e281085471..41f5fbb3bc11 100644 --- a/memory/replace/logalloc/moz.build +++ b/memory/replace/logalloc/moz.build @@ -4,7 +4,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -ReplaceMalloc('logalloc') +SharedLibrary('logalloc') include('logalloc.mozbuild') diff --git a/memory/replace/logalloc/replay/Makefile.in b/memory/replace/logalloc/replay/Makefile.in index 9c1e871b42a6..441e3ddf02bd 100644 --- a/memory/replace/logalloc/replay/Makefile.in +++ b/memory/replace/logalloc/replay/Makefile.in @@ -16,9 +16,7 @@ LOGALLOC_VAR = LD_PRELOAD endif endif -ifndef MOZ_REPLACE_MALLOC_STATIC LOGALLOC = $(LOGALLOC_VAR)=$(CURDIR)/../$(DLL_PREFIX)logalloc$(DLL_SUFFIX) -endif LOGALLOC_MINIMAL = $(LOGALLOC_VAR)=$(CURDIR)/../minimal/$(DLL_PREFIX)logalloc_minimal$(DLL_SUFFIX) expected_output.log: $(srcdir)/replay.log diff --git a/memory/replace/logalloc/replay/moz.build b/memory/replace/logalloc/replay/moz.build index 4764bc052914..6c4834a32020 100644 --- a/memory/replace/logalloc/replay/moz.build +++ b/memory/replace/logalloc/replay/moz.build @@ -7,24 +7,12 @@ Program('logalloc-replay') SOURCES += [ + '../FdPrintf.cpp', '/mfbt/Assertions.cpp', '/mfbt/Unused.cpp', 'Replay.cpp', ] -if CONFIG['MOZ_REPLACE_MALLOC_STATIC'] and CONFIG['MOZ_DMD']: - UNIFIED_SOURCES += [ - '/mfbt/HashFunctions.cpp', - '/mfbt/JSONWriter.cpp', - '/mfbt/Poison.cpp', - '/mozglue/misc/StackWalk.cpp', - ] - -if not CONFIG['MOZ_REPLACE_MALLOC_STATIC']: - SOURCES += [ - '../FdPrintf.cpp', - ] - LOCAL_INCLUDES += [ '..', ] diff --git a/memory/replace/moz.build b/memory/replace/moz.build index 4858b3379015..850574e86396 100644 --- a/memory/replace/moz.build +++ b/memory/replace/moz.build @@ -4,14 +4,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -@template -def ReplaceMalloc(name): - if CONFIG['MOZ_REPLACE_MALLOC_STATIC']: - DEFINES['MOZ_REPLACE_MALLOC_PREFIX'] = name.replace('-', '_') - FINAL_LIBRARY = 'memory' - else: - SharedLibrary(name) - DIRS += [ 'logalloc', ] diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index c862a02a56af..0a815014764a 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -920,10 +920,33 @@ class RunProgram(MachCommandBase): if show_dump_stats: dmd_params.append('--show-dump-stats=yes') + bin_dir = os.path.dirname(binpath) + lib_name = self.substs['DLL_PREFIX'] + 'dmd' + self.substs['DLL_SUFFIX'] + dmd_lib = os.path.join(bin_dir, lib_name) + if not os.path.exists(dmd_lib): + print("Please build with |--enable-dmd| to use DMD.") + return 1 + + env_vars = { + "Darwin": { + "DYLD_INSERT_LIBRARIES": dmd_lib, + "LD_LIBRARY_PATH": bin_dir, + }, + "Linux": { + "LD_PRELOAD": dmd_lib, + "LD_LIBRARY_PATH": bin_dir, + }, + "WINNT": { + "MOZ_REPLACE_MALLOC_LIB": dmd_lib, + }, + } + + arch = self.substs['OS_ARCH'] + if dmd_params: - extra_env['DMD'] = ' '.join(dmd_params) - else: - extra_env['DMD'] = '1' + env_vars[arch]["DMD"] = " ".join(dmd_params) + + extra_env.update(env_vars.get(arch, {})) return self.run_process(args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env) diff --git a/testing/awsy/mach_commands.py b/testing/awsy/mach_commands.py index 503cd3e6d366..d7a67b8e53cf 100644 --- a/testing/awsy/mach_commands.py +++ b/testing/awsy/mach_commands.py @@ -144,10 +144,32 @@ class MachCommands(MachCommandBase): # Setup DMD env vars if necessary. if kwargs['dmd']: - bin_dir = os.path.dirname(binary) + dmd_params = [] - if 'DMD' not in os.environ: - os.environ['DMD'] = '1' + bin_dir = os.path.dirname(binary) + lib_name = self.substs['DLL_PREFIX'] + 'dmd' + self.substs['DLL_SUFFIX'] + dmd_lib = os.path.join(bin_dir, lib_name) + if not os.path.exists(dmd_lib): + print("Please build with |--enable-dmd| to use DMD.") + return 1 + + env_vars = { + "Darwin": { + "DYLD_INSERT_LIBRARIES": dmd_lib, + "LD_LIBRARY_PATH": bin_dir, + }, + "Linux": { + "LD_PRELOAD": dmd_lib, + "LD_LIBRARY_PATH": bin_dir, + }, + "WINNT": { + "MOZ_REPLACE_MALLOC_LIB": dmd_lib, + }, + } + + arch = self.substs['OS_ARCH'] + for k, v in env_vars[arch].iteritems(): + os.environ[k] = v # Also add the bin dir to the python path so we can use dmd.py if bin_dir not in sys.path: diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py index 27422931d96a..2af78bdf83c5 100644 --- a/testing/mochitest/mochitest_options.py +++ b/testing/mochitest/mochitest_options.py @@ -434,6 +434,12 @@ class MochitestArguments(ArgumentContainer): "default": False, "help": "Run tests with DMD active.", }], + [["--dmd-path"], + {"default": None, + "dest": "dmdPath", + "help": "Specifies the path to the directory containing the shared library for DMD.", + "suppress": True, + }], [["--dump-output-directory"], {"default": None, "dest": "dumpOutputDirectory", @@ -695,6 +701,16 @@ class MochitestArguments(ArgumentContainer): if options.profilePath: options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) + if options.dmdPath: + options.dmdPath = self.get_full_path(options.dmdPath, parser.oldcwd) + + if options.dmd and not options.dmdPath: + if build_obj: + options.dmdPath = build_obj.bindir + else: + parser.error( + "could not find dmd libraries, specify them with --dmd-path") + if options.utilityPath: options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index 476dac70b758..1caaf886dff2 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -1622,6 +1622,7 @@ toolbar#nav-bar { xrePath=options.xrePath, env=env, debugger=debugger, + dmdPath=options.dmdPath, lsanPath=lsanPath, ubsanPath=ubsanPath) @@ -1633,9 +1634,6 @@ toolbar#nav-bar { if options.headless: browserEnv["MOZ_HEADLESS"] = '1' - if options.dmd: - browserEnv["DMD"] = os.environ.get('DMD', '1') - # These variables are necessary for correct application startup; change # via the commandline at your own risk. browserEnv["XPCOM_DEBUG_BREAK"] = "stack" diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index 5c121ff0c277..d7e7ce1a87b9 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -280,8 +280,6 @@ class MochiRemote(MochitestDesktop): browserEnv["MOZ_LOG_FILE"] = os.path.join( self.remoteMozLog, self.mozLogName) - if options.dmd: - browserEnv['DMD'] = '1' return browserEnv def runApp(self, *args, **kwargs): @@ -356,6 +354,13 @@ def run_test_harness(parser, options): mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot + if options.dmdPath: + dmdLibrary = "libdmd.so" + dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) + dm.removeFile(dmdPathOnDevice) + dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) + options.dmdPath = deviceRoot + options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] diff --git a/testing/mozbase/mozrunner/mozrunner/utils.py b/testing/mozbase/mozrunner/mozrunner/utils.py index 34253d738496..e02ff98da7b2 100755 --- a/testing/mozbase/mozrunner/mozrunner/utils.py +++ b/testing/mozbase/mozrunner/mozrunner/utils.py @@ -84,7 +84,7 @@ def _raw_log(): def test_environment(xrePath, env=None, crashreporter=True, debugger=False, - lsanPath=None, ubsanPath=None, log=None): + dmdPath=None, lsanPath=None, ubsanPath=None, log=None): """ populate OS environment variables for mochitest and reftests. @@ -101,6 +101,30 @@ def test_environment(xrePath, env=None, crashreporter=True, debugger=False, else: ldLibraryPath = xrePath + envVar = None + dmdLibrary = None + preloadEnvVar = None + if mozinfo.isUnix: + envVar = "LD_LIBRARY_PATH" + dmdLibrary = "libdmd.so" + preloadEnvVar = "LD_PRELOAD" + elif mozinfo.isMac: + envVar = "DYLD_LIBRARY_PATH" + dmdLibrary = "libdmd.dylib" + preloadEnvVar = "DYLD_INSERT_LIBRARIES" + elif mozinfo.isWin: + envVar = "PATH" + dmdLibrary = "dmd.dll" + preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" + if envVar: + envValue = ((env.get(envVar), str(ldLibraryPath)) + if mozinfo.isWin + else (ldLibraryPath, dmdPath, env.get(envVar))) + env[envVar] = os.path.pathsep.join([path for path in envValue if path]) + + if dmdPath and dmdLibrary and preloadEnvVar: + env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) + # crashreporter env['GNOME_DISABLE_CRASH_DIALOG'] = '1' env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1' diff --git a/testing/xpcshell/runxpcshelltests.py b/testing/xpcshell/runxpcshelltests.py index cfe238e50e7e..8aba8bfc396e 100755 --- a/testing/xpcshell/runxpcshelltests.py +++ b/testing/xpcshell/runxpcshelltests.py @@ -653,8 +653,23 @@ class XPCShellTestThread(Thread): self.complete_command = cmdH + cmdT + cmdI + args if self.test_object.get('dmd') == 'true': + if sys.platform.startswith('linux'): + preloadEnvVar = 'LD_PRELOAD' + libdmd = os.path.join(self.xrePath, 'libdmd.so') + elif sys.platform == 'osx' or sys.platform == 'darwin': + preloadEnvVar = 'DYLD_INSERT_LIBRARIES' + # self.xrePath is /Contents/Resources. + # We need /Contents/MacOS/libdmd.dylib. + contents_dir = os.path.dirname(self.xrePath) + libdmd = os.path.join(contents_dir, 'MacOS', 'libdmd.dylib') + elif sys.platform == 'win32': + preloadEnvVar = 'MOZ_REPLACE_MALLOC_LIB' + libdmd = os.path.join(self.xrePath, 'dmd.dll') + self.env['PYTHON'] = sys.executable self.env['BREAKPAD_SYMBOLS_PATH'] = self.symbolsPath + self.env['DMD_PRELOAD_VAR'] = preloadEnvVar + self.env['DMD_PRELOAD_VALUE'] = libdmd if self.test_object.get('subprocess') == 'true': self.env['PYTHON'] = sys.executable