Bug 1356701 - Export unprefixed malloc and duplication functions on OSX. r=njn

Going through the system zone allocator for every call to realloc/free
on OSX is costly, because the zone allocator needs to first verify that
the allocations do belong to the allocator it invokes (which ends up
calling jemalloc's malloc_usable_size), which is unnecessary when we
expect the allocations to belong to jemalloc.

So, we export the malloc/realloc/free/etc. symbols from
libmozglue.dylib, such that libraries and programs linked against it
call directly into jemalloc instead of going through the system zone
allocator, effectively shortcutting the allocator verification.

The risk is that some things in Gecko try to realloc/free pointers it
got from system libraries, if those were allocated with a system zone
that is not jemalloc.

--HG--
extra : rebase_source : ee0b29e1275176f52e64f4648dfa7ce25d61292e

memory/build/mozmemory_wrap.c

@@ -68,7 +68,6 @@ mozmem_malloc_impl(_ZdaPvRKSt9nothrow_t)(void *ptr)
 #undef strndup
 #undef strdup
 
-#ifndef XP_DARWIN
 MOZ_MEMORY_API char *
 strndup_impl(const char *src, size_t len)
 {
@@ -86,7 +85,6 @@ strdup_impl(const char *src)
   size_t len = strlen(src);
   return strndup_impl(src, len);
 }
-#endif /* XP_DARWIN */
 
 #ifdef ANDROID
 #include <stdarg.h>

memory/build/mozmemory_wrap.h

@@ -53,11 +53,10 @@
  *
  * - On MacOSX, the system libc has a zone allocator, which allows us to
  *   hook custom malloc implementation functions without exporting them.
- *   The malloc implementation functions are all prefixed with "je_" and used
- *   this way from the custom zone allocator. They are not exported.
- *   Duplication functions are not included, since they will call the custom
- *   zone allocator anyways. Jemalloc-specific functions are also left
- *   unprefixed.
+ *   However, since we want things in Firefox to skip the system zone
+ *   allocator, the malloc implementation functions are all exported
+ *   unprefixed, as well as duplication functions.
+ *   Jemalloc-specific functions are also left unprefixed.
  *
  * - On Android and Gonk, all functions are left unprefixed. Additionally,
  *   C++ allocation functions (operator new/delete) are also exported and
@@ -134,7 +133,7 @@
 #    define mozmem_jemalloc_impl(a)   je_ ## a
 #  else
 #    define MOZ_JEMALLOC_API MOZ_EXTERN_C MFBT_API
-#    if (defined(XP_WIN) || defined(XP_DARWIN))
+#    if defined(XP_WIN)
 #      if defined(MOZ_REPLACE_MALLOC)
 #        define mozmem_malloc_impl(a)   a ## _impl
 #      else

memory/build/zone.c

@@ -94,6 +94,11 @@ extern void malloc_zone_unregister(malloc_zone_t *zone);
 
 extern malloc_zone_t *malloc_default_purgeable_zone(void);
 
+extern malloc_zone_t* malloc_zone_from_ptr(const void* ptr);
+
+extern void malloc_zone_free(malloc_zone_t* zone, void* ptr);
+
+extern void* malloc_zone_realloc(malloc_zone_t* zone, void* ptr, size_t size);
 
 /*
  * The following is a OSX zone allocator implementation.
@@ -126,7 +131,34 @@ zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
 {
   if (malloc_usable_size_impl(ptr))
     return realloc_impl(ptr, size);
-  return realloc(ptr, size);
+
+  // Sometimes, system libraries call malloc_zone_* functions with the wrong
+  // zone (e.g. CoreFoundation does). In that case, we need to find the real
+  // one. We can't call libSystem's realloc directly because we're exporting
+  // realloc from libmozglue and we'd pick that one, so we manually find the
+  // right zone and realloc with it.
+  malloc_zone_t* real_zone = malloc_zone_from_ptr(ptr);
+  // The system allocator crashes voluntarily by default when a pointer can't
+  // be traced back to a zone. Do the same.
+  MOZ_RELEASE_ASSERT(real_zone);
+  MOZ_RELEASE_ASSERT(real_zone != zone);
+  return malloc_zone_realloc(real_zone, ptr, size);
+}
+
+static void
+other_zone_free(malloc_zone_t* original_zone, void* ptr)
+{
+  // Sometimes, system libraries call malloc_zone_* functions with the wrong
+  // zone (e.g. CoreFoundation does). In that case, we need to find the real
+  // one. We can't call libSystem's free directly because we're exporting
+  // free from libmozglue and we'd pick that one, so we manually find the
+  // right zone and free with it.
+  malloc_zone_t* zone = malloc_zone_from_ptr(ptr);
+  // The system allocator crashes voluntarily by default when a pointer can't
+  // be traced back to a zone. Do the same.
+  MOZ_RELEASE_ASSERT(zone);
+  MOZ_RELEASE_ASSERT(zone != original_zone);
+  return malloc_zone_free(zone, ptr);
 }
 
 static void
@@ -136,7 +168,7 @@ zone_free(malloc_zone_t *zone, void *ptr)
     free_impl(ptr);
     return;
   }
-  free(ptr);
+  other_zone_free(zone, ptr);
 }
 
 static void
@@ -148,7 +180,7 @@ zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
     free_impl(ptr);
     return;
   }
-  free(ptr);
+  other_zone_free(zone, ptr);
 }
 
 static void *