Factor out libc code into a header. (#624)

* Factor out libc code into a header.

This pulls the main definitions of the various libc malloc functions
into a header for easier use and inclusion in other projects.

* Clang-tidy fixes.

* Clang-tidy fixes really.

* More code quality changes

* Minor fix

* Clangformat

src/snmalloc/ds_core/redblacktree.h

@@ -68,7 +68,9 @@ namespace snmalloc
 
   template<typename Rep>
   concept RBRep = //
-    RBRepTypes<Rep> && RBRepMethods<Rep> &&
+    RBRepTypes<Rep> //
+    && RBRepMethods<Rep> //
+    &&
     ConceptSame<decltype(Rep::null), std::add_const_t<typename Rep::Contents>>;
 #endif
 

src/snmalloc/mem/localalloc.h

@@ -184,6 +184,13 @@ namespace snmalloc
       }
 
       return check_init([&](CoreAlloc* core_alloc) {
+        if (size > bits::one_at_bit(bits::BITS - 1))
+        {
+          // Cannot allocate something that is more that half the size of the
+          // address space
+          errno = ENOMEM;
+          return capptr::Alloc<void>{nullptr};
+        }
         // Grab slab of correct size
         // Set remote as large allocator remote.
         auto [chunk, meta] = Config::Backend::alloc_chunk(
@@ -712,7 +719,7 @@ namespace snmalloc
         auto pm_size = sizeclass_full_to_size(pm_sc);
         snmalloc_check_client(
           mitigations(sanity_checks),
-          sc == pm_sc,
+          (sc == pm_sc) || (p == nullptr),
           "Dealloc rounded size mismatch: {} != {}",
           rsize,
           pm_size);

src/snmalloc/mem/sizeclasstable.h

@@ -478,6 +478,12 @@ namespace snmalloc
   {
     if (size > sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1))
     {
+      if (size > bits::one_at_bit(bits::BITS - 1))
+      {
+        // This size is too large, no rounding should occur as will result in a
+        // failed allocation later.
+        return size;
+      }
       return bits::next_pow2(size);
     }
     // If realloc(ptr, 0) returns nullptr, some consumers treat this as a

src/snmalloc/override/libc.h

@@ -0,0 +1,177 @@
+#pragma once
+
+#include "../global/global.h"
+
+#include <errno.h>
+#include <string.h>
+
+namespace snmalloc::libc
+{
+  SNMALLOC_SLOW_PATH inline void* set_error(int err = ENOMEM)
+  {
+    errno = err;
+    return nullptr;
+  }
+
+  SNMALLOC_SLOW_PATH inline int set_error_and_return(int err = ENOMEM)
+  {
+    errno = err;
+    return err;
+  }
+
+  inline void* __malloc_end_pointer(void* ptr)
+  {
+    return ThreadAlloc::get().external_pointer<OnePastEnd>(ptr);
+  }
+
+  SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size)
+  {
+    return ThreadAlloc::get().alloc(size);
+  }
+
+  SNMALLOC_FAST_PATH_INLINE void free(void* ptr)
+  {
+    ThreadAlloc::get().dealloc(ptr);
+  }
+
+  SNMALLOC_FAST_PATH_INLINE void free_sized(void* ptr, size_t size)
+  {
+    ThreadAlloc::get().dealloc(ptr, size);
+  }
+
+  SNMALLOC_FAST_PATH_INLINE void* calloc(size_t nmemb, size_t size)
+  {
+    bool overflow = false;
+    size_t sz = bits::umul(size, nmemb, overflow);
+    if (SNMALLOC_UNLIKELY(overflow))
+    {
+      return set_error();
+    }
+    return ThreadAlloc::get().alloc<ZeroMem::YesZero>(sz);
+  }
+
+  SNMALLOC_FAST_PATH_INLINE void* realloc(void* ptr, size_t size)
+  {
+    auto& a = ThreadAlloc::get();
+    size_t sz = a.alloc_size(ptr);
+    // Keep the current allocation if the given size is in the same sizeclass.
+    if (sz == round_size(size))
+    {
+#ifdef SNMALLOC_PASS_THROUGH
+      // snmallocs alignment guarantees can be broken by realloc in pass-through
+      // this is not exercised, by existing clients, but is tested.
+      if (pointer_align_up(ptr, natural_alignment(size)) == ptr)
+        return ptr;
+#else
+      return ptr;
+#endif
+    }
+
+    void* p = a.alloc(size);
+    if (SNMALLOC_LIKELY(p != nullptr))
+    {
+      sz = bits::min(size, sz);
+      // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
+      // otherwise.
+      if (SNMALLOC_UNLIKELY(sz != 0))
+        ::memcpy(p, ptr, sz);
+      a.dealloc(ptr);
+    }
+    else if (SNMALLOC_LIKELY(size == 0))
+    {
+      a.dealloc(ptr);
+    }
+    else
+    {
+      return set_error();
+    }
+    return p;
+  }
+
+  inline size_t malloc_usable_size(const void* ptr)
+  {
+    return ThreadAlloc::get().alloc_size(ptr);
+  }
+
+  inline void* reallocarray(void* ptr, size_t nmemb, size_t size)
+  {
+    bool overflow = false;
+    size_t sz = bits::umul(size, nmemb, overflow);
+    if (SNMALLOC_UNLIKELY(overflow))
+    {
+      return set_error();
+    }
+    return realloc(ptr, sz);
+  }
+
+  inline int reallocarr(void* ptr_, size_t nmemb, size_t size)
+  {
+    int err = errno;
+    auto& a = ThreadAlloc::get();
+    bool overflow = false;
+    size_t sz = bits::umul(size, nmemb, overflow);
+    if (SNMALLOC_UNLIKELY(sz == 0))
+    {
+      errno = err;
+      return 0;
+    }
+    if (SNMALLOC_UNLIKELY(overflow))
+    {
+      return set_error_and_return(EOVERFLOW);
+    }
+
+    void** ptr = reinterpret_cast<void**>(ptr_);
+    void* p = a.alloc(sz);
+    if (SNMALLOC_UNLIKELY(p == nullptr))
+    {
+      return set_error_and_return(ENOMEM);
+    }
+
+    sz = bits::min(sz, a.alloc_size(*ptr));
+
+    SNMALLOC_ASSUME(*ptr != nullptr || sz == 0);
+    // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
+    // otherwise.
+    if (SNMALLOC_UNLIKELY(sz != 0))
+      ::memcpy(p, *ptr, sz);
+    errno = err;
+    a.dealloc(*ptr);
+    *ptr = p;
+    return 0;
+  }
+
+  inline void* memalign(size_t alignment, size_t size)
+  {
+    if (SNMALLOC_UNLIKELY(
+          alignment < sizeof(uintptr_t) || !bits::is_pow2(alignment)))
+    {
+      return set_error(EINVAL);
+    }
+
+    return malloc(aligned_size(alignment, size));
+  }
+
+  inline void* aligned_alloc(size_t alignment, size_t size)
+  {
+    SNMALLOC_ASSERT((size % alignment) == 0);
+    return memalign(alignment, size);
+  }
+
+  inline int posix_memalign(void** memptr, size_t alignment, size_t size)
+  {
+    if (SNMALLOC_UNLIKELY(
+          (alignment < sizeof(uintptr_t) || !bits::is_pow2(alignment))))
+    {
+      return EINVAL;
+    }
+
+    void* p = memalign(alignment, size);
+    if (SNMALLOC_UNLIKELY(p == nullptr))
+    {
+      if (size != 0)
+        return ENOMEM;
+    }
+    *memptr = p;
+    return 0;
+  }
+} // namespace snmalloc::libc

src/snmalloc/override/malloc.cc

@@ -1,8 +1,6 @@
+#include "libc.h"
 #include "override.h"
 
-#include <errno.h>
-#include <string.h>
-
 using namespace snmalloc;
 
 #ifndef MALLOC_USABLE_SIZE_QUALIFIER
@@ -13,52 +11,34 @@ extern "C"
 {
   SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr)
   {
-    return ThreadAlloc::get().external_pointer<OnePastEnd>(ptr);
+    return snmalloc::libc::__malloc_end_pointer(ptr);
   }
 
   SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size)
   {
-    return ThreadAlloc::get().alloc(size);
+    return snmalloc::libc::malloc(size);
   }
 
   SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr)
   {
-    ThreadAlloc::get().dealloc(ptr);
+    snmalloc::libc::free(ptr);
   }
 
   SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(cfree)(void* ptr)
   {
-    ThreadAlloc::get().dealloc(ptr);
+    snmalloc::libc::free(ptr);
-  }
-
-  /**
-   * Clang was helpfully inlining the constant return value, and
-   * thus converting from a tail call to an ordinary call.
-   */
-  SNMALLOC_EXPORT inline void* snmalloc_not_allocated = nullptr;
-
-  static SNMALLOC_SLOW_PATH void* SNMALLOC_NAME_MANGLE(snmalloc_set_error)()
-  {
-    errno = ENOMEM;
-    return snmalloc_not_allocated;
   }
 
   SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size)
   {
-    bool overflow = false;
+    return snmalloc::libc::calloc(nmemb, size);
-    size_t sz = bits::umul(size, nmemb, overflow);
-    if (SNMALLOC_UNLIKELY(overflow))
-    {
-      return SNMALLOC_NAME_MANGLE(snmalloc_set_error)();
-    }
-    return ThreadAlloc::get().alloc<ZeroMem::YesZero>(sz);
   }
 
   SNMALLOC_EXPORT
   size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(
     MALLOC_USABLE_SIZE_QUALIFIER void* ptr)
   {
-    return ThreadAlloc::get().alloc_size(ptr);
+    return snmalloc::libc::malloc_usable_size(ptr);
   }
 
   SNMALLOC_EXPORT
@@ -69,162 +49,53 @@ extern "C"
 
   SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size)
   {
-    auto& a = ThreadAlloc::get();
+    return snmalloc::libc::realloc(ptr, size);
-    size_t sz = a.alloc_size(ptr);
-    // Keep the current allocation if the given size is in the same sizeclass.
-    if (sz == round_size(size))
-    {
-#ifdef SNMALLOC_PASS_THROUGH
-      // snmallocs alignment guarantees can be broken by realloc in pass-through
-      // this is not exercised, by existing clients, but is tested.
-      if (pointer_align_up(ptr, natural_alignment(size)) == ptr)
-        return ptr;
-#else
-      return ptr;
-#endif
-    }
-
-    if (size == (size_t)-1)
-    {
-      errno = ENOMEM;
-      return nullptr;
-    }
-
-    void* p = a.alloc(size);
-    if (SNMALLOC_LIKELY(p != nullptr))
-    {
-      sz = bits::min(size, sz);
-      // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
-      // otherwise.
-      if (sz != 0)
-        memcpy(p, ptr, sz);
-      a.dealloc(ptr);
-    }
-    else if (SNMALLOC_LIKELY(size == 0))
-    {
-      a.dealloc(ptr);
-    }
-    else
-    {
-      errno = ENOMEM;
-    }
-    return p;
   }
 
 #if !defined(SNMALLOC_NO_REALLOCARRAY)
   SNMALLOC_EXPORT void*
   SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size)
   {
-    bool overflow = false;
+    return snmalloc::libc::reallocarray(ptr, nmemb, size);
-    size_t sz = bits::umul(size, nmemb, overflow);
-    if (overflow)
-    {
-      errno = ENOMEM;
-      return nullptr;
-    }
-    return SNMALLOC_NAME_MANGLE(realloc)(ptr, sz);
   }
 #endif
 
 #if !defined(SNMALLOC_NO_REALLOCARR)
   SNMALLOC_EXPORT int
-  SNMALLOC_NAME_MANGLE(reallocarr)(void* ptr_, size_t nmemb, size_t size)
+  SNMALLOC_NAME_MANGLE(reallocarr)(void* ptr, size_t nmemb, size_t size)
   {
-    int err = errno;
+    return snmalloc::libc::reallocarr(ptr, nmemb, size);
-    auto& a = ThreadAlloc::get();
-    bool overflow = false;
-    size_t sz = bits::umul(size, nmemb, overflow);
-    if (sz == 0)
-    {
-      errno = err;
-      return 0;
-    }
-    if (overflow)
-    {
-      errno = err;
-      return EOVERFLOW;
-    }
-
-    void** ptr = reinterpret_cast<void**>(ptr_);
-    void* p = a.alloc(sz);
-    if (p == nullptr)
-    {
-      errno = ENOMEM;
-      return ENOMEM;
-    }
-
-    sz = bits::min(sz, a.alloc_size(*ptr));
-
-    SNMALLOC_ASSUME(*ptr != nullptr || sz == 0);
-    // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy
-    // otherwise.
-    if (sz != 0)
-      memcpy(p, *ptr, sz);
-    errno = err;
-    a.dealloc(*ptr);
-    *ptr = p;
-    return 0;
   }
 #endif
 
   SNMALLOC_EXPORT void*
   SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size)
   {
-    if ((alignment == 0) || (alignment == size_t(-1)))
+    return snmalloc::libc::memalign(alignment, size);
-    {
-      errno = EINVAL;
-      return nullptr;
-    }
-
-    if ((size + alignment) < size)
-    {
-      errno = ENOMEM;
-      return nullptr;
-    }
-
-    return SNMALLOC_NAME_MANGLE(malloc)(aligned_size(alignment, size));
   }
 
   SNMALLOC_EXPORT void*
   SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size)
   {
-    SNMALLOC_ASSERT((size % alignment) == 0);
+    return snmalloc::libc::memalign(alignment, size);
-    return SNMALLOC_NAME_MANGLE(memalign)(alignment, size);
   }
 
   SNMALLOC_EXPORT int SNMALLOC_NAME_MANGLE(posix_memalign)(
     void** memptr, size_t alignment, size_t size)
   {
-    if ((alignment < sizeof(uintptr_t) || ((alignment & (alignment - 1)) != 0)))
+    return snmalloc::libc::posix_memalign(memptr, alignment, size);
-    {
-      return EINVAL;
-    }
-
-    void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size);
-    if (SNMALLOC_UNLIKELY(p == nullptr))
-    {
-      if (size != 0)
-        return ENOMEM;
-    }
-    *memptr = p;
-    return 0;
   }
 
 #if !defined(__FreeBSD__) && !defined(__OpenBSD__)
   SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(valloc)(size_t size)
   {
-    return SNMALLOC_NAME_MANGLE(memalign)(OS_PAGE_SIZE, size);
+    return snmalloc::libc::memalign(OS_PAGE_SIZE, size);
   }
 #endif
 
   SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size)
   {
-    if (size == size_t(-1))
+    return snmalloc::libc::memalign(
-    {
-      errno = ENOMEM;
-      return nullptr;
-    }
-    return SNMALLOC_NAME_MANGLE(memalign)(
       OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1));
   }
 

src/snmalloc/override/new.cc

@@ -20,102 +20,96 @@ using namespace snmalloc;
 
 void* operator new(size_t size)
 {
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void* operator new[](size_t size)
 {
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void* operator new(size_t size, std::nothrow_t&)
 {
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void* operator new[](size_t size, std::nothrow_t&)
 {
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void operator delete(void* p) EXCEPTSPEC
 {
-  ThreadAlloc::get().dealloc(p);
+  snmalloc::libc::free(p);
 }
 
 void operator delete(void* p, size_t size) EXCEPTSPEC
 {
-  if (p == nullptr)
+  snmalloc::libc::free_sized(p, size);
-    return;
-  ThreadAlloc::get().dealloc(p, size);
 }
 
 void operator delete(void* p, std::nothrow_t&)
 {
-  ThreadAlloc::get().dealloc(p);
+  snmalloc::libc::free(p);
 }
 
 void operator delete[](void* p) EXCEPTSPEC
 {
-  ThreadAlloc::get().dealloc(p);
+  snmalloc::libc::free(p);
 }
 
 void operator delete[](void* p, size_t size) EXCEPTSPEC
 {
-  if (p == nullptr)
+  snmalloc::libc::free_sized(p, size);
-    return;
-  ThreadAlloc::get().dealloc(p, size);
 }
 
 void operator delete[](void* p, std::nothrow_t&)
 {
-  ThreadAlloc::get().dealloc(p);
+  snmalloc::libc::free(p);
 }
 
 void* operator new(size_t size, std::align_val_t val)
 {
   size = aligned_size(size_t(val), size);
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void* operator new[](size_t size, std::align_val_t val)
 {
   size = aligned_size(size_t(val), size);
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void* operator new(size_t size, std::align_val_t val, std::nothrow_t&)
 {
   size = aligned_size(size_t(val), size);
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void* operator new[](size_t size, std::align_val_t val, std::nothrow_t&)
 {
   size = aligned_size(size_t(val), size);
-  return ThreadAlloc::get().alloc(size);
+  return snmalloc::libc::malloc(size);
 }
 
 void operator delete(void* p, std::align_val_t) EXCEPTSPEC
 {
-  ThreadAlloc::get().dealloc(p);
+  snmalloc::libc::free(p);
 }
 
 void operator delete[](void* p, std::align_val_t) EXCEPTSPEC
 {
-  ThreadAlloc::get().dealloc(p);
+  snmalloc::libc::free(p);
 }
 
 void operator delete(void* p, size_t size, std::align_val_t val) EXCEPTSPEC
 {
   size = aligned_size(size_t(val), size);
-  ThreadAlloc::get().dealloc(p, size);
+  snmalloc::libc::free_sized(p, size);
 }
 
 void operator delete[](void* p, size_t size, std::align_val_t val) EXCEPTSPEC
 {
-  if (p == nullptr)
-    return;
   size = aligned_size(size_t(val), size);
-  ThreadAlloc::get().dealloc(p, size);
+  snmalloc::libc::free_sized(p, size);
 }

src/snmalloc/snmalloc_front.h

@@ -1 +1,2 @@
 #include "global/global.h"
+#include "override/libc.h"