Configurable client meta-data (#662)

This provide a way to configure snmalloc to provide per object meta-data that is out of band. This can be used to provide different mitigations on top of snmalloc, such as storing memory tags in a compressed form, or provide a miracle pointer like feature.

This also includes a couple of TSAN fixes as it wasn't fully on in CI.

.github/workflows/main.yml

@@ -224,15 +224,14 @@ jobs:
       matrix:
         # Build just release variant as Debug is too slow.
         build-type: [ Release ]
+        os: ["ubuntu-latest", "ubuntu-20.04"]
         include:
           - os: "ubuntu-latest"
-            continue-on-error: # Don't class as an error if this fails, until we have a more reliablity.
             variant: "libc++ (TSan + UBSan)"
             dependencies: "sudo apt install ninja-build"
             extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread"
           # Also test specifically with clang-10 (on ubuntu-20.04)
           - os: "ubuntu-20.04"
-            continue-on-error: # Don't class as an error if this fails, until we have a more reliablity.
             variant: "clang-10 libc++ (TSan + UBSan)"
             dependencies: "sudo apt install ninja-build"
             extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread"
@@ -452,7 +451,7 @@ jobs:
         git diff --exit-code
     - name: Run clang-tidy
       run: |
-        clang-tidy-15  src/snmalloc/override/malloc.cc  -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0
+        clang-tidy-15  src/snmalloc/override/malloc.cc  -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 -Isrc
         if [ -f tidy.fail ] ; then
           cat tidy.fail
           exit 1

CMakeLists.txt

@@ -331,6 +331,9 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
         if(SNMALLOC_SANITIZER)
           target_compile_options(${TESTNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer)
           target_link_libraries(${TESTNAME} -fsanitize=${SNMALLOC_SANITIZER})
+          if (${SNMALLOC_SANITIZER} MATCHES "thread")
+            target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED)
+          endif()
         endif()
 
         add_warning_flags(${TESTNAME})

src/snmalloc/backend/backend.h

@@ -23,9 +23,6 @@ namespace snmalloc
     using Pal = PAL;
     using SlabMetadata = typename PagemapEntry::SlabMetadata;
 
-    static constexpr size_t SizeofMetadata =
-      bits::next_pow2_const(sizeof(SlabMetadata));
-
   public:
     /**
      * Provide a block of meta-data with size and align.
@@ -90,13 +87,26 @@ namespace snmalloc
      *   (remote, sizeclass, slab_metadata)
      * where slab_metadata, is the second element of the pair return.
      */
-    static std::pair<capptr::Chunk<void>, SlabMetadata*>
-    alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras)
+    static std::pair<capptr::Chunk<void>, SlabMetadata*> alloc_chunk(
+      LocalState& local_state,
+      size_t size,
+      uintptr_t ras,
+      sizeclass_t sizeclass)
     {
       SNMALLOC_ASSERT(bits::is_pow2(size));
       SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
 
-      auto meta_cap = local_state.get_meta_range().alloc_range(SizeofMetadata);
+      // Calculate the extra bytes required to store the client meta-data.
+      size_t extra_bytes = SlabMetadata::get_extra_bytes(sizeclass);
+
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
+
+#ifdef SNMALLOC_TRACING
+      message<1024>(
+        "Allocating metadata of size: {} ({})", meta_size, extra_bytes);
+#endif
+
+      auto meta_cap = local_state.get_meta_range().alloc_range(meta_size);
 
       auto meta = meta_cap.template as_reinterpret<SlabMetadata>().unsafe_ptr();
 
@@ -113,7 +123,7 @@ namespace snmalloc
 #endif
       if (p == nullptr)
       {
-        local_state.get_meta_range().dealloc_range(meta_cap, SizeofMetadata);
+        local_state.get_meta_range().dealloc_range(meta_cap, meta_size);
         errno = ENOMEM;
 #ifdef SNMALLOC_TRACING
         message<1024>("Out of memory");
@@ -140,7 +150,8 @@ namespace snmalloc
       LocalState& local_state,
       SlabMetadata& slab_metadata,
       capptr::Alloc<void> alloc,
-      size_t size)
+      size_t size,
+      sizeclass_t sizeclass)
     {
       /*
        * The backend takes possession of these chunks now, by disassociating
@@ -167,8 +178,12 @@ namespace snmalloc
        */
       capptr::Arena<void> arena = Authmap::amplify(alloc);
 
+      // Calculate the extra bytes required to store the client meta-data.
+      size_t extra_bytes = SlabMetadata::get_extra_bytes(sizeclass);
+
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
       local_state.get_meta_range().dealloc_range(
-        capptr::Arena<void>::unsafe_from(&slab_metadata), SizeofMetadata);
+        capptr::Arena<void>::unsafe_from(&slab_metadata), meta_size);
 
       local_state.get_object_range()->dealloc_range(arena, size);
     }

src/snmalloc/backend/fixedglobalconfig.h

@@ -8,11 +8,14 @@ namespace snmalloc
   /**
    * A single fixed address range allocator configuration
    */
-  template<SNMALLOC_CONCEPT(IsPAL) PAL>
+  template<
+    SNMALLOC_CONCEPT(IsPAL) PAL,
+    typename ClientMetaDataProvider = NoClientMetaDataProvider>
   class FixedRangeConfig final : public CommonConfig
   {
   public:
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
+    using ClientMeta = ClientMetaDataProvider;
 
   private:
     using ConcretePagemap =

src/snmalloc/backend/globalconfig.h

@@ -1,13 +1,9 @@
 #pragma once
-// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own
-// definition of `snmalloc::Alloc` before including any files that include
-// `snmalloc.h` or consume the global allocation APIs.
-#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
 
-#  include "../backend_helpers/backend_helpers.h"
-#  include "backend.h"
-#  include "meta_protected_range.h"
-#  include "standard_range.h"
+#include "../backend_helpers/backend_helpers.h"
+#include "backend.h"
+#include "meta_protected_range.h"
+#include "standard_range.h"
 
 namespace snmalloc
 {
@@ -28,13 +24,16 @@ namespace snmalloc
    * The Configuration sets up a Pagemap for the backend to use, and the state
    * required to build new allocators (GlobalPoolState).
    */
-  class StandardConfig final : public CommonConfig
+  template<typename ClientMetaDataProvider = NoClientMetaDataProvider>
+  class StandardConfigClientMeta final : public CommonConfig
   {
-    using GlobalPoolState = PoolState<CoreAllocator<StandardConfig>>;
+    using GlobalPoolState = PoolState<
+      CoreAllocator<StandardConfigClientMeta<ClientMetaDataProvider>>>;
 
   public:
     using Pal = DefaultPal;
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
+    using ClientMeta = ClientMetaDataProvider;
 
   private:
     using ConcretePagemap =
@@ -98,9 +97,9 @@ namespace snmalloc
     SNMALLOC_SLOW_PATH static void ensure_init_slow()
     {
       FlagLock lock{initialisation_lock};
-#  ifdef SNMALLOC_TRACING
+#ifdef SNMALLOC_TRACING
       message<1024>("Run init_impl");
-#  endif
+#endif
 
       if (initialised)
         return;
@@ -162,10 +161,4 @@ namespace snmalloc
       snmalloc::register_clean_up();
     }
   };
-
-  /**
-   * Create allocator type for this configuration.
-   */
-  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfig>;
 } // namespace snmalloc
-#endif

src/snmalloc/backend_helpers/commonconfig.h

@@ -95,6 +95,39 @@ namespace snmalloc
     bool HasDomesticate = false;
   };
 
+  struct NoClientMetaDataProvider
+  {
+    using StorageType = Empty;
+    using DataRef = Empty&;
+
+    static size_t required_count(size_t)
+    {
+      return 1;
+    }
+
+    static DataRef get(StorageType* base, size_t)
+    {
+      return *base;
+    }
+  };
+
+  template<typename T>
+  struct ArrayClientMetaDataProvider
+  {
+    using StorageType = T;
+    using DataRef = T&;
+
+    static size_t required_count(size_t max_count)
+    {
+      return max_count;
+    }
+
+    static DataRef get(StorageType* base, size_t index)
+    {
+      return base[index];
+    }
+  };
+
   /**
    * Class containing definitions that are likely to be used by all except for
    * the most unusual back-end implementations.  This can be subclassed as a

src/snmalloc/backend_helpers/defaultpagemapentry.h

@@ -64,9 +64,14 @@ namespace snmalloc
     SNMALLOC_FAST_PATH DefaultPagemapEntryT() = default;
   };
 
-  class DefaultSlabMetadata : public FrontendSlabMetadata<DefaultSlabMetadata>
+  template<typename ClientMetaDataProvider>
+  class DefaultSlabMetadata : public FrontendSlabMetadata<
+                                DefaultSlabMetadata<ClientMetaDataProvider>,
+                                ClientMetaDataProvider>
   {};
 
-  using DefaultPagemapEntry = DefaultPagemapEntryT<DefaultSlabMetadata>;
+  template<typename ClientMetaDataProvider>
+  using DefaultPagemapEntry =
+    DefaultPagemapEntryT<DefaultSlabMetadata<ClientMetaDataProvider>>;
 
 } // namespace snmalloc

src/snmalloc/ds/mpmcstack.h

@@ -4,12 +4,6 @@
 #include "aba.h"
 #include "allocconfig.h"
 
-#if defined(__has_feature)
-#  if __has_feature(thread_sanitizer)
-#    define SNMALLOC_THREAD_SANITIZER_ENABLED
-#  endif
-#endif
-
 namespace snmalloc
 {
   template<class T, Construction c = RequiresInit>

src/snmalloc/ds/pagemap.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "../ds_core/ds_core.h"
+
 namespace snmalloc
 {
   /**
@@ -179,7 +181,13 @@ namespace snmalloc
       // Allocate a power of two extra to allow the placement of the
       // pagemap be difficult to guess if randomize_position set.
       size_t additional_size =
+#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED
+        // When running with TSAN we failed to allocate the very large range
+        // randomly
+        randomize_position ? bits::next_pow2(REQUIRED_SIZE) : 0;
+#else
         randomize_position ? bits::next_pow2(REQUIRED_SIZE) * 4 : 0;
+#endif
       size_t request_size = REQUIRED_SIZE + additional_size;
 
       auto new_body_untyped = PAL::reserve(request_size);

src/snmalloc/global/global.h

@@ -1,4 +1,5 @@
 #include "bounds_checks.h"
+#include "libc.h"
 #include "memcpy.h"
 #include "scopedalloc.h"
 #include "threadalloc.h"

src/snmalloc/global/libc.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "../global/global.h"
+#include "threadalloc.h"
 
 #include <errno.h>
 #include <string.h>
@@ -176,4 +176,17 @@ namespace snmalloc::libc
     *memptr = p;
     return 0;
   }
+
+  inline typename snmalloc::Alloc::Config::ClientMeta::DataRef
+  get_client_meta_data(void* p)
+  {
+    return ThreadAlloc::get().get_client_meta_data(p);
+  }
+
+  inline std::add_const_t<typename snmalloc::Alloc::Config::ClientMeta::DataRef>
+  get_client_meta_data_const(void* p)
+  {
+    return ThreadAlloc::get().get_client_meta_data_const(p);
+  }
+
 } // namespace snmalloc::libc

src/snmalloc/global/memcpy.h

@@ -1,5 +1,4 @@
 #pragma once
-#include "../backend/globalconfig.h"
 #include "bounds_checks.h"
 
 namespace snmalloc

src/snmalloc/global/scopedalloc.h

@@ -1,5 +1,4 @@
 #pragma once
-#include "../backend/globalconfig.h"
 
 /**
  * This header requires that Alloc has been defined.

src/snmalloc/global/threadalloc.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "../backend/globalconfig.h"
-
 #if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC)
 #  define SNMALLOC_THREAD_TEARDOWN_DEFINED
 #endif

src/snmalloc/mem/backend_concept.h

@@ -2,6 +2,7 @@
 
 #ifdef __cpp_concepts
 #  include "../ds/ds.h"
+#  include "sizeclasstable.h"
 
 #  include <cstddef>
 namespace snmalloc
@@ -97,9 +98,13 @@ namespace snmalloc
 
   template<typename LocalState, typename PagemapEntry, typename Backend>
   concept IsBackend =
-    requires(LocalState& local_state, size_t size, uintptr_t ras) {
+    requires(
+      LocalState& local_state,
+      size_t size,
+      uintptr_t ras,
+      sizeclass_t sizeclass) {
       {
-        Backend::alloc_chunk(local_state, size, ras)
+        Backend::alloc_chunk(local_state, size, ras, sizeclass)
         } -> ConceptSame<
           std::pair<capptr::Chunk<void>, typename Backend::SlabMetadata*>>;
     } &&
@@ -112,9 +117,11 @@ namespace snmalloc
       LocalState& local_state,
       typename Backend::SlabMetadata& slab_metadata,
       capptr::Alloc<void> alloc,
-      size_t size) {
+      size_t size,
+      sizeclass_t sizeclass) {
       {
-        Backend::dealloc_chunk(local_state, slab_metadata, alloc, size)
+        Backend::dealloc_chunk(
+          local_state, slab_metadata, alloc, size, sizeclass)
         } -> ConceptSame<void>;
     } &&
     requires(address_t p) {

src/snmalloc/mem/corealloc.h

@@ -368,7 +368,8 @@ namespace snmalloc
           get_backend_local_state(),
           *meta,
           start,
-          sizeclass_to_slab_size(sizeclass));
+          sizeclass_to_slab_size(sizeclass),
+          sizeclass_t::from_small_class(sizeclass));
       });
     }
 
@@ -401,7 +402,7 @@ namespace snmalloc
         meta->node.remove();
 
         Config::Backend::dealloc_chunk(
-          get_backend_local_state(), *meta, p, size);
+          get_backend_local_state(), *meta, p, size, entry.get_sizeclass());
 
         return;
       }
@@ -796,7 +797,8 @@ namespace snmalloc
         get_backend_local_state(),
         slab_size,
         PagemapEntry::encode(
-          public_state(), sizeclass_t::from_small_class(sizeclass)));
+          public_state(), sizeclass_t::from_small_class(sizeclass)),
+        sizeclass_t::from_small_class(sizeclass));
 
       if (slab == nullptr)
       {

src/snmalloc/mem/freelist.h

@@ -187,7 +187,7 @@ namespace snmalloc
                 signed_prev(address_cast(this), address_cast(n_tame), key));
             }
           }
-          Aal::prefetch(&(n_tame->next_object));
+          Aal::prefetch(n_tame.unsafe_ptr());
           return n_tame;
         }
 

src/snmalloc/mem/localalloc.h

@@ -197,7 +197,8 @@ namespace snmalloc
           core_alloc->get_backend_local_state(),
           large_size_to_chunk_size(size),
           PagemapEntry::encode(
-            core_alloc->public_state(), size_to_sizeclass_full(size)));
+            core_alloc->public_state(), size_to_sizeclass_full(size)),
+          size_to_sizeclass_full(size));
         // set up meta data so sizeclass is correct, and hence alloc size, and
         // external pointer.
 #ifdef SNMALLOC_TRACING
@@ -816,6 +817,57 @@ namespace snmalloc
       }
     }
 
+    /**
+     * @brief Get the client meta data for the snmalloc allocation covering this
+     * pointer.
+     */
+    typename Config::ClientMeta::DataRef get_client_meta_data(void* p)
+    {
+      const PagemapEntry& entry =
+        Config::Backend::template get_metaentry(address_cast(p));
+
+      size_t index = slab_index(entry.get_sizeclass(), address_cast(p));
+
+      auto* meta_slab = entry.get_slab_metadata();
+
+      if (SNMALLOC_UNLIKELY(entry.is_backend_owned()))
+      {
+        error("Cannot access meta-data for write for freed memory!");
+      }
+
+      if (SNMALLOC_UNLIKELY(meta_slab == nullptr))
+      {
+        error(
+          "Cannot access meta-data for non-snmalloc object in writable form!");
+      }
+
+      return meta_slab->get_meta_for_object(index);
+    }
+
+    /**
+     * @brief Get the client meta data for the snmalloc allocation covering this
+     * pointer.
+     */
+    std::add_const_t<typename Config::ClientMeta::DataRef>
+    get_client_meta_data_const(void* p)
+    {
+      const PagemapEntry& entry =
+        Config::Backend::template get_metaentry<true>(address_cast(p));
+
+      size_t index = slab_index(entry.get_sizeclass(), address_cast(p));
+
+      auto* meta_slab = entry.get_slab_metadata();
+
+      if (SNMALLOC_UNLIKELY(
+            (meta_slab == nullptr) || (entry.is_backend_owned())))
+      {
+        static typename Config::ClientMeta::StorageType null_meta_store{};
+        return Config::ClientMeta::get(&null_meta_store, 0);
+      }
+
+      return meta_slab->get_meta_for_object(index);
+    }
+
     /**
      * Returns the number of remaining bytes in an object.
      *

src/snmalloc/mem/metadata.h

@@ -368,21 +368,26 @@ namespace snmalloc
   class FrontendSlabMetadata_Trait
   {
   private:
-    template<typename BackendType>
+    template<typename BackendType, typename ClientMeta_>
     friend class FrontendSlabMetadata;
 
     // Can only be constructed by FrontendSlabMetadata
-    FrontendSlabMetadata_Trait() = default;
+    constexpr FrontendSlabMetadata_Trait() = default;
   };
 
   /**
    * The FrontendSlabMetadata represent the metadata associated with a single
    * slab.
    */
-  template<typename BackendType>
+  template<typename BackendType, typename ClientMeta_>
   class FrontendSlabMetadata : public FrontendSlabMetadata_Trait
   {
   public:
+    /**
+     * Type that encapsulates logic for accessing client meta-data.
+     */
+    using ClientMeta = ClientMeta_;
+
     /**
      * Used to link slab metadata together in various other data-structures.
      * This is used with `SeqSet` and so may actually hold a subclass of this
@@ -424,6 +429,13 @@ namespace snmalloc
      */
     bool large_ = false;
 
+    /**
+     * Stores client meta-data for this slab. This must be last element in the
+     * slab. The meta data will actually allocate multiple elements after this
+     * type, so that client_meta_[1] will work for the required meta-data size.
+     */
+    SNMALLOC_NO_UNIQUE_ADDRESS typename ClientMeta::StorageType client_meta_{};
+
     uint16_t& needed()
     {
       return needed_;
@@ -452,6 +464,9 @@ namespace snmalloc
       set_sleeping(sizeclass, 0);
 
       large_ = false;
+
+      new (&client_meta_)
+        typename ClientMeta::StorageType[get_client_storage_count(sizeclass)];
     }
 
     /**
@@ -469,6 +484,8 @@ namespace snmalloc
 
       // Jump to slow path on first deallocation.
       needed() = 1;
+
+      new (&client_meta_) typename ClientMeta::StorageType();
     }
 
     /**
@@ -583,6 +600,33 @@ namespace snmalloc
     {
       return address_cast(free_queue.read_head(0, key));
     }
+
+    typename ClientMeta::DataRef get_meta_for_object(size_t index)
+    {
+      return ClientMeta::get(&client_meta_, index);
+    }
+
+    static size_t get_client_storage_count(smallsizeclass_t sizeclass)
+    {
+      auto count = sizeclass_to_slab_object_count(sizeclass);
+      auto result = ClientMeta::required_count(count);
+      if (result == 0)
+        return 1;
+      return result;
+    }
+
+    static size_t get_extra_bytes(sizeclass_t sizeclass)
+    {
+      if (sizeclass.is_small())
+        // We remove one from the extra-bytes as there is one in the metadata to
+        // start with.
+        return (get_client_storage_count(sizeclass.as_small()) - 1) *
+          sizeof(typename ClientMeta::StorageType);
+
+      // For large classes there is only a single entry, so this is covered by
+      // the existing entry in the metaslab, and further bytes are not required.
+      return 0;
+    }
   };
 
   /**
@@ -646,7 +690,7 @@ namespace snmalloc
      */
     [[nodiscard]] SNMALLOC_FAST_PATH SlabMetadata* get_slab_metadata() const
     {
-      SNMALLOC_ASSERT(get_remote() != nullptr);
+      SNMALLOC_ASSERT(!is_backend_owned());
       return unsafe_from_uintptr<SlabMetadata>(meta & ~META_BOUNDARY_BIT);
     }
   };

src/snmalloc/mem/sizeclasstable.h

@@ -333,14 +333,11 @@ namespace snmalloc
       .capacity;
   }
 
-  constexpr address_t start_of_object(sizeclass_t sc, address_t addr)
+  SNMALLOC_FAST_PATH constexpr size_t slab_index(sizeclass_t sc, address_t addr)
   {
     auto meta = sizeclass_metadata.fast(sc);
-    address_t slab_start = addr & ~meta.slab_mask;
     size_t offset = addr & meta.slab_mask;
-    size_t size = meta.size;
-
-    if constexpr (sizeof(addr) >= 8)
+    if constexpr (sizeof(offset) >= 8)
     {
       // Only works for 64 bit multiplication, as the following will overflow in
       // 32bit.
@@ -351,17 +348,27 @@ namespace snmalloc
       // the slab_mask by making the `div_mult` zero. The link uses 128 bit
       // multiplication, we have shrunk the range of the calculation to remove
       // this dependency.
-      size_t offset_start = ((offset * meta.div_mult) >> DIV_MULT_SHIFT) * size;
-      return slab_start + offset_start;
+      size_t index = ((offset * meta.div_mult) >> DIV_MULT_SHIFT);
+      return index;
     }
     else
     {
+      size_t size = meta.size;
       if (size == 0)
         return 0;
-      return slab_start + (offset / size) * size;
+      return offset / size;
     }
   }
 
+  SNMALLOC_FAST_PATH constexpr address_t
+  start_of_object(sizeclass_t sc, address_t addr)
+  {
+    auto meta = sizeclass_metadata.fast(sc);
+    address_t slab_start = addr & ~meta.slab_mask;
+    size_t index = slab_index(sc, addr);
+    return slab_start + (index * meta.size);
+  }
+
   constexpr size_t index_in_object(sizeclass_t sc, address_t addr)
   {
     return addr - start_of_object(sc, addr);

src/snmalloc/override/malloc.cc

@@ -1,4 +1,3 @@
-#include "libc.h"
 #include "override.h"
 
 using namespace snmalloc;

src/snmalloc/override/new.cc

@@ -1,4 +1,4 @@
-#include "libc.h"
+#include "snmalloc/snmalloc.h"
 
 #ifdef _WIN32
 #  ifdef __clang__

src/snmalloc/override/override.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "../global/global.h"
+#include "snmalloc/snmalloc.h"
 
 #ifndef SNMALLOC_EXPORT
 #  define SNMALLOC_EXPORT

src/snmalloc/override/rust.cc

@@ -1,5 +1,5 @@
 #define SNMALLOC_NAME_MANGLE(a) sn_##a
-#include "malloc.cc"
+#include "snmalloc/snmalloc.h"
 
 #include <cstring>
 
@@ -48,6 +48,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)(
 extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)(
   size_t* current_memory_usage, size_t* peak_memory_usage)
 {
-  *current_memory_usage = StandardConfig::Backend::get_current_usage();
-  *peak_memory_usage = StandardConfig::Backend::get_peak_usage();
+  *current_memory_usage = Alloc::Config::Backend::get_current_usage();
+  *peak_memory_usage = Alloc::Config::Backend::get_peak_usage();
 }

src/snmalloc/pal/pal_posix.h

@@ -9,7 +9,6 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <stdio.h>
-#include <stdlib.h>
 #include <string.h>
 #include <strings.h>
 #include <sys/mman.h>

src/snmalloc/snmalloc.h

@@ -3,8 +3,22 @@
 // Core implementation of snmalloc independent of the configuration mode
 #include "snmalloc_core.h"
 
-// If the user has defined SNMALLOC_PROVIDE_OWN_CONFIG, this include does
-// nothing.  Otherwise, it provide a default configuration of snmalloc::Alloc.
+// Provides the global configuration for the snmalloc implementation.
 #include "backend/globalconfig.h"
+
+// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own
+// definition of `snmalloc::Alloc` before including any files that include
+// `snmalloc.h` or consume the global allocation APIs.
+#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
+namespace snmalloc
+{
+  /**
+   * Create allocator type for this configuration.
+   */
+  using Alloc = snmalloc::LocalAllocator<
+    snmalloc::StandardConfigClientMeta<NoClientMetaDataProvider>>;
+} // namespace snmalloc
+#endif
+
 // User facing API surface, needs to know what `Alloc` is.
 #include "snmalloc_front.h"

src/snmalloc/snmalloc_front.h

@@ -1,2 +1 @@
 #include "global/global.h"
-#include "override/libc.h"

src/test/func/client_meta/client_meta.cc

@@ -0,0 +1,68 @@
+/**
+ * This test performs a very simple use of the client_meta data feature in
+ * snmalloc.
+ */
+
+#include "test/setup.h"
+
+#include <iostream>
+#include <snmalloc/backend/globalconfig.h>
+#include <snmalloc/snmalloc_core.h>
+#include <vector>
+
+namespace snmalloc
+{
+  // Create an allocator that stores an std::atomic<size_t>> per allocation.
+  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfigClientMeta<
+    ArrayClientMetaDataProvider<std::atomic<size_t>>>>;
+}
+#define SNMALLOC_PROVIDE_OWN_CONFIG
+#include <snmalloc/snmalloc.h>
+
+int main()
+{
+#ifdef SNMALLOC_PASS_THROUGH
+  // This test does not make sense in pass-through
+  return 0;
+#else
+  // Allocate a bunch of objects, and store the index into the meta-data.
+  std::vector<void*> ptrs;
+  for (size_t i = 0; i < 10000; i++)
+  {
+    auto p = snmalloc::libc::malloc(1024);
+    auto& meta = snmalloc::libc::get_client_meta_data(p);
+    meta = i;
+    ptrs.push_back(p);
+    memset(p, (uint8_t)i, 1024);
+  }
+
+  // Check meta-data contains expected value, and that the memory contains
+  // the expected pattern.
+  for (size_t i = 0; i < 10000; i++)
+  {
+    auto p = ptrs[i];
+    auto& meta = snmalloc::libc::get_client_meta_data(p);
+    if (meta != i)
+    {
+      std::cout << "Failed at index " << i << std::endl;
+      abort();
+    }
+    for (size_t j = 0; j < 1024; j++)
+    {
+      if (reinterpret_cast<uint8_t*>(p)[j] != (uint8_t)i)
+      {
+        std::cout << "Failed at index " << i << " byte " << j << std::endl;
+        abort();
+      }
+    }
+    snmalloc::libc::free(p);
+  }
+
+  // Access in a read-only way meta-data associated with the stack.
+  // This would fail if it was accessed for write.
+  auto& meta = snmalloc::libc::get_client_meta_data_const(&ptrs);
+  std::cout << "meta for stack" << meta << std::endl;
+
+  return 0;
+#endif
+}

src/test/func/domestication/domestication.cc

@@ -23,7 +23,8 @@ namespace snmalloc
   {
   public:
     using Pal = DefaultPal;
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<NoClientMetaDataProvider>;
+    using ClientMeta = NoClientMetaDataProvider;
 
   private:
     using ConcretePagemap =

src/test/func/malloc/malloc.cc

@@ -375,6 +375,6 @@ int main(int argc, char** argv)
     our_malloc_usable_size(nullptr) == 0,
     "malloc_usable_size(nullptr) should be zero");
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
   return 0;
 }

src/test/func/memory/memory.cc

@@ -184,7 +184,7 @@ void test_calloc()
     alloc.dealloc(p, size);
   }
 
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 }
 
 void test_double_alloc()
@@ -229,7 +229,7 @@ void test_double_alloc()
       }
     }
   }
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 }
 
 void test_external_pointer()
@@ -275,7 +275,7 @@ void test_external_pointer()
     alloc.dealloc(p1, size);
   }
 
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 };
 
 void check_offset(void* base, void* interior)

src/test/func/miracle_ptr/miracle_ptr.cc

@@ -0,0 +1,203 @@
+/**
+ * This file demonstrates how the snmalloc library could be implemented to
+ * provide a miracle pointer like feature.  This is not a hardened
+ * implementation and is purely for illustrative purposes.
+ *
+ * Do not use as is.
+ */
+
+#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED
+int main()
+{
+  return 0;
+}
+#else
+
+#  include "test/setup.h"
+
+#  include <iostream>
+#  include <memory>
+#  include <snmalloc/backend/globalconfig.h>
+#  include <snmalloc/snmalloc_core.h>
+
+namespace snmalloc
+{
+  // Instantiate the allocator with a client meta data provider that uses an
+  // atomic size_t to store the reference count.
+  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfigClientMeta<
+    ArrayClientMetaDataProvider<std::atomic<size_t>>>>;
+}
+#  define SNMALLOC_PROVIDE_OWN_CONFIG
+#  include <snmalloc/snmalloc.h>
+
+SNMALLOC_SLOW_PATH void error(std::string msg)
+{
+  std::cout << msg << std::endl;
+  abort();
+}
+
+SNMALLOC_FAST_PATH_INLINE void check(bool b, std::string msg)
+{
+  if (SNMALLOC_UNLIKELY(!b))
+    error(msg);
+}
+
+namespace snmalloc::miracle
+{
+  // snmalloc meta-data representation
+  //   * 2n + 1:  Represents an object that has not been deallocated with n
+  //              additional references to it
+  //   * 2n    :  Represents a deallocated object that
+  //              has n additional references to it
+
+  inline void* malloc(size_t size)
+  {
+    auto p = snmalloc::libc::malloc(size);
+    if (SNMALLOC_UNLIKELY(p == nullptr))
+      return nullptr;
+
+    snmalloc::libc::get_client_meta_data(p) = 1;
+    return p;
+  }
+
+  inline void free(void* ptr)
+  {
+    if (ptr == nullptr)
+      return;
+
+    // TODO could build a check into this that it is the start of the object?
+    auto previous =
+      snmalloc::libc::get_client_meta_data(ptr).fetch_add((size_t)-1);
+
+    if (SNMALLOC_LIKELY(previous == 1))
+    {
+      std::cout << "Freeing " << ptr << std::endl;
+      snmalloc::libc::free(ptr);
+      return;
+    }
+
+    check((previous & 1) == 1, "Double free detected");
+
+    // We have additional references to this object.
+    // We should not free it.
+    // TOOD this assumes this is not an internal pointer.
+    memset(ptr, 0, snmalloc::libc::malloc_usable_size(ptr));
+  }
+
+  inline void acquire(void* p)
+  {
+    auto previous =
+      snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)2);
+
+    // Can we take new pointers to a deallocated object?
+    check((previous & 1) == 1, "Acquiring a deallocated object");
+  }
+
+  inline void release(void* p)
+  {
+    auto previous =
+      snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)-2);
+
+    if (previous > 2)
+      return;
+
+    check(previous == 2, "Releasing an object with insufficient references");
+
+    std::cout << "Freeing from release " << p << std::endl;
+    snmalloc::libc::free(p);
+  }
+
+  /**
+   * This class can be used to replace a raw pointer. It will automatically use
+   * the underlying backup reference counting design from the miracle pointer
+   * docs.
+   */
+  template<typename T>
+  class raw_ptr
+  {
+    T* p;
+
+  public:
+    raw_ptr() : p(nullptr) {}
+
+    raw_ptr(T* p) : p(p)
+    {
+      snmalloc::miracle::acquire(p);
+    }
+
+    T& operator*()
+    {
+      return *p;
+    }
+
+    ~raw_ptr()
+    {
+      if (p == nullptr)
+        return;
+      snmalloc::miracle::release(p);
+    }
+
+    raw_ptr(const raw_ptr& rp) : p(rp.p)
+    {
+      snmalloc::miracle::acquire(p);
+    }
+
+    raw_ptr& operator=(const raw_ptr& other)
+    {
+      p = other.p;
+      snmalloc::miracle::acquire(other.p);
+      return *this;
+    }
+
+    raw_ptr(raw_ptr&& other) : p(other.p)
+    {
+      other.p = nullptr;
+    }
+
+    raw_ptr& operator=(raw_ptr&& other)
+    {
+      p = other.p;
+      other.p = nullptr;
+      return *this;
+    }
+  };
+} // namespace snmalloc::miracle
+
+/**
+ * Overload new and delete to use the "miracle pointer" implementation.
+ */
+void* operator new(size_t size)
+{
+  return snmalloc::miracle::malloc(size);
+}
+
+void operator delete(void* p)
+{
+  snmalloc::miracle::free(p);
+}
+
+void operator delete(void* p, size_t)
+{
+  snmalloc::miracle::free(p);
+}
+
+int main()
+{
+#  ifndef SNMALLOC_PASS_THROUGH
+  snmalloc::miracle::raw_ptr<int> p;
+  {
+    auto up1 = std::make_unique<int>(41);
+    auto up = std::make_unique<int>(42);
+    auto up2 = std::make_unique<int>(40);
+    auto up3 = std::make_unique<int>(39);
+    p = up.get();
+    check(*p == 42, "Failed to set p");
+  }
+  // Still safe to access here.  The unique_ptr has been destroyed, but the
+  // raw_ptr has kept the memory live.
+  // Current implementation zeros the memory when the unique_ptr is destroyed.
+  check(*p == 0, "Failed to keep memory live");
+#  endif
+  return 0;
+}
+#endif

src/test/func/statistics/stats.cc

@@ -17,7 +17,7 @@ void debug_check_empty_1()
 
   auto r = a.alloc(size);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != false)
   {
     std::cout << "debug_check_empty failed to detect leaked memory:" << size
@@ -27,7 +27,7 @@ void debug_check_empty_1()
 
   a.dealloc(r);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != true)
   {
     std::cout << "debug_check_empty failed to say empty:" << size << std::endl;
@@ -36,7 +36,7 @@ void debug_check_empty_1()
 
   r = a.alloc(size);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != false)
   {
     std::cout << "debug_check_empty failed to detect leaked memory:" << size
@@ -46,7 +46,7 @@ void debug_check_empty_1()
 
   a.dealloc(r);
 
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
   if (result != true)
   {
     std::cout << "debug_check_empty failed to say empty:" << size << std::endl;
@@ -72,7 +72,7 @@ void debug_check_empty_2()
     }
     auto r = a.alloc(size);
     allocs.push_back(r);
-    snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+    snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
     if (result != false)
     {
       std::cout << "False empty after " << i << " allocations of " << size
@@ -88,7 +88,7 @@ void debug_check_empty_2()
     {
       std::cout << "." << std::flush;
     }
-    snmalloc::debug_check_empty<snmalloc::StandardConfig>(&result);
+    snmalloc::debug_check_empty<snmalloc::Alloc::Config>(&result);
     if (result != false)
     {
       std::cout << "False empty after " << i << " deallocations of " << size
@@ -98,7 +98,7 @@ void debug_check_empty_2()
     a.dealloc(allocs[i]);
   }
   std::cout << std::endl;
-  snmalloc::debug_check_empty<snmalloc::StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 }
 
 int main()

src/test/func/thread_alloc_external/thread_alloc_external.cc

@@ -12,7 +12,8 @@
 
 namespace snmalloc
 {
-  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfig>;
+  using Alloc = snmalloc::LocalAllocator<
+    snmalloc::StandardConfigClientMeta<NoClientMetaDataProvider>>;
 }
 
 using namespace snmalloc;

src/test/perf/contention/contention.cc

@@ -154,7 +154,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size)
   }
 
 #ifndef NDEBUG
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 #endif
 };
 

src/test/perf/external_pointer/externalpointer.cc

@@ -47,7 +47,7 @@ namespace test
       alloc.dealloc(objects[i]);
     }
 
-    snmalloc::debug_check_empty<StandardConfig>();
+    snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
   }
 
   void test_external_pointer(xoroshiro::p128r64& r)

src/test/perf/memcpy/memcpy.cc

@@ -1,5 +1,4 @@
-#include "snmalloc/global/memcpy.h"
-
+#include <snmalloc/snmalloc.h>
 #include <test/measuretime.h>
 #include <test/opt.h>
 #include <vector>

src/test/perf/singlethread/singlethread.cc

@@ -60,7 +60,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write)
     }
   }
 
-  snmalloc::debug_check_empty<StandardConfig>();
+  snmalloc::debug_check_empty<snmalloc::Alloc::Config>();
 }
 
 int main(int, char**)