Pull aligned allocation out of the PAL.

The PAL can now advertise that it supports aligned allocation.  If it
does not, then the memory provider will do the alignment for it.

This change still leaves the PAL responsible for systematic testing, but
it should now be much easier to lift that out.

src/mem/largealloc.h

@@ -54,7 +54,7 @@ namespace snmalloc
     std::pair<void*, size_t> reserve_block() noexcept
     {
       size_t size = SUPERSLAB_SIZE;
-      void* r = ((PAL*)this)->template reserve<false>(&size, SUPERSLAB_SIZE);
+      void* r = reserve<false>(&size, SUPERSLAB_SIZE);
 
       if (size < SUPERSLAB_SIZE)
         error("out of memory");
@@ -197,6 +197,40 @@ namespace snmalloc
       }
     }
 
+    template<bool committed>
+    void* reserve(size_t* size, size_t align) noexcept
+    {
+      if constexpr (pal_supports<AlignedAllocation>())
+      {
+        return PAL::template reserve<committed>(size, align);
+      }
+      else
+      {
+        size_t request = *size;
+        // Add align, so we can guarantee to provide at least size.
+        request += align;
+        // Alignment must be a power of 2.
+        assert(align == bits::next_pow2(align));
+
+        void* p = PAL::template reserve<committed>(&request);
+
+        *size = request;
+        uintptr_t p0 = (uintptr_t)p;
+        uintptr_t start = bits::align_up(p0, align);
+
+        if (start > (uintptr_t)p0)
+        {
+          uintptr_t end = bits::align_down(p0 + request, align);
+          *size = end - start;
+          PAL::notify_not_using(p, start - p0);
+          PAL::notify_not_using(
+            reinterpret_cast<void*>(end), (p0 + request) - end);
+          p = reinterpret_cast<void*>(start);
+        }
+        return p;
+      }
+    }
+
     ALWAYSINLINE void lazy_decommit_if_needed()
     {
 #ifdef TEST_LAZY_DECOMMIT

src/pal/pal.h

@@ -17,7 +17,17 @@ namespace snmalloc
      * `expensive_low_memory_check()` method that returns a `bool` indicating
      * whether low memory conditions are still in effect.
      */
-    LowMemoryNotification = (1 << 0)
+    LowMemoryNotification = (1 << 0),
+    /**
+     * This PAL natively supports allocation with a guaranteed alignment.  If
+     * this is not supported, then we will over-allocate and round the
+     * allocation.
+     *
+     * A PAL that does supports this must expose a `request()` method that takes
+     * a size and alignment.  A PAL that does *not* support it must expose a
+     * `request()` method that takes only a size.
+     */
+    AlignedAllocation = (1 << 1)
   };
 }
 

src/pal/pal_free_bsd_kernel.h

@@ -28,7 +28,7 @@ namespace snmalloc
      * Bitmap of PalFeatures flags indicating the optional features that this
      * PAL supports.
      */
-    static constexpr uint64_t pal_features = 0;
+    static constexpr uint64_t pal_features = AlignedAllocation;
     void error(const char* const str)
     {
       panic("snmalloc error: %s", str);

src/pal/pal_freebsd.h

@@ -18,7 +18,7 @@ namespace snmalloc
      * Bitmap of PalFeatures flags indicating the optional features that this
      * PAL supports.
      */
-    static constexpr uint64_t pal_features = 0;
+    static constexpr uint64_t pal_features = AlignedAllocation;
     static void error(const char* const str)
     {
       puts(str);

src/pal/pal_linux.h

@@ -61,17 +61,11 @@ namespace snmalloc
     }
 
     template<bool committed>
-    void* reserve(size_t* size, size_t align) noexcept
+    void* reserve(size_t* size) noexcept
     {
-      size_t request = *size;
-      // Add align, so we can guarantee to provide at least size.
-      request += align;
-      // Alignment must be a power of 2.
-      assert(align == bits::next_pow2(align));
-
       void* p = mmap(
         NULL,
-        request,
+        *size,
         PROT_READ | PROT_WRITE,
         MAP_PRIVATE | MAP_ANONYMOUS,
         -1,
@@ -79,18 +73,7 @@ namespace snmalloc
 
       if (p == MAP_FAILED)
         error("Out of memory");
-      *size = request;
-      uintptr_t p0 = (uintptr_t)p;
-      uintptr_t start = bits::align_up(p0, align);
 
-      if (start > (uintptr_t)p0)
-      {
-        uintptr_t end = bits::align_down(p0 + request, align);
-        *size = end - start;
-        munmap(p, start - p0);
-        munmap((void*)end, (p0 + request) - end);
-        p = (void*)start;
-      }
       return p;
     }
   };

src/pal/pal_open_enclave.h

@@ -18,7 +18,7 @@ namespace snmalloc
      * Bitmap of PalFeatures flags indicating the optional features that this
      * PAL supports.
      */
-    static constexpr uint64_t pal_features = 0;
+    static constexpr uint64_t pal_features = AlignedAllocation;
     static void error(const char* const str)
     {
       UNUSED(str);

src/pal/pal_windows.h

@@ -68,7 +68,11 @@ namespace snmalloc
      * Bitmap of PalFeatures flags indicating the optional features that this
      * PAL supports.  This PAL supports low-memory notifications.
      */
-    static constexpr uint64_t pal_features = LowMemoryNotification;
+    static constexpr uint64_t pal_features = LowMemoryNotification
+#  if defined(PLATFORM_HAS_VIRTUALALLOC2) || defined(USE_SYSTEMATIC_TESTING)
+      | AlignedAllocation
+#  endif
+      ;
     /**
      * Counter values for the number of times that a low-pressure notification
      * has been delivered.  Callers should compare this with a previous value
@@ -139,8 +143,6 @@ namespace snmalloc
       static size_t bump_ptr = (size_t)0x4000'0000'0000;
       return bump_ptr;
     }
-#  endif
-
     template<bool committed>
     void* reserve(size_t* size, size_t align) noexcept
     {
@@ -149,26 +151,6 @@ namespace snmalloc
       if (committed)
         flags |= MEM_COMMIT;
 
-#  ifdef PLATFORM_HAS_VIRTUALALLOC2
-      // If we're on Windows 10 or newer, we can use the VirtualAlloc2
-      // function.  The FromApp variant is useable by UWP applications and
-      // cannot allocate executable memory.
-      MEM_ADDRESS_REQUIREMENTS addressReqs = {0};
-      MEM_EXTENDED_PARAMETER param = {0};
-      addressReqs.Alignment = align;
-      param.Type = MemExtendedParameterAddressRequirements;
-      param.Pointer = &addressReqs;
-      return VirtualAlloc2FromApp(
-        nullptr, nullptr, *size, flags, PAGE_READWRITE, &param, 1);
-#  else
-      // Add align, so we can guarantee to provide at least size.
-      size_t request = *size + align;
-
-      // Alignment must be a power of 2.
-      assert(align == bits::next_pow2(align));
-
-      void* p;
-#    ifdef USE_SYSTEMATIC_TESTING
       size_t retries = 1000;
       do
       {
@@ -178,9 +160,6 @@ namespace snmalloc
         systematic_bump_ptr() += request;
         retries--;
       } while (p == nullptr && retries > 0);
-#    else
-      p = VirtualAlloc(nullptr, request, flags, PAGE_READWRITE);
-#    endif
 
       uintptr_t aligned_p = bits::align_up((size_t)p, align);
 
@@ -193,8 +172,51 @@ namespace snmalloc
       }
       *size = request;
       return p;
-#  endif
     }
+#  elif defined(PLATFORM_HAS_VIRTUALALLOC2)
+    template<bool committed>
+    void* reserve(size_t* size, size_t align) noexcept
+    {
+      DWORD flags = MEM_RESERVE;
+
+      if (committed)
+        flags |= MEM_COMMIT;
+
+      // Windows doesn't let you request memory less than 64KB aligned.  Most
+      // operating systems will simply give you something more aligned than you
+      // ask for, but Windows complains about invalid parameters.
+      const size_t min_align = 64 * 1024;
+      if (align < min_align)
+        align = min_align;
+
+      // If we're on Windows 10 or newer, we can use the VirtualAlloc2
+      // function.  The FromApp variant is useable by UWP applications and
+      // cannot allocate executable memory.
+      MEM_ADDRESS_REQUIREMENTS addressReqs = {0};
+      MEM_EXTENDED_PARAMETER param = {0};
+      addressReqs.Alignment = align;
+      param.Type = MemExtendedParameterAddressRequirements;
+      param.Pointer = &addressReqs;
+      void* ret = VirtualAlloc2FromApp(
+        nullptr, nullptr, *size, flags, PAGE_READWRITE, &param, 1);
+      if (ret == nullptr)
+      {
+        error("Failed to allocate memory\n");
+      }
+      return ret;
+    }
+#  else
+    template<bool committed>
+    void* reserve(size_t* size) noexcept
+    {
+      DWORD flags = MEM_RESERVE;
+
+      if (committed)
+        flags |= MEM_COMMIT;
+
+      return VirtualAlloc(nullptr, *size, flags, PAGE_READWRITE);
+    }
+#  endif
   };
   HEADER_GLOBAL std::atomic<uint64_t> PALWindows::pressure_epoch;
   HEADER_GLOBAL std::atomic<bool> PALWindows::registered_for_notifications;

src/test/func/fixed_region/fixed_region.cc

@@ -30,10 +30,10 @@ extern "C" void oe_abort()
 using namespace snmalloc;
 int main()
 {
-  DefaultPal pal;
+  MemoryProviderStateMixin<DefaultPal> mp;
 
   size_t size = 1ULL << 28;
-  oe_base = pal.reserve<true>(&size, 0);
+  oe_base = mp.reserve<true>(&size, 0);
   oe_end = (uint8_t*)oe_base + size;
   std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;
 
@@ -49,4 +49,4 @@ int main()
     if (oe_end < r1)
       abort();
   }
-}
+}

src/test/func/two_alloc_types/main.cc

@@ -35,10 +35,10 @@ extern "C" void enclave_free(void*);
 using namespace snmalloc;
 int main()
 {
-  DefaultPal pal;
+  MemoryProviderStateMixin<DefaultPal> mp;
 
   size_t size = 1ULL << 28;
-  oe_base = pal.reserve<true>(&size, 0);
+  oe_base = mp.reserve<true>(&size, 1);
   oe_end = (uint8_t*)oe_base + size;
   std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;