This commit is contained in:
Amaury Chamayou 2020-07-09 16:40:00 +01:00 коммит произвёл GitHub
Родитель 30c6ebe522
Коммит 1909515480
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
53 изменённых файлов: 1218 добавлений и 548 удалений

76
3rdparty/snmalloc/CMakeLists.txt поставляемый
Просмотреть файл

@ -1,7 +1,13 @@
cmake_minimum_required(VERSION 3.8)
project(snmalloc C CXX)
if (NOT CMAKE_BUILD_TYPE)
message(STATUS "No build type selected, default to: Release")
set(CMAKE_BUILD_TYPE "Release")
endif()
include(CheckCXXCompilerFlag)
include(CheckCSourceCompiles)
option(USE_SNMALLOC_STATS "Track allocation stats" OFF)
option(SNMALLOC_CI_BUILD "Disable features not sensible for CI" OFF)
@ -9,8 +15,17 @@ option(USE_MEASURE "Measure performance with histograms" OFF)
option(EXPOSE_EXTERNAL_PAGEMAP "Expose the global pagemap" OFF)
option(EXPOSE_EXTERNAL_RESERVE "Expose an interface to reserve memory using the default memory provider" OFF)
option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF)
option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON)
option(SNMALLOC_QEMU_WORKAROUND "Disable using madvise(DONT_NEED) to zero memory on Linux" Off)
option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off)
set(CACHE_FRIENDLY_OFFSET OFF CACHE STRING "Base offset to place linked-list nodes.")
set(SNMALLOC_STATIC_LIBRARY_PREFIX "sn_" CACHE STRING "Static library function prefix")
CHECK_C_SOURCE_COMPILES("
#include <malloc.h>
size_t malloc_usable_size(const void* ptr) { return 0; }
int main() { return 0; }
" CONST_QUALIFIED_MALLOC_USABLE_SIZE)
if ((CMAKE_BUILD_TYPE STREQUAL "Release") AND (NOT SNMALLOC_CI_BUILD))
option(USE_POSIX_COMMIT_CHECKS "Instrument Posix PAL to check for access to unused blocks of memory." Off)
@ -36,6 +51,10 @@ macro(warnings_high)
endif()
endmacro()
macro(oe_simulate target)
target_compile_definitions(${target} PRIVATE SNMALLOC_USE_SMALL_CHUNKS)
endmacro()
macro(clangformat_targets)
# The clang-format tool is installed under a variety of different names. Try
# to find a sensible one. Only look for versions 9 explicitly - we don't
@ -43,8 +62,7 @@ macro(clangformat_targets)
# tool. It does not work with older versions as AfterCaseLabel is not supported
# in earlier versions.
find_program(CLANG_FORMAT NAMES
clang-format-9
clang-format)
clang-format-9)
# If we've found a clang-format tool, generate a target for it, otherwise emit
# a warning.
@ -135,6 +153,10 @@ if(USE_POSIX_COMMIT_CHECKS)
target_compile_definitions(snmalloc_lib INTERFACE -DUSE_POSIX_COMMIT_CHECKS)
endif()
if(CONST_QUALIFIED_MALLOC_USABLE_SIZE)
target_compile_definitions(snmalloc_lib INTERFACE -DMALLOC_USABLE_SIZE_QUALIFIER=const)
endif()
# To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY
# in containing Cmake file.
@ -146,20 +168,32 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi")
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG")
else()
add_compile_options(-fno-exceptions -fno-rtti -g -ftls-model=initial-exec -fomit-frame-pointer)
add_compile_options(-fno-exceptions -fno-rtti -g -fomit-frame-pointer)
# Static TLS model unsupported on Haiku
if (NOT CMAKE_SYSTEM_NAME MATCHES "Haiku")
add_compile_options(-ftls-model=initial-exec)
endif()
if(SNMALLOC_CI_BUILD OR (${CMAKE_BUILD_TYPE} MATCHES "Debug"))
# Get better stack traces in CI and Debug.
target_link_libraries(snmalloc_lib INTERFACE "-rdynamic")
endif()
if((${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") OR
(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86") OR
(${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm"))
if(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE)
check_cxx_compiler_flag(-march=native SUPPORT_MARCH_NATIVE)
if (SUPPORT_MARCH_NATIVE)
add_compile_options(-march=native)
else()
message(WARNING "Compiler does not support `-march=native` required by SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE")
endif()
endif()
find_package(Backtrace)
if(${Backtrace_FOUND})
target_compile_definitions(snmalloc_lib INTERFACE -DBACKTRACE_HEADER="${Backtrace_HEADER}")
target_link_libraries(snmalloc_lib INTERFACE ${Backtrace_LIBRARIES})
target_include_directories(snmalloc_lib INTERFACE ${Backtrace_INCLUD_DIRS})
endif()
endif()
macro(subdirlist result curdir)
@ -204,17 +238,34 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
endmacro()
if (SNMALLOC_STATIC_LIBRARY)
add_shim(snmallocshim-static STATIC src/override/malloc.cc)
add_shim(snmallocshim-1mib-static STATIC src/override/malloc.cc)
add_shim(snmallocshim-16mib-static STATIC src/override/malloc.cc)
target_compile_definitions(snmallocshim-16mib-static PRIVATE SNMALLOC_USE_LARGE_CHUNKS
SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX})
target_compile_definitions(snmallocshim-static PRIVATE
SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX})
target_compile_definitions(snmallocshim-1mib-static PRIVATE
SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX})
endif ()
if(NOT WIN32)
set(SHARED_FILES src/override/new.cc src/override/malloc.cc)
add_shim(snmallocshim SHARED ${SHARED_FILES})
add_shim(snmallocshim-1mib SHARED ${SHARED_FILES})
target_compile_definitions(snmallocshim-1mib PRIVATE IS_ADDRESS_SPACE_CONSTRAINED)
add_shim(snmallocshim-16mib SHARED ${SHARED_FILES})
target_compile_definitions(snmallocshim-16mib PRIVATE SNMALOC_USE_LARGE_CHUNKS)
# Build a shim with some settings from oe.
add_shim(snmallocshim-oe SHARED ${SHARED_FILES})
oe_simulate(snmallocshim-oe)
endif()
if(SNMALLOC_RUST_SUPPORT)
add_shim(snmallocshim-rust STATIC src/override/rust.cc)
add_shim(snmallocshim-1mib-rust STATIC src/override/rust.cc)
target_compile_definitions(snmallocshim-1mib-rust PRIVATE IS_ADDRESS_SPACE_CONSTRAINED)
add_shim(snmallocshim-16mib-rust STATIC src/override/rust.cc)
target_compile_definitions(snmallocshim-16mib-rust PRIVATE SNMALLOC_USE_LARGE_CHUNKS)
endif()
enable_testing()
@ -225,14 +276,17 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)
foreach(TEST_CATEGORY ${TEST_CATEGORIES})
subdirlist(TESTS ${TESTDIR}/${TEST_CATEGORY})
foreach(TEST ${TESTS})
foreach(SUPER_SLAB_SIZE 1;16)
foreach(SUPER_SLAB_SIZE 1;16;oe)
unset(SRC)
aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC)
set(TESTNAME "${TEST_CATEGORY}-${TEST}-${SUPER_SLAB_SIZE}")
add_executable(${TESTNAME} ${SRC})
if (${SUPER_SLAB_SIZE} EQUAL 1)
target_compile_definitions(${TESTNAME} PRIVATE IS_ADDRESS_SPACE_CONSTRAINED)
if (${SUPER_SLAB_SIZE} EQUAL 16)
target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_USE_LARGE_CHUNKS)
endif()
if (${SUPER_SLAB_SIZE} EQUAL oe)
oe_simulate(${TESTNAME})
endif()
target_link_libraries(${TESTNAME} snmalloc_lib)
if (${TEST} MATCHES "release-.*")

42
3rdparty/snmalloc/README.md поставляемый
Просмотреть файл

@ -105,6 +105,16 @@ your toolchain:
LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja
```
## Cross Compile for Android
Android support is out-of-the-box.
To cross-compile the library for arm android, you can simply invoke CMake with the toolchain file and the andorid api settings (for more infomation, check this [document](https://developer.android.com/ndk/guides/cmake)).
For example, you can cross-compile for `arm64-v8a` with the following command:
```
cmake /path/to/snmalloc -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a
```
# CMake Feature Flags
These can be added to your cmake command line.
@ -150,7 +160,7 @@ your system.
The PAL must implement the following methods:
```c++
void error(const char* const str) noexcept;
[[noreturn]] void error(const char* const str) noexcept;
```
Report a fatal error and exit.
@ -184,16 +194,24 @@ pages, rather than zeroing them synchronously in this call
```c++
template<bool committed>
void* reserve(size_t size, size_t align);
template<bool committed>
void* reserve(size_t size) noexcept;
void* reserve_aligned(size_t size) noexcept;
std::pair<void*, size_t> reserve_at_least(size_t size) noexcept;
```
Only one of these needs to be implemented, depending on whether the underlying
system can provide strongly aligned memory regions.
If the system guarantees only page alignment, implement the second and snmalloc
will over-allocate and then trim the requested region.
If the system guarantees only page alignment, implement the second. The Pal is
free to overallocate based on the platforms desire and snmalloc
will find suitably aligned blocks inside the region. `reserve_at_least` should
not commit memory as snmalloc will commit the range of memory it requires of what
is returned.
If the system provides strong alignment, implement the first to return memory
at the desired alignment.
at the desired alignment. If providing the first, then the `Pal` should also
specify the minimum size block it can provide:
```
static constexpr size_t minimum_alloc_size = ...;
```
Finally, you need to define a field to indicate the features that your PAL supports:
```c++
@ -225,6 +243,16 @@ The [Windows](src/pal/pal_windows.h), and
[FreeBSD kernel](src/pal/pal_freebsd_kernel.h) implementations give examples of
non-POSIX environments that snmalloc supports.
The POSIX PAL uses `mmap` to map memory.
Some POSIX or POSIX-like systems require minor tweaks to this behaviour.
Rather than requiring these to copy and paste the code, a PAL that inherits from the POSIX PAL can define one or both of these (`static constexpr`) fields to customise the `mmap` behaviour.
- `default_mmap_flags` allows a PAL to provide additional `MAP_*`
flags to all `mmap` calls.
- `anonymous_memory_fd` allows the PAL to override the default file
descriptor used for memory mappings.
# Contributing
This project welcomes contributions and suggestions. Most contributions require you to agree to a

62
3rdparty/snmalloc/azure-pipelines.yml поставляемый
Просмотреть файл

@ -9,97 +9,123 @@ jobs:
displayName: Linux
pool:
vmImage: 'ubuntu-18.04'
container: snmallocciteam/build_linux_x64:latest
strategy:
matrix:
Clang-7 Debug:
64-bit Clang-7 Debug:
CC: clang-7
CXX: clang++-7
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
Clang-7 Release:
64-bit Clang-7 Release:
CC: clang-7
CXX: clang++-7
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
Clang-8 Debug:
64-bit Clang-8 Debug:
CC: clang-8
CXX: clang++-8
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
Clang-8 Release:
64-bit Clang-8 Release:
CC: clang-8
CXX: clang++-8
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
Clang-9 Debug:
64-bit Clang-9 Debug:
CC: clang-9
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
Clang-9 Release:
64-bit Clang-9 Release:
CC: clang-9
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
GCC-8 Debug:
64-bit GCC-8 Debug:
CC: gcc-8
CXX: g++-8
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
GCC-8 Release:
64-bit GCC-8 Release:
CC: gcc-8
CXX: g++-8
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
Self Host:
64-bit Self Host:
CC: clang-7
CXX: clang++-7
BuildType: Debug
SelfHost: true
CMakeArgs: ''
Image: snmallocciteam/build_linux_x64:latest
Cache Friendly:
64-bit Cache Friendly:
CC: clang-7
CXX: clang++-7
BuildType: Debug
SelfHost: false
CMakeArgs: '-DCACHE_FRIENDLY_OFFSET=64'
Image: snmallocciteam/build_linux_x64:latest
32-bit Clang-9 Debug:
CC: clang-9
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x86:latest
32-bit Clang-9 Release:
CC: clang-9
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: ''
Image: snmallocciteam/build_linux_x86:latest
container: $[ variables['Image'] ]
steps:
- script: |
set -eo pipefail
ci/scripts/build.sh
env:
CC: $(CC)
CXX: $(CXX)
BUILD_TYPE: $(BuildType)
CMAKE_ARGS: $(CMakeArgs)
failOnStderr: true
displayName: 'Build'
- script: |
set -eo pipefail
ci/scripts/test.sh
env:
SELF_HOST: $(SelfHost)
BUILD_TYPE: $(BuildType)
failOnStderr: true
displayName: 'Test'
- job:
@ -114,7 +140,7 @@ jobs:
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: ''
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_arm64:latest
64-bit Clang-9 Release:
@ -122,7 +148,7 @@ jobs:
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: ''
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_arm64:latest
32-bit Clang-9 Debug:
@ -130,7 +156,7 @@ jobs:
CXX: clang++-9
BuildType: Debug
SelfHost: false
CMakeArgs: ''
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_armhf:latest
32-bit Clang-9 Release:
@ -138,7 +164,7 @@ jobs:
CXX: clang++-9
BuildType: Release
SelfHost: false
CMakeArgs: ''
CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On'
Image: snmallocciteam/build_linux_armhf:latest
steps:
@ -308,7 +334,7 @@ jobs:
- script: |
set -eo pipefail
ninja clangformat
git diff --exit-code $(Build.SourceVersion)
git diff --exit-code
workingDirectory: build
failOnStderr: true

7
3rdparty/snmalloc/ci/linux_x86 поставляемый Normal file
Просмотреть файл

@ -0,0 +1,7 @@
FROM multiarch/ubuntu-core:x86-bionic
WORKDIR /src
RUN apt update \
&& apt install --no-install-recommends -y ninja-build clang++-9 cmake \
&& apt -y clean

6
3rdparty/snmalloc/ci/scripts/test.sh поставляемый
Просмотреть файл

@ -4,9 +4,11 @@ cd build
if [ $SELF_HOST = false ]; then
ctest -j 4 --output-on-failure -C $BUILD_TYPE
else
sudo cp libsnmallocshim.so libsnmallocshim-1mib.so /usr/local/lib/
sudo cp libsnmallocshim.so libsnmallocshim-16mib.so libsnmallocshim-oe.so /usr/local/lib/
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim-1mib.so ninja
LD_PRELOAD=/usr/local/lib/libsnmallocshim-16mib.so ninja
ninja clean
LD_PRELOAD=/usr/local/lib/libsnmallocshim-oe.so ninja
fi

45
3rdparty/snmalloc/src/aal/aal.h поставляемый
Просмотреть файл

@ -19,6 +19,10 @@
# define PLATFORM_IS_ARM
#endif
#if defined(__powerpc__) || defined(__powerpc64__)
# define PLATFORM_IS_POWERPC
#endif
namespace snmalloc
{
/**
@ -36,6 +40,20 @@ namespace snmalloc
* This architecture cannot access cpu cycles counters.
*/
NoCpuCycleCounters = (1 << 1),
/**
* This architecture enforces strict pointer provenance; we bound the
* pointers given out on malloc() and friends and must, therefore retain
* internal high-privilege pointers for recycling memory on free().
*/
StrictProvenance = (1 << 2),
};
enum AalName : int
{
ARM,
PowerPC,
X86,
X86_SGX,
};
/**
@ -46,6 +64,31 @@ namespace snmalloc
template<class Arch>
struct AAL_Generic : Arch
{
/*
* Provide a default specification of address_t as uintptr_t for Arch-es
* that support IntegerPointers. Those Arch-es without IntegerPoihnters
* must explicitly give their address_t.
*
* This somewhat obtuse way of spelling the defaulting is necessary so
* that all arguments to std::conditional_t are valid, even if they
* wouldn't be valid in context. One might rather wish to say
*
* std::conditional_t<..., uintptr_t, Arch::address_t>
*
* but that requires that Arch::address_t always be given, precisely
* the thing we're trying to avoid with the conditional.
*/
struct default_address_t
{
using address_t = uintptr_t;
};
using address_t = typename std::conditional_t<
(Arch::aal_features & IntegerPointers) != 0,
default_address_t,
Arch>::address_t;
/**
* Prefetch a specific address.
*
@ -102,6 +145,8 @@ namespace snmalloc
# include "aal_x86_sgx.h"
#elif defined(PLATFORM_IS_ARM)
# include "aal_arm.h"
#elif defined(PLATFORM_IS_POWERPC)
# include "aal_powerpc.h"
#endif
namespace snmalloc

6
3rdparty/snmalloc/src/aal/aal_arm.h поставляемый
Просмотреть файл

@ -12,7 +12,7 @@
# endif
#endif
#include <iostream>
#include <cstddef>
namespace snmalloc
{
/**
@ -27,6 +27,10 @@ namespace snmalloc
static constexpr uint64_t aal_features =
IntegerPointers | NoCpuCycleCounters;
static constexpr enum AalName aal_name = ARM;
static constexpr size_t smallest_page_size = 0x1000;
/**
* On pipelined processors, notify the core that we are in a spin loop and
* that speculative execution past this point may not be a performance gain.

37
3rdparty/snmalloc/src/aal/aal_powerpc.h поставляемый Normal file
Просмотреть файл

@ -0,0 +1,37 @@
#pragma once
#if defined(__powerpc64__)
# define SNMALLOC_VA_BITS_64
#else
# define SNMALLOC_VA_BITS_32
#endif
namespace snmalloc
{
/**
* ARM-specific architecture abstraction layer.
*/
class AAL_PowerPC
{
public:
/**
* Bitmap of AalFeature flags
*/
static constexpr uint64_t aal_features = IntegerPointers;
static constexpr enum AalName aal_name = PowerPC;
static constexpr size_t smallest_page_size = 0x1000;
/**
* On pipelined processors, notify the core that we are in a spin loop and
* that speculative execution past this point may not be a performance gain.
*/
static inline void pause()
{
__asm__ volatile("or 27,27,27"); // "yield"
}
};
using AAL_Arch = AAL_PowerPC;
} // namespace snmalloc

4
3rdparty/snmalloc/src/aal/aal_x86.h поставляемый
Просмотреть файл

@ -60,6 +60,10 @@ namespace snmalloc
*/
static constexpr uint64_t aal_features = IntegerPointers;
static constexpr enum AalName aal_name = X86;
static constexpr size_t smallest_page_size = 0x1000;
/**
* On pipelined processors, notify the core that we are in a spin loop and
* that speculative execution past this point may not be a performance gain.

14
3rdparty/snmalloc/src/aal/aal_x86_sgx.h поставляемый
Просмотреть файл

@ -3,8 +3,6 @@
#ifdef _MSC_VER
# include <immintrin.h>
# include <intrin.h>
#else
# include <emmintrin.h>
#endif
#if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
@ -28,13 +26,21 @@ namespace snmalloc
*/
static constexpr uint64_t aal_features = IntegerPointers;
static constexpr enum AalName aal_name = X86_SGX;
static constexpr size_t smallest_page_size = 0x1000;
/**
* On pipelined processors, notify the core that we are in a spin loop and
* that speculative execution past this point may not be a performance gain.
*/
static inline void pause()
{
#ifdef _MSC_VER
_mm_pause();
#else
asm volatile("pause");
#endif
}
/**
@ -42,7 +48,11 @@ namespace snmalloc
*/
static inline void prefetch(void* ptr)
{
#ifdef _MSC_VER
_mm_prefetch(reinterpret_cast<const char*>(ptr), _MM_HINT_T0);
#else
asm volatile("prefetcht0 %0" ::"m"(ptr));
#endif
}
/**

5
3rdparty/snmalloc/src/ds/aba.h поставляемый
Просмотреть файл

@ -99,9 +99,11 @@ namespace snmalloc
(__int64)value,
(__int64*)&old);
# else
# if defined(__GNUC__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)
# if defined(__GNUC__) && defined(SNMALLOC_VA_BITS_64) && \
!defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)
#error You must compile with -mcx16 to enable 16-byte atomic compare and swap.
# endif
Linked xchg{value, old.aba + 1};
std::atomic<Linked>& addr = parent->linked;
@ -119,6 +121,7 @@ namespace snmalloc
}
Cmp(const Cmp&) = delete;
Cmp(Cmp&&) noexcept = default;
};
// This method is used in Verona

41
3rdparty/snmalloc/src/ds/address.h поставляемый
Просмотреть файл

@ -13,7 +13,7 @@ namespace snmalloc
* separated into two types, one for raw addresses and one for addresses that
* can be cast back to pointers.
*/
using address_t = uintptr_t;
using address_t = Aal::address_t;
/**
* Perform pointer arithmetic and return the adjusted pointer.
@ -42,16 +42,6 @@ namespace snmalloc
return reinterpret_cast<address_t>(ptr);
}
/**
* Cast from an address back to a pointer of the specified type. All uses of
* this will eventually need auditing for CHERI compatibility.
*/
template<typename T>
inline T* pointer_cast(address_t address)
{
return reinterpret_cast<T*>(address);
}
/**
* Test if a pointer is aligned to a given size, which must be a power of
* two.
@ -61,8 +51,7 @@ namespace snmalloc
{
static_assert(bits::next_pow2_const(alignment) == alignment);
return ((static_cast<size_t>(address_cast(p)) | size) & (alignment - 1)) ==
0;
return ((address_cast(p) | size) & (alignment - 1)) == 0;
}
/**
@ -81,7 +70,8 @@ namespace snmalloc
#if __has_builtin(__builtin_align_down)
return static_cast<T*>(__builtin_align_down(p, alignment));
#else
return pointer_cast<T>(bits::align_down(address_cast(p), alignment));
return reinterpret_cast<T*>(
bits::align_down(reinterpret_cast<uintptr_t>(p), alignment));
#endif
}
}
@ -102,11 +92,29 @@ namespace snmalloc
#if __has_builtin(__builtin_align_up)
return static_cast<T*>(__builtin_align_up(p, alignment));
#else
return pointer_cast<T>(bits::align_up(address_cast(p), alignment));
return reinterpret_cast<T*>(
bits::align_up(reinterpret_cast<uintptr_t>(p), alignment));
#endif
}
}
/**
* Align a pointer down to a dynamically specified granularity, which must be
* a power of two.
*/
template<typename T = void>
SNMALLOC_FAST_PATH T* pointer_align_down(void* p, size_t alignment)
{
SNMALLOC_ASSERT(alignment > 0);
SNMALLOC_ASSERT(bits::next_pow2(alignment) == alignment);
#if __has_builtin(__builtin_align_down)
return static_cast<T*>(__builtin_align_down(p, alignment));
#else
return reinterpret_cast<T*>(
bits::align_down(reinterpret_cast<uintptr_t>(p), alignment));
#endif
}
/**
* Align a pointer up to a dynamically specified granularity, which must
* be a power of two.
@ -119,7 +127,8 @@ namespace snmalloc
#if __has_builtin(__builtin_align_up)
return static_cast<T*>(__builtin_align_up(p, alignment));
#else
return pointer_cast<T>(bits::align_up(address_cast(p), alignment));
return reinterpret_cast<T*>(
bits::align_up(reinterpret_cast<uintptr_t>(p), alignment));
#endif
}

92
3rdparty/snmalloc/src/ds/cdllist.h поставляемый
Просмотреть файл

@ -7,13 +7,8 @@
namespace snmalloc
{
/**
* Special class for cyclic doubly linked non-empty linked list
*
* This code assumes there is always one element in the list. The client
* must ensure there is a sentinal element.
*/
class CDLLNode
template<typename T>
class CDLLNodeBase
{
/**
* to_next is used to handle a zero initialised data structure.
@ -22,38 +17,86 @@ namespace snmalloc
*/
ptrdiff_t to_next = 0;
// TODO: CHERI will need a real pointer too
// CDLLNode* next = nullptr;
CDLLNode* prev = nullptr;
void set_next(CDLLNode* c)
protected:
void set_next(T* c)
{
// TODO: CHERI will need a real pointer too
// next = c;
to_next = pointer_diff_signed(this, c);
}
public:
SNMALLOC_FAST_PATH bool is_empty()
{
return to_next == 0;
}
SNMALLOC_FAST_PATH T* get_next()
{
return pointer_offset_signed(static_cast<T*>(this), to_next);
}
};
template<typename T>
class CDLLNodeBaseNext
{
/**
* Like to_next in the pointer-less case, this version still works with
* zero-initialized data structure. To make `is_empty` work in this case,
* next is set to `nullptr` rather than `this` when the list is empty.
*
*/
T* next = nullptr;
protected:
void set_next(T* c)
{
next = (c == static_cast<T*>(this)) ? nullptr : c;
}
public:
SNMALLOC_FAST_PATH bool is_empty()
{
return next == nullptr;
}
SNMALLOC_FAST_PATH T* get_next()
{
return next == nullptr ? static_cast<T*>(this) : next;
}
};
template<typename T>
using CDLLNodeParent = std::conditional_t<
aal_supports<StrictProvenance>,
CDLLNodeBaseNext<T>,
CDLLNodeBase<T>>;
/**
* Special class for cyclic doubly linked non-empty linked list
*
* This code assumes there is always one element in the list. The client
* must ensure there is a sentinal element.
*/
class CDLLNode : public CDLLNodeParent<CDLLNode>
{
CDLLNode* prev = nullptr;
public:
/**
* Single element cyclic list. This is the empty case.
*/
CDLLNode()
{
set_next(this);
this->set_next(this);
prev = this;
}
SNMALLOC_FAST_PATH bool is_empty()
{
return to_next == 0;
}
/**
* Removes this element from the cyclic list is it part of.
*/
SNMALLOC_FAST_PATH void remove()
{
SNMALLOC_ASSERT(!is_empty());
SNMALLOC_ASSERT(!this->is_empty());
debug_check();
get_next()->prev = prev;
prev->set_next(get_next());
@ -67,13 +110,6 @@ namespace snmalloc
#endif
}
SNMALLOC_FAST_PATH CDLLNode* get_next()
{
// TODO: CHERI will require a real pointer
// return next;
return pointer_offset_signed(this, to_next);
}
SNMALLOC_FAST_PATH CDLLNode* get_prev()
{
return prev;

2
3rdparty/snmalloc/src/ds/defines.h поставляемый
Просмотреть файл

@ -37,7 +37,7 @@
namespace snmalloc
{
// Forwards reference so that the platform can define how to handle errors.
void error(const char* const str);
[[noreturn]] void error(const char* const str);
} // namespace snmalloc
#define TOSTRING(expr) TOSTRING2(expr)

4
3rdparty/snmalloc/src/ds/dllist.h поставляемый
Просмотреть файл

@ -17,7 +17,7 @@ namespace snmalloc
* are always the same, invalid pointer values with different sentinels are
* always different.
*/
template<uintptr_t OtherSentinel>
template<address_t OtherSentinel>
constexpr bool operator==(const InvalidPointer<OtherSentinel>&)
{
return Sentinel == OtherSentinel;
@ -27,7 +27,7 @@ namespace snmalloc
* are always the same, invalid pointer values with different sentinels are
* always different.
*/
template<uintptr_t OtherSentinel>
template<address_t OtherSentinel>
constexpr bool operator!=(const InvalidPointer<OtherSentinel>&)
{
return Sentinel != OtherSentinel;

2
3rdparty/snmalloc/src/ds/helpers.h поставляемый
Просмотреть файл

@ -14,7 +14,7 @@ namespace snmalloc
class Singleton
{
inline static std::atomic_flag flag;
inline static std::atomic<bool> initialised = false;
inline static std::atomic<bool> initialised{false};
inline static Object obj;
public:

6
3rdparty/snmalloc/src/ds/mpscq.h поставляемый
Просмотреть файл

@ -14,7 +14,7 @@ namespace snmalloc
std::is_same<decltype(T::next), std::atomic<T*>>::value,
"T->next must be a std::atomic<T*>");
std::atomic<T*> back = nullptr;
std::atomic<T*> back{nullptr};
T* front = nullptr;
public:
@ -72,10 +72,10 @@ namespace snmalloc
SNMALLOC_ASSERT(front);
std::atomic_thread_fence(std::memory_order_acquire);
invariant();
return std::pair(first, true);
return {first, true};
}
return std::pair(nullptr, false);
return {nullptr, false};
}
};
} // namespace snmalloc

243
3rdparty/snmalloc/src/mem/address_space.h поставляемый Normal file
Просмотреть файл

@ -0,0 +1,243 @@
#include "../ds/address.h"
#include "../ds/flaglock.h"
#include "../pal/pal.h"
#include <array>
namespace snmalloc
{
/**
* Implements a power of two allocator, where all blocks are aligned to the
* same power of two as their size. This is what snmalloc uses to get
* alignment of very large sizeclasses.
*
* It cannot unreserve memory, so this does not require the
* usual complexity of a buddy allocator.
*/
template<typename Pal>
class AddressSpaceManager : public Pal
{
/**
* Stores the blocks of address space
*
* The first level of array indexes based on power of two size.
*
* The first entry ranges[n][0] is just a pointer to an address range
* of size 2^n.
*
* The second entry ranges[n][1] is a pointer to a linked list of blocks
* of this size. The final block in the list is not committed, so we commit
* on pop for this corner case.
*
* Invariants
* ranges[n][1] != nullptr => ranges[n][0] != nullptr
*
* bits::BITS is used for simplicity, we do not use below the pointer size,
* and large entries will be unlikely to be supported by the platform.
*/
std::array<std::array<void*, 2>, bits::BITS> ranges = {};
/**
* This is infrequently used code, a spin lock simplifies the code
* considerably, and should never be on the fast path.
*/
std::atomic_flag spin_lock = ATOMIC_FLAG_INIT;
/**
* Checks a block satisfies its invariant.
*/
inline void check_block(void* base, size_t align_bits)
{
SNMALLOC_ASSERT(
base == pointer_align_up(base, bits::one_at_bit(align_bits)));
// All blocks need to be bigger than a pointer.
SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*));
UNUSED(base);
UNUSED(align_bits);
}
/**
* Adds a block to `ranges`.
*/
void add_block(size_t align_bits, void* base)
{
check_block(base, align_bits);
SNMALLOC_ASSERT(align_bits < 64);
if (ranges[align_bits][0] == nullptr)
{
// Prefer first slot if available.
ranges[align_bits][0] = base;
return;
}
if (ranges[align_bits][1] != nullptr)
{
// Add to linked list.
commit_block(base, sizeof(void*));
*reinterpret_cast<void**>(base) = ranges[align_bits][1];
check_block(ranges[align_bits][1], align_bits);
}
// Update head of list
ranges[align_bits][1] = base;
check_block(ranges[align_bits][1], align_bits);
}
/**
* Find a block of the correct size. May split larger blocks
* to satisfy this request.
*/
void* remove_block(size_t align_bits)
{
auto first = ranges[align_bits][0];
if (first == nullptr)
{
if (align_bits == (bits::BITS - 1))
{
// Out of memory
return nullptr;
}
// Look for larger block and split up recursively
void* bigger = remove_block(align_bits + 1);
if (bigger != nullptr)
{
void* left_over =
pointer_offset(bigger, bits::one_at_bit(align_bits));
ranges[align_bits][0] = left_over;
check_block(left_over, align_bits);
}
check_block(bigger, align_bits + 1);
return bigger;
}
auto second = ranges[align_bits][1];
if (second != nullptr)
{
commit_block(second, sizeof(void*));
auto next = *reinterpret_cast<void**>(second);
ranges[align_bits][1] = next;
// Zero memory. Client assumes memory contains only zeros.
*reinterpret_cast<void**>(second) = nullptr;
check_block(second, align_bits);
check_block(next, align_bits);
return second;
}
check_block(first, align_bits);
ranges[align_bits][0] = nullptr;
return first;
}
/**
* Add a range of memory to the address space.
* Divides blocks into power of two sizes with natural alignment
*/
void add_range(void* base, size_t length)
{
// Find the minimum set of maximally aligned blocks in this range.
// Each block's alignment and size are equal.
while (length >= sizeof(void*))
{
size_t base_align_bits = bits::ctz(address_cast(base));
size_t length_align_bits = (bits::BITS - 1) - bits::clz(length);
size_t align_bits = bits::min(base_align_bits, length_align_bits);
size_t align = bits::one_at_bit(align_bits);
check_block(base, align_bits);
add_block(align_bits, base);
base = pointer_offset(base, align);
length -= align;
}
}
/**
* Commit a block of memory
*/
void commit_block(void* base, size_t size)
{
// Rounding required for sub-page allocations.
auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(base);
auto page_end =
pointer_align_up<OS_PAGE_SIZE, char>(pointer_offset(base, size));
Pal::template notify_using<NoZero>(
page_start, static_cast<size_t>(page_end - page_start));
}
public:
/**
* Returns a pointer to a block of memory of the supplied size.
* The block will be committed, if specified by the template parameter.
* The returned block is guaranteed to be aligened to the size.
*
* Only request 2^n sizes, and not less than a pointer.
*/
template<bool committed>
void* reserve(size_t size)
{
SNMALLOC_ASSERT(bits::next_pow2(size) == size);
SNMALLOC_ASSERT(size >= sizeof(void*));
if constexpr (pal_supports<AlignedAllocation, Pal>)
{
if (size >= Pal::minimum_alloc_size)
return static_cast<Pal*>(this)->template reserve_aligned<committed>(
size);
}
void* res;
{
FlagLock lock(spin_lock);
res = remove_block(bits::next_pow2_bits(size));
if (res == nullptr)
{
// Allocation failed ask OS for more memory
void* block;
size_t block_size;
if constexpr (pal_supports<AlignedAllocation, Pal>)
{
block_size = Pal::minimum_alloc_size;
block = static_cast<Pal*>(this)->template reserve_aligned<false>(
block_size);
}
else
{
// Need at least 2 times the space to guarantee alignment.
// Hold lock here as a race could cause additional requests to
// the Pal, and this could lead to suprious OOM. This is
// particularly bad if the Pal gives all the memory on first call.
auto block_and_size =
static_cast<Pal*>(this)->reserve_at_least(size * 2);
block = block_and_size.first;
block_size = block_and_size.second;
// Ensure block is pointer aligned.
if (
pointer_align_up(block, sizeof(void*)) != block ||
bits::align_up(block_size, sizeof(void*)) > block_size)
{
auto diff =
pointer_diff(block, pointer_align_up(block, sizeof(void*)));
block_size = block_size - diff;
block_size = bits::align_down(block_size, sizeof(void*));
}
}
if (block == nullptr)
{
return nullptr;
}
add_range(block, block_size);
// still holding lock so guaranteed to succeed.
res = remove_block(bits::next_pow2_bits(size));
}
}
// Don't need lock while committing pages.
if constexpr (committed)
commit_block(res, size);
return res;
}
};
} // namespace snmalloc

87
3rdparty/snmalloc/src/mem/alloc.h поставляемый
Просмотреть файл

@ -17,6 +17,7 @@
#include "sizeclasstable.h"
#include "slab.h"
#include <array>
#include <functional>
namespace snmalloc
@ -284,6 +285,7 @@ namespace snmalloc
UNUSED(size);
return free(p);
#else
SNMALLOC_ASSERT(p != nullptr);
check_size(p, size);
if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1)))
{
@ -393,7 +395,7 @@ namespace snmalloc
}
template<Boundary location = Start>
static address_t external_address(void* p)
static void* external_pointer(void* p)
{
#ifdef USE_MALLOC
error("Unsupported");
@ -422,12 +424,13 @@ namespace snmalloc
return external_pointer<location>(p, sc, slab_end);
}
auto ss = address_cast(super);
auto ss = super;
while (size > 64)
{
// This is a large alloc redirect.
ss = ss - (1ULL << (size - 64));
ss = pointer_offset_signed(
ss, -(static_cast<ptrdiff_t>(1) << (size - 64)));
size = ChunkMap::get(ss);
}
@ -435,38 +438,35 @@ namespace snmalloc
{
if constexpr ((location == End) || (location == OnePastEnd))
// We don't know the End, so return MAX_PTR
return UINTPTR_MAX;
return pointer_offset<void>(nullptr, UINTPTR_MAX);
else
// We don't know the Start, so return MIN_PTR
return 0;
return nullptr;
}
// This is a large alloc, mask off to the slab size.
if constexpr (location == Start)
return ss;
else if constexpr (location == End)
return (ss + (1ULL << size) - 1ULL);
return pointer_offset(ss, (1ULL << size) - 1ULL);
else
return (ss + (1ULL << size));
return pointer_offset(ss, 1ULL << size);
#endif
}
template<Boundary location = Start>
static void* external_pointer(void* p)
private:
SNMALLOC_SLOW_PATH static size_t alloc_size_error()
{
return pointer_cast<void>(external_address<location>(p));
error("Not allocated by this allocator");
}
static size_t alloc_size(void* p)
public:
SNMALLOC_FAST_PATH static size_t alloc_size(const void* p)
{
// This must be called on an external pointer.
size_t size = ChunkMap::get(address_cast(p));
if (size == 0)
{
error("Not allocated by this allocator");
}
else if (size == CMSuperslab)
if (likely(size == CMSuperslab))
{
Superslab* super = Superslab::get(p);
@ -477,7 +477,8 @@ namespace snmalloc
return sizeclass_to_size(meta.sizeclass);
}
else if (size == CMMediumslab)
if (likely(size == CMMediumslab))
{
Mediumslab* slab = Mediumslab::get(p);
// Reading a remote sizeclass won't fail, since the other allocator
@ -485,7 +486,12 @@ namespace snmalloc
return sizeclass_to_size(slab->get_sizeclass());
}
return 1ULL << size;
if (likely(size != 0))
{
return 1ULL << size;
}
return alloc_size_error();
}
size_t get_id()
@ -508,14 +514,9 @@ namespace snmalloc
* A stub Remote object that will always be the head of this list;
* never taken for further processing.
*/
Remote head;
Remote head{};
Remote* last;
RemoteList()
{
clear();
}
Remote* last{&head};
void clear()
{
@ -538,8 +539,8 @@ namespace snmalloc
* need to dispatch everything, we can check if we are a real allocator
* and lazily provide a real allocator.
*/
int64_t capacity = 0;
RemoteList list[REMOTE_SLOTS];
int64_t capacity{0};
std::array<RemoteList, REMOTE_SLOTS> list{};
/// Used to find the index into the array of queues for remote
/// deallocation
@ -728,12 +729,6 @@ namespace snmalloc
size_t size1 = sizeclass_to_size(sc1);
size_t size2 = sizeclass_to_size(sc2);
// All medium size classes are page aligned.
if (i > NUM_SMALL_CLASSES)
{
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(nullptr, size1));
}
SNMALLOC_ASSERT(sc1 == i);
SNMALLOC_ASSERT(sc1 == sc2);
SNMALLOC_ASSERT(size1 == size);
@ -823,31 +818,35 @@ namespace snmalloc
}
template<Boundary location>
static uintptr_t
static void*
external_pointer(void* p, sizeclass_t sizeclass, void* end_point)
{
size_t rsize = sizeclass_to_size(sizeclass);
void* end_point_correction = location == End ?
(static_cast<uint8_t*>(end_point) - 1) :
(location == OnePastEnd ? end_point :
(static_cast<uint8_t*>(end_point) - rsize));
pointer_offset_signed(end_point, -1) :
(location == OnePastEnd ?
end_point :
pointer_offset_signed(end_point, -static_cast<ptrdiff_t>(rsize)));
ptrdiff_t offset_from_end =
(static_cast<uint8_t*>(end_point) - 1) - static_cast<uint8_t*>(p);
size_t offset_from_end =
pointer_diff(p, pointer_offset_signed(end_point, -1));
size_t end_to_end =
round_by_sizeclass(rsize, static_cast<size_t>(offset_from_end));
size_t end_to_end = round_by_sizeclass(rsize, offset_from_end);
return address_cast<uint8_t>(
static_cast<uint8_t*>(end_point_correction) - end_to_end);
return pointer_offset_signed(
end_point_correction, -static_cast<ptrdiff_t>(end_to_end));
}
void init_message_queue()
{
// Manufacture an allocation to prime the queue
// Using an actual allocation removes a conditional of a critical path.
// Using an actual allocation removes a conditional from a critical path.
Remote* dummy = reinterpret_cast<Remote*>(alloc<YesZero>(MIN_ALLOC_SIZE));
if (dummy == nullptr)
{
error("Critical error: Out-of-memory during initialisation.");
}
dummy->set_target_id(id());
message_queue().init(dummy);
}

37
3rdparty/snmalloc/src/mem/allocconfig.h поставляемый
Просмотреть файл

@ -41,20 +41,21 @@ namespace snmalloc
// Specifies smaller slab and super slab sizes for address space
// constrained scenarios.
static constexpr size_t ADDRESS_SPACE_CONSTRAINED =
#ifdef IS_ADDRESS_SPACE_CONSTRAINED
true
#else
static constexpr size_t USE_LARGE_CHUNKS =
#ifdef SNMALLOC_USE_LARGE_CHUNKS
// In 32 bit uses smaller superslab.
(!bits::is64())
(bits::is64())
#else
false
#endif
;
static constexpr size_t RESERVE_MULTIPLE =
#ifdef USE_RESERVE_MULTIPLE
USE_RESERVE_MULTIPLE
// Specifies even smaller slab and super slab sizes for open enclave.
static constexpr size_t USE_SMALL_CHUNKS =
#ifdef SNMALLOC_USE_SMALL_CHUNKS
true
#else
bits::is64() ? 16 : 2
false
#endif
;
@ -92,18 +93,7 @@ namespace snmalloc
// Used to isolate values on cache lines to prevent false sharing.
static constexpr size_t CACHELINE_SIZE = 64;
// Used to keep Superslab metadata committed.
static constexpr size_t OS_PAGE_SIZE = 0x1000;
static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS;
// Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h)
// define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable
// name, to avoid conflicts, but if we do see a macro definition then check
// that our value matches the platform's expected value.
#ifdef PAGE_SIZE
static_assert(
PAGE_SIZE == OS_PAGE_SIZE,
"Page size from system header does not match snmalloc config page size.");
#endif
// Minimum allocation size is space for two pointers.
static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*));
@ -111,19 +101,20 @@ namespace snmalloc
static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE);
// Slabs are 64 KiB unless constrained to 16 KiB.
static constexpr size_t SLAB_BITS = ADDRESS_SPACE_CONSTRAINED ? 14 : 16;
static constexpr size_t SLAB_BITS =
USE_SMALL_CHUNKS ? 13 : (USE_LARGE_CHUNKS ? 16 : 14);
static constexpr size_t SLAB_SIZE = 1 << SLAB_BITS;
static constexpr size_t SLAB_MASK = ~(SLAB_SIZE - 1);
// Superslabs are composed of this many slabs. Slab offsets are encoded as
// a byte, so the maximum count is 256. This must be a power of two to
// allow fast masking to find a superslab start address.
static constexpr size_t SLAB_COUNT_BITS = ADDRESS_SPACE_CONSTRAINED ? 6 : 8;
static constexpr size_t SLAB_COUNT_BITS =
USE_SMALL_CHUNKS ? 5 : (USE_LARGE_CHUNKS ? 8 : 6);
static constexpr size_t SLAB_COUNT = 1 << SLAB_COUNT_BITS;
static constexpr size_t SUPERSLAB_SIZE = SLAB_SIZE * SLAB_COUNT;
static constexpr size_t SUPERSLAB_MASK = ~(SUPERSLAB_SIZE - 1);
static constexpr size_t SUPERSLAB_BITS = SLAB_BITS + SLAB_COUNT_BITS;
static constexpr size_t RESERVE_SIZE = SUPERSLAB_SIZE * RESERVE_MULTIPLE;
static_assert((1ULL << SUPERSLAB_BITS) == SUPERSLAB_SIZE, "Sanity check");

178
3rdparty/snmalloc/src/mem/largealloc.h поставляемый
Просмотреть файл

@ -4,6 +4,7 @@
#include "../ds/helpers.h"
#include "../ds/mpmcstack.h"
#include "../pal/pal.h"
#include "address_space.h"
#include "allocstats.h"
#include "baseslab.h"
#include "sizeclass.h"
@ -58,27 +59,17 @@ namespace snmalloc
template<class PAL>
class MemoryProviderStateMixin : public PalNotificationObject, public PAL
{
/**
* Flag to protect the bump allocator
*/
std::atomic_flag lock = ATOMIC_FLAG_INIT;
/**
* Pointer to block being bump allocated
*/
void* bump = nullptr;
/**
* Space remaining in this block being bump allocated
*/
size_t remaining = 0;
/**
* Simple flag for checking if another instance of lazy-decommit is
* running
*/
std::atomic_flag lazy_decommit_guard = {};
/**
* Manages address space for this memory provider.
*/
AddressSpaceManager<PAL> address_space = {};
public:
/**
* Stack of large allocations that have been returned for reuse.
@ -91,12 +82,15 @@ namespace snmalloc
static MemoryProviderStateMixin<PAL>* make() noexcept
{
// Temporary stack-based storage to start the allocator in.
MemoryProviderStateMixin<PAL> local;
MemoryProviderStateMixin<PAL> local{};
// Allocate permanent storage for the allocator usung temporary allocator
MemoryProviderStateMixin<PAL>* allocated =
local.alloc_chunk<MemoryProviderStateMixin<PAL>, 1>();
if (allocated == nullptr)
error("Failed to initialise system!");
#ifdef GCC_VERSION_EIGHT_PLUS
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wclass-memaccess"
@ -105,7 +99,10 @@ namespace snmalloc
// memcpy is safe as this is entirely single threaded: the move
// constructors were removed as unsafe to move std::atomic in a
// concurrent setting.
memcpy(allocated, &local, sizeof(MemoryProviderStateMixin<PAL>));
::memcpy(
&(allocated->address_space),
&(local.address_space),
sizeof(AddressSpaceManager<PAL>));
#ifdef GCC_VERSION_EIGHT_PLUS
# pragma GCC diagnostic pop
#endif
@ -121,22 +118,6 @@ namespace snmalloc
}
private:
void new_block()
{
// Reserve the smallest large_class which is SUPERSLAB_SIZE
void* r = reserve<false>(0);
if (r == nullptr)
Pal::error(
"Unrecoverable internal error: \
failed to allocator internal data structure.");
PAL::template notify_using<NoZero>(r, OS_PAGE_SIZE);
bump = r;
remaining = SUPERSLAB_SIZE;
}
SNMALLOC_SLOW_PATH void lazy_decommit()
{
// If another thread is try to do lazy decommit, let it continue. If
@ -183,25 +164,6 @@ namespace snmalloc
lazy_decommit_guard.clear();
}
void push_space(address_t start, size_t large_class)
{
// All fresh pages so can use "NoZero"
void* p = pointer_cast<void>(start);
if (large_class > 0)
PAL::template notify_using<NoZero>(p, OS_PAGE_SIZE);
else
{
if (decommit_strategy == DecommitSuperLazy)
{
PAL::template notify_using<NoZero>(p, OS_PAGE_SIZE);
p = new (p) Decommittedslab();
}
else
PAL::template notify_using<NoZero>(p, SUPERSLAB_SIZE);
}
large_stack[large_class].push(reinterpret_cast<Largeslab*>(p));
}
/***
* Method for callback object to perform lazy decommit.
*/
@ -222,45 +184,10 @@ namespace snmalloc
{
// Cache line align
size_t size = bits::align_up(sizeof(T), 64);
void* p;
{
FlagLock f(lock);
if constexpr (alignment != 0)
{
char* aligned_bump = pointer_align_up<alignment, char>(bump);
size_t bump_delta = pointer_diff(bump, aligned_bump);
if (bump_delta > remaining)
{
new_block();
}
else
{
remaining -= bump_delta;
bump = aligned_bump;
}
}
if (remaining < size)
{
new_block();
}
p = bump;
bump = pointer_offset(bump, size);
remaining -= size;
}
auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(p);
auto page_end =
pointer_align_up<OS_PAGE_SIZE, char>(pointer_offset(p, size));
PAL::template notify_using<NoZero>(
page_start, static_cast<size_t>(page_end - page_start));
size = bits::max(size, alignment);
void* p = address_space.template reserve<true>(bits::next_pow2(size));
if (p == nullptr)
return nullptr;
return new (p) T(std::forward<Args...>(args)...);
}
@ -268,67 +195,8 @@ namespace snmalloc
void* reserve(size_t large_class) noexcept
{
size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
size_t align = size;
if constexpr (pal_supports<AlignedAllocation, PAL>)
{
return PAL::template reserve<committed>(size, align);
}
else
{
// Reserve 4 times the amount, and put aligned leftovers into the
// large_stack
size_t request = bits::max(size * 4, SUPERSLAB_SIZE * 8);
void* p = PAL::template reserve<false>(request);
if (p == nullptr)
return nullptr;
address_t p0 = address_cast(p);
address_t start = bits::align_up(p0, align);
address_t p1 = p0 + request;
address_t end = start + size;
for (; end < bits::align_down(p1, align); end += size)
{
push_space(end, large_class);
}
// Put offcuts before alignment into the large stack
address_t offcut_end = start;
address_t offcut_start;
for (size_t i = large_class; i > 0;)
{
i--;
size_t offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i;
offcut_start = bits::align_up(p0, offcut_align);
if (offcut_start != offcut_end)
{
push_space(offcut_start, i);
offcut_end = offcut_start;
}
}
// Put offcuts after returned block into the large stack
offcut_start = end;
for (size_t i = large_class; i > 0;)
{
i--;
auto offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i;
offcut_end = bits::align_down(p1, offcut_align);
if (offcut_start != offcut_end)
{
push_space(offcut_start, i);
offcut_start = offcut_end;
}
}
void* result = pointer_cast<void>(start);
if (committed)
PAL::template notify_using<NoZero>(result, size);
return result;
}
return address_space.template reserve<committed>(size);
}
};
@ -366,8 +234,7 @@ namespace snmalloc
p = memory_provider.template reserve<false>(large_class);
if (p == nullptr)
return nullptr;
memory_provider.template notify_using<zero_mem>(
p, bits::align_up(size, OS_PAGE_SIZE));
memory_provider.template notify_using<zero_mem>(p, rsize);
}
else
{
@ -390,8 +257,7 @@ namespace snmalloc
// Passing zero_mem ensures the PAL provides zeroed pages if
// required.
memory_provider.template notify_using<zero_mem>(
pointer_offset(p, OS_PAGE_SIZE),
bits::align_up(size, OS_PAGE_SIZE) - OS_PAGE_SIZE);
pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE);
}
else
{
@ -399,6 +265,8 @@ namespace snmalloc
if constexpr (zero_mem == YesZero)
memory_provider.template zero<true>(
p, bits::align_up(size, OS_PAGE_SIZE));
else
UNUSED(size);
}
}

12
3rdparty/snmalloc/src/mem/mediumslab.h поставляемый
Просмотреть файл

@ -39,9 +39,10 @@ namespace snmalloc
return OS_PAGE_SIZE;
}
static Mediumslab* get(void* p)
static Mediumslab* get(const void* p)
{
return pointer_align_down<SUPERSLAB_SIZE, Mediumslab>(p);
return pointer_align_down<SUPERSLAB_SIZE, Mediumslab>(
const_cast<void*>(p));
}
void init(RemoteAllocator* alloc, sizeclass_t sc, size_t rsize)
@ -84,11 +85,10 @@ namespace snmalloc
void* p = pointer_offset(this, (static_cast<size_t>(index) << 8));
free--;
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, OS_PAGE_SIZE));
size = bits::align_up(size, OS_PAGE_SIZE);
if constexpr (zero_mem == YesZero)
memory_provider.template zero<true>(p, size);
memory_provider.zero(p, size);
else
UNUSED(size);
return p;
}

4
3rdparty/snmalloc/src/mem/metaslab.h поставляемый
Просмотреть файл

@ -140,9 +140,9 @@ namespace snmalloc
return (slab_end - allocation_start) % size == 0;
}
static Slab* get_slab(void* p)
static Slab* get_slab(const void* p)
{
return pointer_align_down<SLAB_SIZE, Slab>(p);
return pointer_align_down<SLAB_SIZE, Slab>(const_cast<void*>(p));
}
static bool is_short(Slab* p)

6
3rdparty/snmalloc/src/mem/pagemap.h поставляемый
Просмотреть файл

@ -188,7 +188,7 @@ namespace snmalloc
{
PagemapEntry* value = get_node<create_addr>(e, result);
if (unlikely(!result))
return std::pair(nullptr, 0);
return {nullptr, 0};
shift -= BITS_PER_INDEX_LEVEL;
ix = (static_cast<size_t>(addr) >> shift) & ENTRIES_MASK;
@ -208,11 +208,11 @@ namespace snmalloc
Leaf* leaf = reinterpret_cast<Leaf*>(get_node<create_addr>(e, result));
if (unlikely(!result))
return std::pair(nullptr, 0);
return {nullptr, 0};
shift -= BITS_FOR_LEAF;
ix = (static_cast<size_t>(addr) >> shift) & LEAF_MASK;
return std::pair(leaf, ix);
return {leaf, ix};
}
template<bool create_addr>

2
3rdparty/snmalloc/src/mem/pooled.h поставляемый
Просмотреть файл

@ -14,7 +14,7 @@ namespace snmalloc
friend class MPMCStack;
/// Used by the pool for chaining together entries when not in use.
std::atomic<T*> next = nullptr;
std::atomic<T*> next{nullptr};
/// Used by the pool to keep the list of all entries ever created.
T* list_next;
std::atomic_flag in_use = ATOMIC_FLAG_INIT;

2
3rdparty/snmalloc/src/mem/remoteallocator.h поставляемый
Просмотреть файл

@ -19,7 +19,7 @@ namespace snmalloc
union
{
Remote* non_atomic_next;
std::atomic<Remote*> next = nullptr;
std::atomic<Remote*> next{nullptr};
};
alloc_id_t allocator_id;

15
3rdparty/snmalloc/src/mem/sizeclass.h поставляемый
Просмотреть файл

@ -1,6 +1,6 @@
#pragma once
#include "../pal/pal_consts.h"
#include "../pal/pal.h"
#include "allocconfig.h"
namespace snmalloc
@ -185,4 +185,17 @@ namespace snmalloc
return ((alignment - 1) | (size - 1)) + 1;
}
SNMALLOC_FAST_PATH static size_t round_size(size_t size)
{
if (size > sizeclass_to_size(NUM_SIZECLASSES - 1))
{
return bits::next_pow2(size);
}
if (size == 0)
{
size = 1;
}
return sizeclass_to_size(size_to_sizeclass(size));
}
} // namespace snmalloc

2
3rdparty/snmalloc/src/mem/slowalloc.h поставляемый
Просмотреть файл

@ -63,6 +63,6 @@ namespace snmalloc
*/
inline SlowAllocator get_slow_allocator()
{
return SlowAllocator{};
return {};
}
} // namespace snmalloc

9
3rdparty/snmalloc/src/mem/superslab.h поставляемый
Просмотреть файл

@ -65,9 +65,10 @@ namespace snmalloc
StatusChange = 2
};
static Superslab* get(void* p)
static Superslab* get(const void* p)
{
return pointer_align_down<SUPERSLAB_SIZE, Superslab>(p);
return pointer_align_down<SUPERSLAB_SIZE, Superslab>(
const_cast<void*>(p));
}
static bool is_short_sizeclass(sizeclass_t sizeclass)
@ -180,8 +181,8 @@ namespace snmalloc
Slab* alloc_slab(sizeclass_t sizeclass)
{
uint8_t h = head;
Slab* slab = pointer_cast<Slab>(
address_cast(this) + (static_cast<size_t>(h) << SLAB_BITS));
Slab* slab = pointer_offset(
reinterpret_cast<Slab*>(this), (static_cast<size_t>(h) << SLAB_BITS));
uint8_t n = meta[h].next;

23
3rdparty/snmalloc/src/override/malloc.cc поставляемый
Просмотреть файл

@ -9,11 +9,19 @@ using namespace snmalloc;
#ifndef SNMALLOC_EXPORT
# define SNMALLOC_EXPORT
#endif
#ifndef SNMALLOC_NAME_MANGLE
#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX
# define __SN_CONCAT(a, b) a##b
# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b)
# define SNMALLOC_NAME_MANGLE(a) \
__SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a)
#elif !defined(SNMALLOC_NAME_MANGLE)
# define SNMALLOC_NAME_MANGLE(a) a
#endif
#ifndef MALLOC_USABLE_SIZE_QUALIFIER
# define MALLOC_USABLE_SIZE_QUALIFIER
#endif
extern "C"
{
SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr)
@ -31,6 +39,11 @@ extern "C"
ThreadAlloc::get_noncachable()->dealloc(ptr);
}
SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(cfree)(void* ptr)
{
SNMALLOC_NAME_MANGLE(free)(ptr);
}
SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size)
{
bool overflow = false;
@ -43,7 +56,9 @@ extern "C"
return ThreadAlloc::get_noncachable()->alloc<ZeroMem::YesZero>(sz);
}
SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(void* ptr)
SNMALLOC_EXPORT
size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)(
MALLOC_USABLE_SIZE_QUALIFIER void* ptr)
{
return Alloc::alloc_size(ptr);
}
@ -75,7 +90,7 @@ extern "C"
#endif
size_t sz = Alloc::alloc_size(ptr);
// Keep the current allocation if the given size is in the same sizeclass.
if (sz == sizeclass_to_size(size_to_sizeclass(size)))
if (sz == round_size(size))
return ptr;
void* p = SNMALLOC_NAME_MANGLE(malloc)(size);

4
3rdparty/snmalloc/src/override/new.cc поставляемый
Просмотреть файл

@ -43,6 +43,8 @@ void operator delete(void* p)EXCEPTSPEC
void operator delete(void* p, size_t size)EXCEPTSPEC
{
if (p == nullptr)
return;
ThreadAlloc::get_noncachable()->dealloc(p, size);
}
@ -58,6 +60,8 @@ void operator delete[](void* p) EXCEPTSPEC
void operator delete[](void* p, size_t size) EXCEPTSPEC
{
if (p == nullptr)
return;
ThreadAlloc::get_noncachable()->dealloc(p, size);
}

35
3rdparty/snmalloc/src/pal/pal.h поставляемый
Просмотреть файл

@ -3,18 +3,20 @@
#include "pal_consts.h"
// If simultating OE, then we need the underlying platform
#if defined(OPEN_ENCLAVE)
# include "pal_open_enclave.h"
#endif
#if !defined(OPEN_ENCLAVE) || defined(OPEN_ENCLAVE_SIMULATION)
# include "pal_apple.h"
# include "pal_freebsd.h"
# include "pal_freebsd_kernel.h"
# include "pal_haiku.h"
# include "pal_linux.h"
# include "pal_netbsd.h"
# include "pal_openbsd.h"
# include "pal_solaris.h"
# include "pal_windows.h"
#endif
#if defined(OPEN_ENCLAVE)
# include "pal_open_enclave.h"
#endif
#include "pal_plain.h"
namespace snmalloc
@ -31,10 +33,14 @@ namespace snmalloc
PALFreeBSDKernel;
# elif defined(__FreeBSD__)
PALFreeBSD;
# elif defined(__HAIKU__)
PALHaiku;
# elif defined(__NetBSD__)
PALNetBSD;
# elif defined(__OpenBSD__)
PALOpenBSD;
# elif defined(__sun)
PALSolaris;
# else
# error Unsupported platform
# endif
@ -49,7 +55,7 @@ namespace snmalloc
DefaultPal;
#endif
SNMALLOC_SLOW_PATH inline void error(const char* const str)
[[noreturn]] SNMALLOC_SLOW_PATH inline void error(const char* const str)
{
Pal::error(str);
}
@ -59,4 +65,25 @@ namespace snmalloc
*/
template<PalFeatures F, typename PAL = Pal>
constexpr static bool pal_supports = (PAL::pal_features & F) == F;
// Used to keep Superslab metadata committed.
static constexpr size_t OS_PAGE_SIZE = Pal::page_size;
static_assert(
bits::next_pow2_const(OS_PAGE_SIZE) == OS_PAGE_SIZE,
"OS_PAGE_SIZE must be a power of two");
static_assert(
OS_PAGE_SIZE % Aal::smallest_page_size == 0,
"The smallest architectural page size must divide OS_PAGE_SIZE");
// Some system headers (e.g. Linux' sys/user.h, FreeBSD's machine/param.h)
// define `PAGE_SIZE` as a macro. We don't use `PAGE_SIZE` as our variable
// name, to avoid conflicts, but if we do see a macro definition then check
// that our value matches the platform's expected value.
#ifdef PAGE_SIZE
static_assert(
PAGE_SIZE == OS_PAGE_SIZE,
"Page size from system header does not match snmalloc config page size.");
#endif
} // namespace snmalloc

58
3rdparty/snmalloc/src/pal/pal_apple.h поставляемый
Просмотреть файл

@ -4,6 +4,7 @@
# include "pal_bsd.h"
# include <mach/vm_statistics.h>
# include <utility>
namespace snmalloc
{
@ -24,65 +25,16 @@ namespace snmalloc
static constexpr uint64_t pal_features = PALBSD::pal_features;
/**
* OS specific function for zeroing memory with the Apple application
* tag id.
*
* See comment below.
*/
template<bool page_aligned = false>
void zero(void* p, size_t size)
{
if (page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size))
{
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, size));
void* r = mmap(
p,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,
pal_anon_id,
0);
if (r != MAP_FAILED)
return;
}
bzero(p, size);
}
/**
* Reserve memory with the Apple application tag id.
*
* See comment below.
*/
template<bool committed>
void* reserve(size_t size)
{
void* p = mmap(
nullptr,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
pal_anon_id,
0);
if (p == MAP_FAILED)
error("Out of memory");
return p;
}
private:
/**
* Anonymous page tag ID
* Anonymous page tag ID.
*
* Darwin platform allows to gives an ID to anonymous pages via
* the VM_MAKE_TAG's macro, from 240 up to 255 are guaranteed
* to be free of usage, however eventually a lower could be taken
* (e.g. LLVM sanitizers has 99) so we can monitor their states
* via vmmap for instance.
* via vmmap for instance. This value is provided to `mmap` as the file
* descriptor for the mapping.
*/
static constexpr int pal_anon_id = VM_MAKE_TAG(PALAnonID);
static constexpr int anonymous_memory_fd = VM_MAKE_TAG(PALAnonID);
};
} // namespace snmalloc
#endif

2
3rdparty/snmalloc/src/pal/pal_bsd.h поставляемый
Просмотреть файл

@ -33,7 +33,7 @@ namespace snmalloc
*/
void notify_not_using(void* p, size_t size) noexcept
{
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, size));
SNMALLOC_ASSERT(is_aligned_block<OS::page_size>(p, size));
madvise(p, size, MADV_FREE);
}
};

11
3rdparty/snmalloc/src/pal/pal_bsd_aligned.h поставляемый
Просмотреть файл

@ -23,18 +23,19 @@ namespace snmalloc
static constexpr uint64_t pal_features =
AlignedAllocation | PALBSD<OS>::pal_features;
static constexpr size_t minimum_alloc_size = 4096;
/**
* Reserve memory at a specific alignment.
*/
template<bool committed>
void* reserve(size_t size, size_t align) noexcept
void* reserve_aligned(size_t size) noexcept
{
// Alignment must be a power of 2.
SNMALLOC_ASSERT(align == bits::next_pow2(align));
SNMALLOC_ASSERT(size == bits::next_pow2(size));
SNMALLOC_ASSERT(size >= minimum_alloc_size);
align = bits::max<size_t>(4096, align);
size_t log2align = bits::next_pow2_bits(align);
size_t log2align = bits::next_pow2_bits(size);
void* p = mmap(
nullptr,

2
3rdparty/snmalloc/src/pal/pal_consts.h поставляемый
Просмотреть файл

@ -78,7 +78,7 @@ namespace snmalloc
/**
* List of callbacks to notify
*/
std::atomic<PalNotificationObject*> callbacks = nullptr;
std::atomic<PalNotificationObject*> callbacks{nullptr};
public:
/**

Просмотреть файл

@ -1,7 +1,6 @@
#pragma once
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
#if defined(FreeBSD_KERNEL)
extern "C"
@ -29,7 +28,7 @@ namespace snmalloc
* PAL supports.
*/
static constexpr uint64_t pal_features = AlignedAllocation;
void error(const char* const str)
[[noreturn]] void error(const char* const str)
{
panic("snmalloc error: %s", str);
}
@ -61,8 +60,12 @@ namespace snmalloc
}
template<bool committed>
void* reserve(size_t size, size_t align)
void* reserve_aligned(size_t size) noexcept
{
SNMALLOC_ASSERT(size == bits::next_pow2(size));
SNMALLOC_ASSERT(size >= minimum_alloc_size);
size_t align = size;
vm_offset_t addr;
if (vmem_xalloc(
kernel_arena,

41
3rdparty/snmalloc/src/pal/pal_haiku.h поставляемый Normal file
Просмотреть файл

@ -0,0 +1,41 @@
#pragma once
#if defined(__HAIKU__)
# include "pal_posix.h"
# include <sys/mman.h>
namespace snmalloc
{
/**
* Platform abstraction layer for Haiku. This provides features for this
* system.
*/
class PALHaiku : public PALPOSIX<PALHaiku>
{
public:
/**
* Bitmap of PalFeatures flags indicating the optional features that this
* PAL supports.
*
*/
static constexpr uint64_t pal_features = PALPOSIX::pal_features;
/**
* Haiku requires an explicit no-reserve flag in `mmap` to guarantee lazy
* commit.
*/
static constexpr int default_mmap_flags = MAP_NORESERVE;
/**
* Notify platform that we will not be needing these pages.
* Haiku does not provide madvise call per say only the posix equivalent.
*/
void notify_not_using(void* p, size_t size) noexcept
{
SNMALLOC_ASSERT(is_aligned_block<page_size>(p, size));
posix_madvise(p, size, POSIX_MADV_DONTNEED);
}
};
} // namespace snmalloc
#endif

10
3rdparty/snmalloc/src/pal/pal_linux.h поставляемый
Просмотреть файл

@ -2,7 +2,6 @@
#if defined(__linux__)
# include "../ds/bits.h"
# include "../mem/allocconfig.h"
# include "pal_posix.h"
# include <string.h>
@ -26,6 +25,9 @@ namespace snmalloc
*/
static constexpr uint64_t pal_features = PALPOSIX::pal_features;
static constexpr size_t page_size =
Aal::aal_name == PowerPC ? 0x10000 : 0x1000;
/**
* OS specific function for zeroing memory.
*
@ -41,12 +43,12 @@ namespace snmalloc
// MADV_DONTNEED. switch back to memset only for QEMU.
# ifndef SNMALLOC_QEMU_WORKAROUND
if (
(page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size)) &&
(size > SLAB_SIZE))
(page_aligned || is_aligned_block<page_size>(p, size)) &&
(size > 16 * page_size))
{
// Only use this on large allocations as memset faster, and doesn't
// introduce IPI so faster for small allocations.
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, size));
SNMALLOC_ASSERT(is_aligned_block<page_size>(p, size));
madvise(p, size, MADV_DONTNEED);
}
else

65
3rdparty/snmalloc/src/pal/pal_open_enclave.h поставляемый
Просмотреть файл

@ -1,55 +1,64 @@
#pragma once
#include "ds/address.h"
#include "ds/flaglock.h"
#include "pal_plain.h"
#include <array>
#ifdef OPEN_ENCLAVE
extern "C" const void* __oe_get_heap_base();
extern "C" const void* __oe_get_heap_end();
extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size);
extern "C" void oe_abort();
extern "C" [[noreturn]] void oe_abort();
namespace snmalloc
{
class PALOpenEnclave
{
std::atomic<void*> oe_base = nullptr;
/// Base of OE heap
static inline void* heap_base = nullptr;
/// Size of OE heap
static inline size_t heap_size;
// This is infrequently used code, a spin lock simplifies the code
// considerably, and should never be on the fast path.
static inline std::atomic_flag spin_lock;
public:
/**
* This will be called by oe_allocator_init to set up enclave heap bounds.
*/
static void setup_initial_range(void* base, void* end)
{
heap_size = pointer_diff(base, end);
heap_base = base;
}
/**
* Bitmap of PalFeatures flags indicating the optional features that this
* PAL supports.
*/
static constexpr uint64_t pal_features = 0;
static void error(const char* const str)
static constexpr size_t page_size = 0x1000;
[[noreturn]] static void error(const char* const str)
{
UNUSED(str);
oe_abort();
}
template<bool committed>
void* reserve(size_t size) noexcept
static std::pair<void*, size_t>
reserve_at_least(size_t request_size) noexcept
{
if (oe_base == 0)
{
void* dummy = NULL;
// If this CAS fails then another thread has initialised this.
oe_base.compare_exchange_strong(
dummy, const_cast<void*>(__oe_get_heap_base()));
}
// First call returns the entire address space
// subsequent calls return {nullptr, 0}
FlagLock lock(spin_lock);
if (request_size > heap_size)
return {nullptr, 0};
void* old_base = oe_base;
void* next_base;
auto end = __oe_get_heap_end();
do
{
auto new_base = old_base;
next_base = pointer_offset(new_base, size);
if (next_base > end)
return nullptr;
} while (!oe_base.compare_exchange_strong(old_base, next_base));
return old_base;
auto result = std::make_pair(heap_base, heap_size);
heap_size = 0;
return result;
}
template<bool page_aligned = false>

1
3rdparty/snmalloc/src/pal/pal_plain.h поставляемый
Просмотреть файл

@ -1,7 +1,6 @@
#pragma once
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
namespace snmalloc
{

119
3rdparty/snmalloc/src/pal/pal_posix.h поставляемый
Просмотреть файл

@ -1,14 +1,16 @@
#pragma once
#include "../ds/address.h"
#include "../mem/allocconfig.h"
#include <execinfo.h>
#if defined(BACKTRACE_HEADER)
# include BACKTRACE_HEADER
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/mman.h>
#include <unistd.h>
#include <utility>
extern "C" int puts(const char* str);
@ -29,6 +31,64 @@ namespace snmalloc
template<class OS>
class PALPOSIX
{
/**
* Helper class to access the `default_mmap_flags` field of `OS` if one
* exists or a default value if not. This provides the default version,
* which is used if `OS::default_mmap_flags` does not exist.
*/
template<typename T, typename = int>
struct DefaultMMAPFlags
{
/**
* If `OS::default_mmap_flags` does not exist, use 0. This value is
* or'd with the other mmap flags and so a value of 0 is a no-op.
*/
static const int flags = 0;
};
/**
* Helper class to access the `default_mmap_flags` field of `OS` if one
* exists or a default value if not. This provides the version that
* accesses the field, allowing other PALs to provide extra arguments to
* the `mmap` calls used here.
*/
template<typename T>
struct DefaultMMAPFlags<T, decltype((void)T::default_mmap_flags, 0)>
{
static const int flags = T::default_mmap_flags;
};
/**
* Helper class to allow `OS` to provide the file descriptor used for
* anonymous memory. This is the default version, which provides the POSIX
* default of -1.
*/
template<typename T, typename = int>
struct AnonFD
{
/**
* If `OS::anonymous_memory_fd` does not exist, use -1. This value is
* defined by POSIX.
*/
static const int fd = -1;
};
/**
* Helper class to allow `OS` to provide the file descriptor used for
* anonymous memory. This exposes the `anonymous_memory_fd` field in `OS`.
*/
template<typename T>
struct AnonFD<T, decltype((void)T::anonymous_memory_fd, 0)>
{
/**
* The PAL's provided file descriptor for anonymous memory. This is
* used, for example, on Apple platforms, which use the file descriptor
* in a `MAP_ANONYMOUS` mapping to encode metadata about the owner of the
* mapping.
*/
static const int fd = T::anonymous_memory_fd;
};
public:
/**
* Bitmap of PalFeatures flags indicating the optional features that this
@ -38,8 +98,11 @@ namespace snmalloc
*/
static constexpr uint64_t pal_features = LazyCommit;
static constexpr size_t page_size = 0x1000;
static void print_stack_trace()
{
#ifdef BACKTRACE_HEADER
constexpr int SIZE = 1024;
void* buffer[SIZE];
auto nptrs = backtrace(buffer, SIZE);
@ -47,12 +110,13 @@ namespace snmalloc
backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO);
puts("");
fflush(stdout);
#endif
}
/**
* Report a fatal error an exit.
*/
static void error(const char* const str) noexcept
[[noreturn]] static void error(const char* const str) noexcept
{
puts(str);
print_stack_trace();
@ -69,7 +133,7 @@ namespace snmalloc
*/
void notify_not_using(void* p, size_t size) noexcept
{
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, size));
SNMALLOC_ASSERT(is_aligned_block<OS::page_size>(p, size));
#ifdef USE_POSIX_COMMIT_CHECKS
mprotect(p, size, PROT_NONE);
#else
@ -89,7 +153,7 @@ namespace snmalloc
void notify_using(void* p, size_t size) noexcept
{
SNMALLOC_ASSERT(
is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
is_aligned_block<OS::page_size>(p, size) || (zero_mem == NoZero));
#ifdef USE_POSIX_COMMIT_CHECKS
mprotect(p, size, PROT_READ | PROT_WRITE);
@ -116,15 +180,15 @@ namespace snmalloc
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size))
if (page_aligned || is_aligned_block<OS::page_size>(p, size))
{
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, size));
SNMALLOC_ASSERT(is_aligned_block<OS::page_size>(p, size));
void* r = mmap(
p,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,
-1,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED | DefaultMMAPFlags<OS>::flags,
AnonFD<OS>::fd,
0);
if (r != MAP_FAILED)
@ -143,21 +207,32 @@ namespace snmalloc
* POSIX does not define a portable interface for specifying alignment
* greater than a page.
*/
template<bool committed>
void* reserve(size_t size) noexcept
std::pair<void*, size_t> reserve_at_least(size_t size) noexcept
{
void* p = mmap(
nullptr,
size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1,
0);
SNMALLOC_ASSERT(size == bits::next_pow2(size));
if (p == MAP_FAILED)
OS::error("Out of memory");
// Magic number for over-allocating chosen by the Pal
// These should be further refined based on experiments.
constexpr size_t min_size =
bits::is64() ? bits::one_at_bit(32) : bits::one_at_bit(28);
return p;
for (size_t size_request = bits::max(size, min_size);
size_request >= size;
size_request = size_request / 2)
{
void* p = mmap(
nullptr,
size_request,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | DefaultMMAPFlags<OS>::flags,
AnonFD<OS>::fd,
0);
if (p != MAP_FAILED)
return {p, size_request};
}
OS::error("Out of memory");
}
};
} // namespace snmalloc

29
3rdparty/snmalloc/src/pal/pal_solaris.h поставляемый Normal file
Просмотреть файл

@ -0,0 +1,29 @@
#pragma once
#if defined(__sun)
# include "pal_posix.h"
namespace snmalloc
{
/**
* Platform abstraction layer for Solaris. This provides features for this
* system.
*/
class PALSolaris : public PALPOSIX<PALSolaris>
{
public:
/**
* Bitmap of PalFeatures flags indicating the optional features that this
* PAL supports.
*
*/
static constexpr uint64_t pal_features = PALPOSIX::pal_features;
/**
* Solaris requires an explicit no-reserve flag in `mmap` to guarantee lazy
* commit.
*/
static constexpr int default_mmap_flags = MAP_NORESERVE;
};
} // namespace snmalloc
#endif

78
3rdparty/snmalloc/src/pal/pal_windows.h поставляемый
Просмотреть файл

@ -2,7 +2,6 @@
#include "../ds/address.h"
#include "../ds/bits.h"
#include "../mem/allocconfig.h"
#ifdef _WIN32
# ifndef _MSC_VER
@ -78,11 +77,15 @@ namespace snmalloc
* PAL supports. This PAL supports low-memory notifications.
*/
static constexpr uint64_t pal_features = LowMemoryNotification
# if defined(PLATFORM_HAS_VIRTUALALLOC2)
# if defined(PLATFORM_HAS_VIRTUALALLOC2) && !defined(USE_SYSTEMATIC_TESTING)
| AlignedAllocation
# endif
;
static constexpr size_t minimum_alloc_size = 0x10000;
static constexpr size_t page_size = 0x1000;
/**
* Check whether the low memory state is still in effect. This is an
* expensive operation and should not be on any fast paths.
@ -105,7 +108,7 @@ namespace snmalloc
low_memory_callbacks.register_notification(callback);
}
static void error(const char* const str)
[[noreturn]] static void error(const char* const str)
{
puts(str);
fflush(stdout);
@ -115,7 +118,7 @@ namespace snmalloc
/// Notify platform that we will not be using these pages
void notify_not_using(void* p, size_t size) noexcept
{
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, size));
SNMALLOC_ASSERT(is_aligned_block<page_size>(p, size));
BOOL ok = VirtualFree(p, size, MEM_DECOMMIT);
@ -128,7 +131,7 @@ namespace snmalloc
void notify_using(void* p, size_t size) noexcept
{
SNMALLOC_ASSERT(
is_aligned_block<OS_PAGE_SIZE>(p, size) || (zero_mem == NoZero));
is_aligned_block<page_size>(p, size) || (zero_mem == NoZero));
void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE);
@ -140,9 +143,9 @@ namespace snmalloc
template<bool page_aligned = false>
void zero(void* p, size_t size) noexcept
{
if (page_aligned || is_aligned_block<OS_PAGE_SIZE>(p, size))
if (page_aligned || is_aligned_block<page_size>(p, size))
{
SNMALLOC_ASSERT(is_aligned_block<OS_PAGE_SIZE>(p, size));
SNMALLOC_ASSERT(is_aligned_block<page_size>(p, size));
notify_not_using(p, size);
notify_using<YesZero>(p, size);
}
@ -156,13 +159,16 @@ namespace snmalloc
static size_t bump_ptr = (size_t)0x4000'0000'0000;
return bump_ptr;
}
template<bool committed>
void* reserve(size_t size) noexcept
{
DWORD flags = MEM_RESERVE;
if (committed)
flags |= MEM_COMMIT;
std::pair<void*, size_t> reserve_at_least(size_t size) noexcept
{
// Magic number for over-allocating chosen by the Pal
// These should be further refined based on experiments.
constexpr size_t min_size =
bits::is64() ? bits::one_at_bit(32) : bits::one_at_bit(28);
auto size_request = bits::max(size, min_size);
DWORD flags = MEM_RESERVE;
size_t retries = 1000;
void* p;
@ -170,34 +176,30 @@ namespace snmalloc
do
{
p = VirtualAlloc(
(void*)systematic_bump_ptr(), size, flags, PAGE_READWRITE);
(void*)systematic_bump_ptr(), size_request, flags, PAGE_READWRITE);
systematic_bump_ptr() += size;
systematic_bump_ptr() += size_request;
retries--;
} while (p == nullptr && retries > 0);
return p;
return {p, size_request};
}
# elif defined(PLATFORM_HAS_VIRTUALALLOC2)
template<bool committed>
void* reserve(size_t size, size_t align) noexcept
void* reserve_aligned(size_t size) noexcept
{
SNMALLOC_ASSERT(size == bits::next_pow2(size));
SNMALLOC_ASSERT(size >= minimum_alloc_size);
DWORD flags = MEM_RESERVE;
if (committed)
flags |= MEM_COMMIT;
// Windows doesn't let you request memory less than 64KB aligned. Most
// operating systems will simply give you something more aligned than you
// ask for, but Windows complains about invalid parameters.
const size_t min_align = 64 * 1024;
if (align < min_align)
align = min_align;
// If we're on Windows 10 or newer, we can use the VirtualAlloc2
// function. The FromApp variant is useable by UWP applications and
// cannot allocate executable memory.
MEM_ADDRESS_REQUIREMENTS addressReqs = {NULL, NULL, align};
MEM_ADDRESS_REQUIREMENTS addressReqs = {NULL, NULL, size};
MEM_EXTENDED_PARAMETER param = {
{MemExtendedParameterAddressRequirements, 0}, {0}};
@ -214,20 +216,26 @@ namespace snmalloc
return ret;
}
# else
template<bool committed>
void* reserve(size_t size) noexcept
std::pair<void*, size_t> reserve_at_least(size_t size) noexcept
{
DWORD flags = MEM_RESERVE;
SNMALLOC_ASSERT(size == bits::next_pow2(size));
if (committed)
flags |= MEM_COMMIT;
void* ret = VirtualAlloc(nullptr, size, flags, PAGE_READWRITE);
if (ret == nullptr)
// Magic number for over-allocating chosen by the Pal
// These should be further refined based on experiments.
constexpr size_t min_size =
bits::is64() ? bits::one_at_bit(32) : bits::one_at_bit(28);
for (size_t size_request = bits::max(size, min_size);
size_request >= size;
size_request = size_request / 2)
{
error("Failed to allocate memory\n");
void* ret =
VirtualAlloc(nullptr, size_request, MEM_RESERVE, PAGE_READWRITE);
if (ret != nullptr)
{
return std::pair(ret, size_request);
}
}
return ret;
error("Failed to allocate memory\n");
}
# endif
};

Просмотреть файл

@ -1,7 +1,6 @@
#define SNMALLOC_SGX
#define OPEN_ENCLAVE
#define OPEN_ENCLAVE_SIMULATION
#define USE_RESERVE_MULTIPLE 1
#include <iostream>
#include <snmalloc.h>
@ -10,18 +9,6 @@
#endif
#define assert please_use_SNMALLOC_ASSERT
void* oe_base;
void* oe_end;
extern "C" const void* __oe_get_heap_base()
{
return oe_base;
}
extern "C" const void* __oe_get_heap_end()
{
return oe_end;
}
extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size)
{
UNUSED(p_size);
@ -43,8 +30,9 @@ int main()
// For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example.
size_t large_class = 28 - SUPERSLAB_BITS;
size_t size = 1ULL << (SUPERSLAB_BITS + large_class);
oe_base = mp.reserve<true>(large_class);
oe_end = (uint8_t*)oe_base + size;
void* oe_base = mp.reserve<true>(large_class);
void* oe_end = (uint8_t*)oe_base + size;
PALOpenEnclave::setup_initial_range(oe_base, oe_end);
std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;
auto a = ThreadAlloc::get();

Просмотреть файл

@ -15,22 +15,44 @@ void check_result(size_t size, size_t align, void* p, int err, bool null)
{
if (p != nullptr)
abort();
}
else
{
if (our_malloc_usable_size(p) < size)
abort();
if (static_cast<size_t>(reinterpret_cast<uintptr_t>(p) % align) != 0)
abort();
our_free(p);
return;
}
const auto alloc_size = our_malloc_usable_size(p);
const auto expected_size = round_size(size);
if ((align == 1) && (alloc_size != expected_size))
{
printf(
"Usable size is %zu, but required to be %zu.\n",
alloc_size,
expected_size);
abort();
}
if ((align != 1) && (alloc_size < expected_size))
{
printf(
"Usable size is %zu, but required to be at least %zu.\n",
alloc_size,
expected_size);
abort();
}
if (static_cast<size_t>(reinterpret_cast<uintptr_t>(p) % align) != 0)
{
printf(
"Address is 0x%zx, but required to be aligned to 0x%zx.\n",
reinterpret_cast<uintptr_t>(p),
align);
abort();
}
our_free(p);
}
void test_calloc(size_t nmemb, size_t size, int err, bool null)
{
fprintf(stderr, "calloc(%d, %d)\n", (int)nmemb, (int)size);
fprintf(stderr, "calloc(%zu, %zu)\n", nmemb, size);
errno = 0;
void* p = our_calloc(nmemb, size);
@ -47,7 +69,11 @@ void test_calloc(size_t nmemb, size_t size, int err, bool null)
void test_realloc(void* p, size_t size, int err, bool null)
{
fprintf(stderr, "realloc(%p(%d), %d)\n", p, int(size), (int)size);
size_t old_size = 0;
if (p != nullptr)
old_size = our_malloc_usable_size(p);
fprintf(stderr, "realloc(%p(%zu), %zu)\n", p, old_size, size);
errno = 0;
auto new_p = our_realloc(p, size);
// Realloc failure case, deallocate original block
@ -58,7 +84,7 @@ void test_realloc(void* p, size_t size, int err, bool null)
void test_posix_memalign(size_t size, size_t align, int err, bool null)
{
fprintf(stderr, "posix_memalign(&p, %d, %d)\n", (int)align, (int)size);
fprintf(stderr, "posix_memalign(&p, %zu, %zu)\n", align, size);
void* p = nullptr;
errno = our_posix_memalign(&p, align, size);
check_result(size, align, p, err, null);
@ -66,7 +92,7 @@ void test_posix_memalign(size_t size, size_t align, int err, bool null)
void test_memalign(size_t size, size_t align, int err, bool null)
{
fprintf(stderr, "memalign(%d, %d)\n", (int)align, (int)size);
fprintf(stderr, "memalign(%zu, %zu)\n", align, size);
errno = 0;
void* p = our_memalign(align, size);
check_result(size, align, p, err, null);
@ -81,9 +107,19 @@ int main(int argc, char** argv)
constexpr int SUCCESS = 0;
test_realloc(our_malloc(64), 4194304, SUCCESS, false);
for (sizeclass_t sc = 0; sc < (SUPERSLAB_BITS + 4); sc++)
{
const size_t size = 1ULL << sc;
printf("malloc: %zu\n", size);
check_result(size, 1, our_malloc(size), SUCCESS, false);
check_result(size + 1, 1, our_malloc(size + 1), SUCCESS, false);
}
test_calloc(0, 0, SUCCESS, false);
for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
const size_t size = sizeclass_to_size(sc);
@ -97,11 +133,37 @@ int main(int argc, char** argv)
test_calloc(n, 0, SUCCESS, false);
}
test_calloc(0, size, SUCCESS, false);
}
for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
const size_t size = sizeclass_to_size(sc);
test_realloc(our_malloc(size), size, SUCCESS, false);
test_realloc(our_malloc(size), 0, SUCCESS, true);
test_realloc(nullptr, size, SUCCESS, false);
test_realloc(our_malloc(size), (size_t)-1, ENOMEM, true);
for (sizeclass_t sc2 = 0; sc2 < NUM_SIZECLASSES; sc2++)
{
const size_t size2 = sizeclass_to_size(sc2);
test_realloc(our_malloc(size), size2, SUCCESS, false);
test_realloc(our_malloc(size + 1), size2, SUCCESS, false);
}
}
for (sizeclass_t sc = 0; sc < (SUPERSLAB_BITS + 4); sc++)
{
const size_t size = 1ULL << sc;
test_realloc(our_malloc(size), size, SUCCESS, false);
test_realloc(our_malloc(size), 0, SUCCESS, true);
test_realloc(nullptr, size, SUCCESS, false);
test_realloc(our_malloc(size), (size_t)-1, ENOMEM, true);
for (sizeclass_t sc2 = 0; sc2 < (SUPERSLAB_BITS + 4); sc2++)
{
const size_t size2 = 1ULL << sc2;
printf("size1: %zu, size2:%zu\n", size, size2);
test_realloc(our_malloc(size), size2, SUCCESS, false);
test_realloc(our_malloc(size + 1), size2, SUCCESS, false);
}
}
test_posix_memalign(0, 0, EINVAL, true);
@ -111,7 +173,7 @@ int main(int argc, char** argv)
for (size_t align = sizeof(uintptr_t); align <= SUPERSLAB_SIZE * 8;
align <<= 1)
{
for (snmalloc::sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++)
{
const size_t size = sizeclass_to_size(sc);
test_posix_memalign(size, align, SUCCESS, false);

Просмотреть файл

@ -4,9 +4,76 @@
#include <test/setup.h>
#include <test/xoroshiro.h>
#include <unordered_set>
#if defined(__linux__) && !defined(SNMALLOC_QEMU_WORKAROUND)
/*
* We only test allocations with limited AS on linux for now.
* It should be a good representative for POSIX systems.
* QEMU `setrlimit64` does not behave as the same as native linux,
* so we need to exclude it from such tests.
*/
# include <sys/resource.h>
# include <sys/sysinfo.h>
# include <unistd.h>
# include <wait.h>
# define TEST_LIMITED
# define KiB (1024ull)
# define MiB (KiB * KiB)
# define GiB (KiB * MiB)
#endif
using namespace snmalloc;
#ifdef TEST_LIMITED
void test_limited(rlim64_t as_limit, size_t& count)
{
auto pid = fork();
if (!pid)
{
auto limit = rlimit64{.rlim_cur = as_limit, .rlim_max = RLIM64_INFINITY};
if (setrlimit64(RLIMIT_AS, &limit))
{
std::abort();
}
if (getrlimit64(RLIMIT_AS, &limit))
{
std::abort();
}
std::cout << "limiting memory to " << limit.rlim_cur / KiB << " KiB"
<< std::endl;
struct sysinfo info
{};
if (sysinfo(&info))
{
std::abort();
}
std::cout << "host freeram: " << info.freeram / KiB << " KiB" << std::endl;
// set the allocation size to the minimum value among:
// 2GiB, 1/8 of the AS limit, 1/8 of the Free RAM
auto upper_bound =
std::min(static_cast<unsigned long long>(limit.rlim_cur >> 3u), 2 * GiB);
upper_bound = std::min(
upper_bound, static_cast<unsigned long long>(info.freeram >> 3u));
std::cout << "trying to alloc " << upper_bound / KiB << " KiB" << std::endl;
auto alloc = ThreadAlloc::get();
std::cout << "allocator initialised" << std::endl;
auto chunk = alloc->alloc(upper_bound);
alloc->dealloc(chunk);
std::cout << "success" << std::endl;
std::exit(0);
}
else
{
int status;
waitpid(pid, &status, 0);
if (status)
{
std::cout << "failed" << std::endl;
count++;
}
}
}
#endif
void test_alloc_dealloc_64k()
{
auto alloc = ThreadAlloc::get();
@ -221,7 +288,7 @@ void test_external_pointer_large()
for (size_t i = 0; i < count; i++)
{
size_t b = snmalloc::bits::is64() ? 28 : 26;
size_t b = SUPERSLAB_BITS + 3;
size_t rand = r.next() & ((1 << b) - 1);
size_t size = (1 << 24) + rand;
total_size += size;
@ -313,7 +380,20 @@ void test_calloc_large_bug()
int main(int argc, char** argv)
{
setup();
#ifdef TEST_LIMITED
size_t count = 0;
test_limited(512 * MiB, count);
test_limited(2 * GiB, count);
test_limited(
8 *
GiB, // 8 * GiB is large enough for a loose upper-bound of our allocations
count);
if (count)
{
std::cout << count << " attempts failed out of 3" << std::endl;
std::abort();
}
#endif
#ifdef USE_SYSTEMATIC_TESTING
opt::Opt opt(argc, argv);
size_t seed = opt.is<size_t>("--seed", 0);
@ -333,6 +413,5 @@ int main(int argc, char** argv)
test_external_pointer();
test_alloc_16M();
test_calloc_16M();
return 0;
}

Просмотреть файл

@ -18,7 +18,7 @@ void test_align_size()
size < snmalloc::sizeclass_to_size(snmalloc::NUM_SIZECLASSES - 1);
size++)
{
size_t rsize = snmalloc::sizeclass_to_size(size_to_sizeclass(size));
size_t rsize = snmalloc::round_size(size);
if (rsize < size)
{

Просмотреть файл

@ -1,11 +1,14 @@
#undef IS_ADDRESS_SPACE_CONSTRAINED
#undef SNMALLOC_USE_LARGE_CHUNKS
#define OPEN_ENCLAVE
#define OPEN_ENCLAVE_SIMULATION
#define USE_RESERVE_MULTIPLE 1
#define NO_BOOTSTRAP_ALLOCATOR
#define IS_ADDRESS_SPACE_CONSTRAINED
#define SNMALLOC_EXPOSE_PAGEMAP
#define SNMALLOC_NAME_MANGLE(a) enclave_##a
// Redefine the namespace, so we can have two versions.
#define snmalloc snmalloc_enclave
#include "../../../override/malloc.cc"
extern "C" void oe_allocator_init(void* base, void* end)
{
snmalloc_enclave::PALOpenEnclave::setup_initial_range(base, end);
}

Просмотреть файл

@ -1,4 +1,7 @@
#undef IS_ADDRESS_SPACE_CONSTRAINED
// Remove parameters feed from test harness
#undef SNMALLOC_USE_LARGE_CHUNKS
#undef SNMALLOC_USE_SMALL_CHUNKS
#define SNMALLOC_NAME_MANGLE(a) host_##a
#define NO_BOOTSTRAP_ALLOCATOR
#define SNMALLOC_EXPOSE_PAGEMAP

Просмотреть файл

@ -5,18 +5,6 @@
#include <string.h>
#include <test/setup.h>
void* oe_base;
void* oe_end;
extern "C" const void* __oe_get_heap_base()
{
return oe_base;
}
extern "C" const void* __oe_get_heap_end()
{
return oe_end;
}
extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size)
{
UNUSED(p_size);
@ -28,6 +16,7 @@ extern "C" void oe_abort()
abort();
}
extern "C" void oe_allocator_init(void* base, void* end);
extern "C" void* host_malloc(size_t);
extern "C" void host_free(void*);
@ -51,8 +40,9 @@ int main()
// For 1MiB superslabs, SUPERSLAB_BITS + 2 is not big enough for the example.
size_t large_class = 26 - SUPERSLAB_BITS;
size_t size = 1ULL << (SUPERSLAB_BITS + large_class);
oe_base = mp.reserve<true>(large_class);
oe_end = (uint8_t*)oe_base + size;
void* oe_base = mp.reserve<true>(large_class);
void* oe_end = (uint8_t*)oe_base + size;
oe_allocator_init(oe_base, oe_end);
std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl;
// Call these functions to trigger asserts if the cast-to-self doesn't work.

Просмотреть файл

@ -86,7 +86,7 @@ void test_tasks_f(size_t id)
*res = size;
size_t* out =
contention[n % swapsize].exchange(res, std::memory_order_relaxed);
contention[n % swapsize].exchange(res, std::memory_order_acq_rel);
if (out != nullptr)
{

Просмотреть файл

@ -133,7 +133,7 @@
"type": "git",
"git": {
"repositoryUrl": "https://github.com/microsoft/snmalloc",
"commitHash": "2b92574123db1d1674f3df9c6dfb3ec10f05ebb3"
"commitHash": "4e1f5829a754ab9c170ec090979d3a670d2d5d1a"
}
}
},