mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1814808 - Move chunk releasing out of the arena lock's critical section r=glandium 

The system calls of releasing a chunk of memory can be costly and should be
done outside the arena lock's critical section so that other threads aren't
blocked waiting for the lock.

Differential Revision: https://phabricator.services.mozilla.com/D166775
This commit is contained in:
Paul Bone 2023-02-17 04:17:24 +00:00
Родитель 70c1e0ad83
Коммит f072e9a5ee
1 изменённых файлов: 64 добавлений и 28 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

92
memory/build/mozjemalloc.cpp
Просмотреть файл

@ -1177,11 +1177,14 @@ struct arena_t {
private:
void InitChunk(arena_chunk_t* aChunk, bool aZeroed);
void DeallocChunk(arena_chunk_t* aChunk);
// This may return a chunk that should be destroyed with chunk_dealloc outside
// of the arena lock. It is not the same chunk as was passed in (since that
// chunk now becomes mSpare).
[[nodiscard]] arena_chunk_t* DeallocChunk(arena_chunk_t* aChunk);
arena_run_t* AllocRun(size_t aSize, bool aLarge, bool aZero);
void DallocRun(arena_run_t* aRun, bool aDirty);
arena_chunk_t* DallocRun(arena_run_t* aRun, bool aDirty);
[[nodiscard]] bool SplitRun(arena_run_t* aRun, size_t aSize, bool aLarge,
bool aZero);
@ -1223,10 +1226,14 @@ struct arena_t {
void* Palloc(size_t aAlignment, size_t aSize);
inline void DallocSmall(arena_chunk_t* aChunk, void* aPtr,
arena_chunk_map_t* aMapElm);
// This may return a chunk that should be destroyed with chunk_dealloc outside
// of the arena lock. It is not the same chunk as was passed in (since that
// chunk now becomes mSpare).
[[nodiscard]] inline arena_chunk_t* DallocSmall(arena_chunk_t* aChunk,
void* aPtr,
arena_chunk_map_t* aMapElm);
void DallocLarge(arena_chunk_t* aChunk, void* aPtr);
[[nodiscard]] arena_chunk_t* DallocLarge(arena_chunk_t* aChunk, void* aPtr);
void* Ralloc(void* aPtr, size_t aSize, size_t aOldSize);
@ -2679,7 +2686,7 @@ void arena_t::InitChunk(arena_chunk_t* aChunk, bool aZeroed) {
#endif
}
void arena_t::DeallocChunk(arena_chunk_t* aChunk) {
arena_chunk_t* arena_t::DeallocChunk(arena_chunk_t* aChunk) {
if (mSpare) {
if (mSpare->ndirty > 0) {
aChunk->arena->mChunksDirty.Remove(mSpare);
@ -2693,7 +2700,6 @@ void arena_t::DeallocChunk(arena_chunk_t* aChunk) {
}
#endif
chunk_dealloc((void*)mSpare, kChunkSize, ARENA_CHUNK);
mStats.mapped -= kChunkSize;
mStats.committed -= gChunkHeaderNumPages;
}
@ -2703,7 +2709,9 @@ void arena_t::DeallocChunk(arena_chunk_t* aChunk) {
// chunk in the chunks_* trees is sufficient for that purpose.
mRunsAvail.Remove(&aChunk->map[gChunkHeaderNumPages]);
arena_chunk_t* chunk_dealloc = mSpare;
mSpare = aChunk;
return chunk_dealloc;
}
arena_run_t* arena_t::AllocRun(size_t aSize, bool aLarge, bool aZero) {
@ -2840,7 +2848,7 @@ void arena_t::Purge(bool aAll) {
}
}
void arena_t::DallocRun(arena_run_t* aRun, bool aDirty) {
arena_chunk_t* arena_t::DallocRun(arena_run_t* aRun, bool aDirty) {
arena_chunk_t* chunk;
size_t size, run_ind, run_pages;
@ -2928,15 +2936,18 @@ void arena_t::DallocRun(arena_run_t* aRun, bool aDirty) {
mRunsAvail.Insert(&chunk->map[run_ind]);
// Deallocate chunk if it is now completely unused.
arena_chunk_t* chunk_dealloc = nullptr;
if ((chunk->map[gChunkHeaderNumPages].bits &
(~gPageSizeMask | CHUNK_MAP_ALLOCATED)) == gMaxLargeClass) {
DeallocChunk(chunk);
chunk_dealloc = DeallocChunk(chunk);
}
// Enforce mMaxDirty.
if (mNumDirty > mMaxDirty) {
Purge(false);
}
return chunk_dealloc;
}
void arena_t::TrimRunHead(arena_chunk_t* aChunk, arena_run_t* aRun,
@ -2953,7 +2964,13 @@ void arena_t::TrimRunHead(arena_chunk_t* aChunk, arena_run_t* aRun,
aChunk->map[pageind + head_npages].bits =
aNewSize | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
DallocRun(aRun, false);
#ifdef MOZ_DEBUG
arena_chunk_t* no_chunk =
#endif
DallocRun(aRun, false);
// This will never release a chunk as there's still at least one allocated
// run.
MOZ_ASSERT(!no_chunk);
}
void arena_t::TrimRunTail(arena_chunk_t* aChunk, arena_run_t* aRun,
@ -2969,7 +2986,14 @@ void arena_t::TrimRunTail(arena_chunk_t* aChunk, arena_run_t* aRun,
aChunk->map[pageind + npages].bits =
(aOldSize - aNewSize) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
DallocRun((arena_run_t*)(uintptr_t(aRun) + aNewSize), aDirty);
#ifdef MOZ_DEBUG
arena_chunk_t* no_chunk =
#endif
DallocRun((arena_run_t*)(uintptr_t(aRun) + aNewSize), aDirty);
// This will never release a chunk as there's still at least one allocated
// run.
MOZ_ASSERT(!no_chunk);
}
arena_run_t* arena_t::GetNonFullBinRun(arena_bin_t* aBin) {
@ -3586,8 +3610,8 @@ MOZ_NEVER_INLINE jemalloc_ptr_info_t* jemalloc_ptr_info(const void* aPtr) {
}
} // namespace Debug
void arena_t::DallocSmall(arena_chunk_t* aChunk, void* aPtr,
arena_chunk_map_t* aMapElm) {
arena_chunk_t* arena_t::DallocSmall(arena_chunk_t* aChunk, void* aPtr,
arena_chunk_map_t* aMapElm) {
arena_run_t* run;
arena_bin_t* bin;
size_t size;
@ -3603,6 +3627,7 @@ void arena_t::DallocSmall(arena_chunk_t* aChunk, void* aPtr,
arena_run_reg_dalloc(run, bin, aPtr, size);
run->mNumFree++;
arena_chunk_t* dealloc_chunk = nullptr;
if (run->mNumFree == bin->mRunNumRegions) {
// Deallocate run.
@ -3622,7 +3647,7 @@ void arena_t::DallocSmall(arena_chunk_t* aChunk, void* aPtr,
#if defined(MOZ_DIAGNOSTIC_ASSERT_ENABLED)
run->mMagic = 0;
#endif
DallocRun(run, true);
dealloc_chunk = DallocRun(run, true);
bin->mNumRuns--;
} else if (run->mNumFree == 1 && run != bin->mCurrentRun) {
// Make sure that bin->mCurrentRun always refers to the lowest
@ -3653,9 +3678,11 @@ void arena_t::DallocSmall(arena_chunk_t* aChunk, void* aPtr,
}
}
mStats.allocated_small -= size;
return dealloc_chunk;
}
void arena_t::DallocLarge(arena_chunk_t* aChunk, void* aPtr) {
arena_chunk_t* arena_t::DallocLarge(arena_chunk_t* aChunk, void* aPtr) {
MOZ_DIAGNOSTIC_ASSERT((uintptr_t(aPtr) & gPageSizeMask) == 0);
size_t pageind = (uintptr_t(aPtr) - uintptr_t(aChunk)) >> gPageSize2Pow;
size_t size = aChunk->map[pageind].bits & ~gPageSizeMask;
@ -3663,7 +3690,7 @@ void arena_t::DallocLarge(arena_chunk_t* aChunk, void* aPtr) {
MaybePoison(aPtr, size);
mStats.allocated_large -= size;
DallocRun((arena_run_t*)aPtr, true);
return DallocRun((arena_run_t*)aPtr, true);
}
static inline void arena_dalloc(void* aPtr, size_t aOffset, arena_t* aArena) {
@ -3677,18 +3704,27 @@ static inline void arena_dalloc(void* aPtr, size_t aOffset, arena_t* aArena) {
MOZ_DIAGNOSTIC_ASSERT(arena->mMagic == ARENA_MAGIC);
MOZ_RELEASE_ASSERT(!aArena || arena == aArena);
MutexAutoLock lock(arena->mLock);
size_t pageind = aOffset >> gPageSize2Pow;
arena_chunk_map_t* mapelm = &chunk->map[pageind];
MOZ_RELEASE_ASSERT((mapelm->bits & CHUNK_MAP_DECOMMITTED) == 0,
"Freeing in decommitted page.");
MOZ_RELEASE_ASSERT((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0, "Double-free?");
if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
// Small allocation.
arena->DallocSmall(chunk, aPtr, mapelm);
} else {
// Large allocation.
arena->DallocLarge(chunk, aPtr);
arena_chunk_t* chunk_dealloc_delay = nullptr;
{
MutexAutoLock lock(arena->mLock);
size_t pageind = aOffset >> gPageSize2Pow;
arena_chunk_map_t* mapelm = &chunk->map[pageind];
MOZ_RELEASE_ASSERT((mapelm->bits & CHUNK_MAP_DECOMMITTED) == 0,
"Freeing in decommitted page.");
MOZ_RELEASE_ASSERT((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0,
"Double-free?");
if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
// Small allocation.
chunk_dealloc_delay = arena->DallocSmall(chunk, aPtr, mapelm);
} else {
// Large allocation.
chunk_dealloc_delay = arena->DallocLarge(chunk, aPtr);
}
}
if (chunk_dealloc_delay) {
chunk_dealloc((void*)chunk_dealloc_delay, kChunkSize, ARENA_CHUNK);
}
}