diff --git a/js/src/util/StringBuffer.h b/js/src/util/StringBuffer.h index 844e3fc9d23b..7a9dfdfe6a67 100644 --- a/js/src/util/StringBuffer.h +++ b/js/src/util/StringBuffer.h @@ -7,6 +7,7 @@ #ifndef util_StringBuffer_h #define util_StringBuffer_h +#include "mozilla/CheckedInt.h" #include "mozilla/MaybeOneOf.h" #include "mozilla/Utf8.h" @@ -20,6 +21,33 @@ class ParserAtomsTable; class TaggedParserAtomIndex; } // namespace frontend +namespace detail { + +// GrowEltsAggressively will multiply the space by a factor of 8 on overflow, to +// avoid very expensive memcpys for large strings (eg giant toJSON output for +// sessionstore.js). Drop back to the normal expansion policy once the buffer +// hits 128MB. +static constexpr size_t AggressiveLimit = 128 << 20; + +template +inline size_t GrowEltsAggressively(size_t aOldElts, size_t aIncr) { + mozilla::CheckedInt required = + mozilla::CheckedInt(aOldElts) + aIncr; + if (!required.isValid()) { + return 0; + } + required = mozilla::RoundUpPow2(required.value()); + required *= 8; + if (!(required * EltSize).isValid() || required.value() > AggressiveLimit) { + // Fall back to doubling behavior if the aggressive growth fails or gets too + // big. + return mozilla::detail::GrowEltsByDoubling(aOldElts, aIncr); + } + return required.value(); +}; + +} // namespace detail + class StringBufferAllocPolicy { TempAllocPolicy impl_; @@ -59,6 +87,12 @@ class StringBufferAllocPolicy { } void reportAllocOverflow() const { impl_.reportAllocOverflow(); } bool checkSimulatedOOM() const { return impl_.checkSimulatedOOM(); } + + // See ComputeGrowth in mfbt/Vector.h. + template + static size_t computeGrowth(size_t aOldElts, size_t aIncr) { + return detail::GrowEltsAggressively(aOldElts, aIncr); + } }; /* diff --git a/mfbt/Vector.h b/mfbt/Vector.h index c8cc5e2ed4e5..8cf16df0af64 100644 --- a/mfbt/Vector.h +++ b/mfbt/Vector.h @@ -34,13 +34,108 @@ namespace detail { /* * Check that the given capacity wastes the minimal amount of space if - * allocated on the heap. This means that aCapacity*sizeof(T) is as close to a + * allocated on the heap. This means that aCapacity*EltSize is as close to a * power-of-two as possible. growStorageBy() is responsible for ensuring this. */ -template +template static bool CapacityHasExcessSpace(size_t aCapacity) { - size_t size = aCapacity * sizeof(T); - return RoundUpPow2(size) - size >= sizeof(T); + size_t size = aCapacity * EltSize; + return RoundUpPow2(size) - size >= EltSize; +} + +/* + * AllocPolicy can optionally provide a `computeGrowth(size_t aOldElts, + * size_t aIncr)` method that returns the new number of elements to allocate + * when the current capacity is `aOldElts` and `aIncr` more are being + * requested. If the AllocPolicy does not have such a method, a fallback + * will be used that mostly will just round the new requested capacity up to + * the next power of two, which results in doubling capacity for the most part. + * + * If the new size would overflow some limit, `computeGrowth` returns 0. + * + * A simpler way would be to make computeGrowth() part of the API for all + * AllocPolicy classes, but this turns out to be rather complex because + * mozalloc.h defines a very widely-used InfallibleAllocPolicy, and yet it + * can only be compiled in limited contexts, eg within `extern "C"` and with + * -std=c++11 rather than a later version. That makes the headers that are + * necessary for the computation unavailable (eg mfbt/MathAlgorithms.h). + */ + +// Fallback version. +template +inline size_t GrowEltsByDoubling(size_t aOldElts, size_t aIncr) { + /* + * When choosing a new capacity, its size in bytes should is as close to 2**N + * bytes as possible. 2**N-sized requests are best because they are unlikely + * to be rounded up by the allocator. Asking for a 2**N number of elements + * isn't as good, because if EltSize is not a power-of-two that would + * result in a non-2**N request size. + */ + + if (aIncr == 1) { + if (aOldElts == 0) { + return 1; + } + + /* This case occurs in ~15--20% of the calls to Vector::growStorageBy. */ + + /* + * Will aOldSize * 4 * sizeof(T) overflow? This condition limits a + * collection to 1GB of memory on a 32-bit system, which is a reasonable + * limit. It also ensures that + * + * static_cast(end()) - static_cast(begin()) + * + * for a Vector doesn't overflow ptrdiff_t (see bug 510319). + */ + if (MOZ_UNLIKELY(aOldElts & + mozilla::tl::MulOverflowMask<4 * EltSize>::value)) { + return 0; + } + + /* + * If we reach here, the existing capacity will have a size that is already + * as close to 2^N as sizeof(T) will allow. Just double the capacity, and + * then there might be space for one more element. + */ + size_t newElts = aOldElts * 2; + if (CapacityHasExcessSpace(newElts)) { + newElts += 1; + } + return newElts; + } + + /* This case occurs in ~2% of the calls to Vector::growStorageBy. */ + size_t newMinCap = aOldElts + aIncr; + + /* Did aOldSize + aIncr overflow? Will newCap * sizeof(T) overflow? */ + if (MOZ_UNLIKELY(newMinCap < aOldElts || + newMinCap & tl::MulOverflowMask<2 * EltSize>::value)) { + return 0; + } + + size_t newMinSize = newMinCap * EltSize; + size_t newSize = RoundUpPow2(newMinSize); + return newSize / EltSize; +}; + +// Fallback version. +template +static size_t ComputeGrowth(size_t aOldElts, size_t aIncr, int) { + return GrowEltsByDoubling(aOldElts, aIncr); +} + +// If the AllocPolicy provides its own computeGrowth implementation, +// use that. +template +static size_t ComputeGrowth( + size_t aOldElts, size_t aIncr, + decltype(std::declval().template computeGrowth(0, 0), + bool()) aOverloadSelector) { + size_t newElts = AP::template computeGrowth(aOldElts, aIncr); + MOZ_ASSERT(newElts <= PTRDIFF_MAX && newElts * EltSize <= PTRDIFF_MAX, + "invalid Vector size (see bug 510319)"); + return newElts; } /* @@ -119,7 +214,7 @@ struct VectorImpl { [[nodiscard]] static inline bool growTo(Vector& aV, size_t aNewCap) { MOZ_ASSERT(!aV.usingInlineStorage()); - MOZ_ASSERT(!CapacityHasExcessSpace(aNewCap)); + MOZ_ASSERT(!CapacityHasExcessSpace(aNewCap)); T* newbuf = aV.template pod_malloc(aNewCap); if (MOZ_UNLIKELY(!newbuf)) { return false; @@ -204,7 +299,7 @@ struct VectorImpl { [[nodiscard]] static inline bool growTo(Vector& aV, size_t aNewCap) { MOZ_ASSERT(!aV.usingInlineStorage()); - MOZ_ASSERT(!CapacityHasExcessSpace(aNewCap)); + MOZ_ASSERT(!CapacityHasExcessSpace(aNewCap)); T* newbuf = aV.template pod_realloc(aV.mBegin, aV.mTail.mCapacity, aNewCap); if (MOZ_UNLIKELY(!newbuf)) { @@ -925,7 +1020,7 @@ inline bool Vector::convertToHeapStorage(size_t aNewCap) { MOZ_ASSERT(usingInlineStorage()); /* Allocate buffer. */ - MOZ_ASSERT(!detail::CapacityHasExcessSpace(aNewCap)); + MOZ_ASSERT(!detail::CapacityHasExcessSpace(aNewCap)); T* newBuf = this->template pod_malloc(aNewCap); if (MOZ_UNLIKELY(!newBuf)) { return false; @@ -946,78 +1041,27 @@ template MOZ_NEVER_INLINE bool Vector::growStorageBy(size_t aIncr) { MOZ_ASSERT(mLength + aIncr > mTail.mCapacity); - /* - * When choosing a new capacity, its size should is as close to 2**N bytes - * as possible. 2**N-sized requests are best because they are unlikely to - * be rounded up by the allocator. Asking for a 2**N number of elements - * isn't as good, because if sizeof(T) is not a power-of-two that would - * result in a non-2**N request size. - */ - size_t newCap; - if (aIncr == 1) { - if (usingInlineStorage()) { - /* This case occurs in ~70--80% of the calls to this function. */ - size_t newSize = - tl::RoundUpPow2<(kInlineCapacity + 1) * sizeof(T)>::value; - newCap = newSize / sizeof(T); - goto convert; - } - - if (mLength == 0) { - /* This case occurs in ~0--10% of the calls to this function. */ - newCap = 1; - goto grow; - } - - /* This case occurs in ~15--20% of the calls to this function. */ - - /* - * Will mLength * 4 *sizeof(T) overflow? This condition limits a vector - * to 1GB of memory on a 32-bit system, which is a reasonable limit. It - * also ensures that - * - * static_cast(end()) - static_cast(begin()) - * - * doesn't overflow ptrdiff_t (see bug 510319). - */ - if (MOZ_UNLIKELY(mLength & tl::MulOverflowMask<4 * sizeof(T)>::value)) { - this->reportAllocOverflow(); - return false; - } - - /* - * If we reach here, the existing capacity will have a size that is already - * as close to 2^N as sizeof(T) will allow. Just double the capacity, and - * then there might be space for one more element. - */ - newCap = mLength * 2; - if (detail::CapacityHasExcessSpace(newCap)) { - newCap += 1; - } - } else { - /* This case occurs in ~2% of the calls to this function. */ - size_t newMinCap = mLength + aIncr; - - /* Did mLength + aIncr overflow? Will newCap * sizeof(T) overflow? */ - if (MOZ_UNLIKELY(newMinCap < mLength || - newMinCap & tl::MulOverflowMask<2 * sizeof(T)>::value)) { - this->reportAllocOverflow(); - return false; - } - - size_t newMinSize = newMinCap * sizeof(T); - size_t newSize = RoundUpPow2(newMinSize); + if (aIncr == 1 && usingInlineStorage()) { + /* This case occurs in ~70--80% of the calls to this function. */ + constexpr size_t newSize = + tl::RoundUpPow2<(kInlineCapacity + 1) * sizeof(T)>::value; + static_assert(newSize / sizeof(T) > 0, + "overflow when exceeding inline Vector storage"); newCap = newSize / sizeof(T); + } else { + newCap = detail::ComputeGrowth(mLength, aIncr, true); + if (MOZ_UNLIKELY(newCap == 0)) { + this->reportAllocOverflow(); + return false; + } } if (usingInlineStorage()) { - convert: return convertToHeapStorage(newCap); } -grow: return Impl::growTo(*this, newCap); }