Metal: Canvas resizing causes webpage to run out of memory

For https://bugs.webkit.org/show_bug.cgi?id=232122

Introduce a maximum resident amount of memory that a single
command buffer can use, before forcing a flush.
As part of ensuring that a command buffer is ready,
check to see if this limit is exceeded. if so,
flush the command buffer, and create a new one.

Bug: angleproject:6880
Change-Id: I5e89735d05adbc174237ab79b006a75bbe89e560
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3369922
Reviewed-by: Gregg Tavares <gman@chromium.org>
Reviewed-by: Kenneth Russell <kbr@chromium.org>
Commit-Queue: Kyle Piddington <kpiddington@apple.com>

src/libANGLE/renderer/metal/ContextMtl.h

@@ -450,7 +450,7 @@ class ContextMtl : public ContextImpl, public mtl::Context
                                    gl::DrawElementsType type,
                                    const void *indices,
                                    GLsizei instanceCount);
-
+    void flushCommandBufferIfNeeded();
     void updateExtendedState(const gl::State &glState);
 
     void updateViewport(FramebufferMtl *framebufferMtl,
@@ -593,7 +593,6 @@ class ContextMtl : public ContextImpl, public mtl::Context
     IncompleteTextureSet mIncompleteTextures;
     bool mIncompleteTexturesInitialized = false;
     ProvokingVertexHelper mProvokingVertexHelper;
-    bool mPreviousRasterizerDiscardEnabledState = false;
 
     mtl::ContextDevice mContextDevice;
 };

src/libANGLE/renderer/metal/ContextMtl.mm

@@ -1591,6 +1591,14 @@ void ContextMtl::flushCommandBuffer(mtl::CommandBufferFinishOperation operation)
     mCmdBuffer.commit(operation);
 }
 
+void ContextMtl::flushCommandBufferIfNeeded()
+{
+    if (mCmdBuffer.needsFlushForDrawCallLimits())
+    {
+        flushCommandBuffer(mtl::NoWait);
+    }
+}
+
 void ContextMtl::present(const gl::Context *context, id<CAMetalDrawable> presentationDrawable)
 {
     ensureCommandBufferReady();
@@ -1742,7 +1750,6 @@ mtl::ComputeCommandEncoder *ContextMtl::getComputeCommandEncoder()
     }
 
     endEncoding(true);
-
     ensureCommandBufferReady();
 
     return &mComputeEncoder.restart();
@@ -1755,6 +1762,7 @@ mtl::ComputeCommandEncoder *ContextMtl::getIndexPreprocessingCommandEncoder()
 
 void ContextMtl::ensureCommandBufferReady()
 {
+    flushCommandBufferIfNeeded();
     mProvokingVertexHelper.ensureCommandBufferReady();
     if (!mCmdBuffer.ready())
     {

src/libANGLE/renderer/metal/IOSurfaceSurfaceMtl.mm

@@ -153,6 +153,15 @@ angle::Result IOSurfaceSurfaceMtl::ensureColorTextureCreated(const gl::Context *
         mColorTexture =
             mtl::Texture::MakeFromMetal(contextMtl->getMetalDevice().newTextureWithDescriptor(
                 texDesc, mIOSurface, mIOSurfacePlane));
+
+        if (mColorTexture)
+        {
+            size_t resourceSize = EstimateTextureSizeInBytes(
+                mColorFormat, mColorTexture->widthAt0(), mColorTexture->heightAt0(),
+                mColorTexture->depthAt0(), mColorTexture->samples(), mColorTexture->mipmapLevels());
+
+            mColorTexture->setEstimatedByteSize(resourceSize);
+        }
     }
 
     mColorRenderTarget.set(mColorTexture, mtl::kZeroNativeMipLevel, 0, mColorFormat);

src/libANGLE/renderer/metal/ImageMtl.mm

@@ -72,6 +72,14 @@ angle::Result TextureImageSiblingMtl::initImpl(DisplayMtl *displayMtl)
         mtl::Format::MetalToAngleFormatID(mNativeTexture->pixelFormat());
     mFormat = displayMtl->getPixelFormat(angleFormatId);
 
+    if (mNativeTexture)
+    {
+        size_t resourceSize = EstimateTextureSizeInBytes(
+            mFormat, mNativeTexture->widthAt0(), mNativeTexture->heightAt0(),
+            mNativeTexture->depthAt0(), mNativeTexture->samples(), mNativeTexture->mipmapLevels());
+        mNativeTexture->setEstimatedByteSize(resourceSize);
+    }
+
     mGLFormat = gl::Format(mFormat.intendedAngleFormat().glInternalFormat);
 
     mRenderable = mFormat.getCaps().depthRenderable || mFormat.getCaps().colorRenderable;

src/libANGLE/renderer/metal/SurfaceMtl.mm

@@ -51,6 +51,11 @@ angle::Result CreateOrResizeTexture(const gl::Context *context,
     if (*textureOut)
     {
         ANGLE_TRY((*textureOut)->resize(contextMtl, width, height));
+        size_t resourceSize = EstimateTextureSizeInBytes(format, width, height, 1, samples, 1);
+        if (*textureOut)
+        {
+            (*textureOut)->setEstimatedByteSize(resourceSize);
+        }
     }
     else if (samples > 1)
     {

src/libANGLE/renderer/metal/mtl_command_buffer.h

@@ -131,6 +131,8 @@ class CommandBuffer final : public WrappedObject<id<MTLCommandBuffer>>, angle::N
     void setActiveCommandEncoder(CommandEncoder *encoder);
     void invalidateActiveCommandEncoder(CommandEncoder *encoder);
 
+    bool needsFlushForDrawCallLimits() const;
+
   private:
     void set(id<MTLCommandBuffer> metalBuffer);
     void cleanup();
@@ -146,6 +148,9 @@ class CommandBuffer final : public WrappedObject<id<MTLCommandBuffer>>, angle::N
     void pushDebugGroupImpl(const std::string &marker);
     void popDebugGroupImpl();
 
+    void setResourceUsedByCommandBuffer(const ResourceRef &resource);
+    void clearResourceListAndSize();
+
     using ParentClass = WrappedObject<id<MTLCommandBuffer>>;
 
     CommandQueue &mCmdQueue;
@@ -158,10 +163,11 @@ class CommandBuffer final : public WrappedObject<id<MTLCommandBuffer>>, angle::N
 
     std::vector<std::string> mPendingDebugSigns;
     std::vector<std::pair<mtl::SharedEventRef, uint64_t>> mPendingSignalEvents;
-
     std::vector<std::string> mDebugGroups;
 
-    bool mCommitted = false;
+    std::unordered_set<id> mResourceList;
+    size_t mWorkingResourceSize = 0;
+    bool mCommitted             = false;
 };
 
 class CommandEncoder : public WrappedObject<id<MTLCommandEncoder>>, angle::NonCopyable

src/libANGLE/renderer/metal/mtl_command_buffer.mm

@@ -603,6 +603,36 @@ void CommandBuffer::present(id<CAMetalDrawable> presentationDrawable)
     [get() presentDrawable:presentationDrawable];
 }
 
+void CommandBuffer::setResourceUsedByCommandBuffer(const ResourceRef &resource)
+{
+    if (resource)
+    {
+        auto result = mResourceList.insert(resource->getID());
+        // If we were able to add a unique Metal resource ID to the list, count it.
+        //
+        // Note that we store Metal IDs here, properly retained in non-ARC environments, rather than
+        // the ResourceRefs. There are some assumptions in TextureMtl in particular about weak refs
+        // to temporary textures being cleared out eagerly. Holding on to additional references here
+        // implies that that texture is still being used, and would require additional code to clear
+        // out temporary render targets upon texture redefinition.
+        if (result.second)
+        {
+            [resource->getID() ANGLE_MTL_RETAIN];
+            mWorkingResourceSize += resource->estimatedByteSize();
+        }
+    }
+}
+
+void CommandBuffer::clearResourceListAndSize()
+{
+    for (const id &metalID : mResourceList)
+    {
+        [metalID ANGLE_MTL_RELEASE];
+    }
+    mResourceList.clear();
+    mWorkingResourceSize = 0;
+}
+
 void CommandBuffer::setWriteDependency(const ResourceRef &resource)
 {
     if (!resource)
@@ -618,11 +648,13 @@ void CommandBuffer::setWriteDependency(const ResourceRef &resource)
     }
 
     resource->setUsedByCommandBufferWithQueueSerial(mQueueSerial, true);
+    setResourceUsedByCommandBuffer(resource);
 }
 
 void CommandBuffer::setReadDependency(const ResourceRef &resource)
 {
     setReadDependency(resource.get());
+    setResourceUsedByCommandBuffer(resource);
 }
 
 void CommandBuffer::setReadDependency(Resource *resource)
@@ -642,6 +674,11 @@ void CommandBuffer::setReadDependency(Resource *resource)
     resource->setUsedByCommandBufferWithQueueSerial(mQueueSerial, false);
 }
 
+bool CommandBuffer::needsFlushForDrawCallLimits() const
+{
+    return mWorkingResourceSize > kMaximumResidentMemorySizeInBytes;
+}
+
 void CommandBuffer::restart()
 {
     uint64_t serial                                  = 0;
@@ -657,7 +694,7 @@ void CommandBuffer::restart()
     {
         pushDebugGroupImpl(marker);
     }
-
+    clearResourceListAndSize();
     ASSERT(metalCmdBuffer);
 }
 
@@ -797,6 +834,8 @@ bool CommandBuffer::commitImpl()
 
     // Do the actual commit
     [get() commit];
+    // Reset the working resource set.
+    clearResourceListAndSize();
     mCommitted = true;
     return true;
 }

src/libANGLE/renderer/metal/mtl_common.h

@@ -142,6 +142,11 @@ constexpr size_t kInlineConstDataMaxSize = 4 * 1024;
 constexpr size_t kDefaultUniformsMaxSize = 4 * 1024;
 constexpr uint32_t kMaxViewports         = 1;
 
+// Restrict in-flight resource usage to 400 MB.
+// A render pass can use more than 400MB, but the command buffer
+// will be flushed next time
+constexpr const size_t kMaximumResidentMemorySizeInBytes = 400 * 1024 * 1024;
+
 constexpr uint32_t kVertexAttribBufferStrideAlignment = 4;
 // Alignment requirement for offset passed to setVertex|FragmentBuffer
 #if TARGET_OS_OSX || TARGET_OS_MACCATALYST

src/libANGLE/renderer/metal/mtl_resources.h

@@ -71,6 +71,9 @@ class Resource : angle::NonCopyable
     bool isCPUReadMemDirty() const { return mUsageRef->cpuReadMemDirty; }
     void resetCPUReadMemDirty() { mUsageRef->cpuReadMemDirty = false; }
 
+    virtual size_t estimatedByteSize() const = 0;
+    virtual id getID() const                 = 0;
+
   protected:
     Resource();
     // Share the GPU usage ref with other resource
@@ -269,6 +272,9 @@ class Texture final : public Resource,
 
     // Explicitly sync content between CPU and GPU
     void syncContent(ContextMtl *context, mtl::BlitCommandEncoder *encoder);
+    void setEstimatedByteSize(size_t bytes) { mEstimatedByteSize = bytes; }
+    size_t estimatedByteSize() const override { return mEstimatedByteSize; }
+    id getID() const override { return get(); }
 
   private:
     using ParentClass = WrappedObject<id<MTLTexture>>;
@@ -335,6 +341,8 @@ class Texture final : public Resource,
     TextureRef mStencilView;
     // Readable copy of texture
     TextureRef mReadCopy;
+
+    size_t mEstimatedByteSize = 0;
 };
 
 class Buffer final : public Resource, public WrappedObject<id<MTLBuffer>>
@@ -383,6 +391,9 @@ class Buffer final : public Resource, public WrappedObject<id<MTLBuffer>>
     // Explicitly sync content between CPU and GPU
     void syncContent(ContextMtl *context, mtl::BlitCommandEncoder *encoder);
 
+    size_t estimatedByteSize() const override { return size(); }
+    id getID() const override { return get(); }
+
   private:
     Buffer(ContextMtl *context, bool forceUseSharedMem, size_t size, const uint8_t *data);
     Buffer(ContextMtl *context,

src/libANGLE/renderer/metal/mtl_resources.mm

@@ -289,7 +289,6 @@ angle::Result Texture::MakeTexture(ContextMtl *context,
         return angle::Result::Stop;
     }
     refOut->reset(new Texture(context, desc, mips, renderTargetOnly, allowFormatView, memoryLess));
-
     if (!refOut || !refOut->get())
     {
         ANGLE_MTL_CHECK(context, false, GL_OUT_OF_MEMORY);
@@ -299,6 +298,13 @@ angle::Result Texture::MakeTexture(ContextMtl *context,
         refOut->get()->setColorWritableMask(GetEmulatedColorWriteMask(mtlFormat));
     }
 
+    size_t estimatedBytes = EstimateTextureSizeInBytes(
+        mtlFormat, desc.width, desc.height, desc.depth, desc.sampleCount, desc.mipmapLevelCount);
+    if (refOut)
+    {
+        refOut->get()->setEstimatedByteSize(memoryLess ? 0 : estimatedBytes);
+    }
+
     return angle::Result::Continue;
 }
 
@@ -310,6 +316,7 @@ angle::Result Texture::MakeTexture(ContextMtl *context,
                                    bool renderTargetOnly,
                                    TextureRef *refOut)
 {
+
     refOut->reset(new Texture(context, desc, surfaceRef, slice, renderTargetOnly));
 
     if (!(*refOut) || !(*refOut)->get())
@@ -321,6 +328,10 @@ angle::Result Texture::MakeTexture(ContextMtl *context,
         refOut->get()->setColorWritableMask(GetEmulatedColorWriteMask(mtlFormat));
     }
 
+    size_t estimatedBytes = EstimateTextureSizeInBytes(
+        mtlFormat, desc.width, desc.height, desc.depth, desc.sampleCount, desc.mipmapLevelCount);
+    refOut->get()->setEstimatedByteSize(estimatedBytes);
+
     return angle::Result::Continue;
 }
 
@@ -454,6 +465,8 @@ Texture::Texture(Texture *original, MTLPixelFormat format)
         auto view = [original->get() newTextureViewWithPixelFormat:format];
 
         set([view ANGLE_MTL_AUTORELEASE]);
+        // Texture views consume no additional memory
+        mEstimatedByteSize = 0;
     }
 }
 
@@ -469,6 +482,8 @@ Texture::Texture(Texture *original, MTLTextureType type, NSRange mipmapLevelRang
                                                             slices:slices];
 
         set([view ANGLE_MTL_AUTORELEASE]);
+        // Texture views consume no additional memory
+        mEstimatedByteSize = 0;
     }
 }
 
@@ -487,6 +502,8 @@ Texture::Texture(Texture *original, const TextureSwizzleChannels &swizzle)
                                   swizzle:swizzle];
 
         set([view ANGLE_MTL_AUTORELEASE]);
+        // Texture views consume no additional memory
+        mEstimatedByteSize = 0;
     }
 #else
     UNREACHABLE();

src/libANGLE/renderer/metal/mtl_utils.h

@@ -158,6 +158,12 @@ MTLColorWriteMask GetEmulatedColorWriteMask(const mtl::Format &mtlFormat,
                                             bool *emulatedChannelsOut);
 MTLColorWriteMask GetEmulatedColorWriteMask(const mtl::Format &mtlFormat);
 bool IsFormatEmulated(const mtl::Format &mtlFormat);
+size_t EstimateTextureSizeInBytes(const mtl::Format &mtlFormat,
+                                  size_t width,
+                                  size_t height,
+                                  size_t depth,
+                                  size_t sampleCount,
+                                  size_t numMips);
 
 NSUInteger GetMaxRenderTargetSizeForDeviceInBytes(const mtl::ContextDevice &device);
 NSUInteger GetMaxNumberOfRenderTargetsForDevice(const mtl::ContextDevice &device);

src/libANGLE/renderer/metal/mtl_utils.mm

@@ -1181,6 +1181,36 @@ bool IsFormatEmulated(const mtl::Format &mtlFormat)
     return isFormatEmulated;
 }
 
+size_t EstimateTextureSizeInBytes(const mtl::Format &mtlFormat,
+                                  size_t width,
+                                  size_t height,
+                                  size_t depth,
+                                  size_t sampleCount,
+                                  size_t numMips)
+{
+    size_t textureSizeInBytes;
+    if (mtlFormat.getCaps().compressed)
+    {
+        GLuint textureSize;
+        gl::Extents size((int)width, (int)height, (int)depth);
+        if (!mtlFormat.intendedInternalFormat().computeCompressedImageSize(size, &textureSize))
+        {
+            return 0;
+        }
+        textureSizeInBytes = textureSize;
+    }
+    else
+    {
+        textureSizeInBytes = mtlFormat.getCaps().pixelBytes * width * height * depth * sampleCount;
+    }
+    if (numMips > 1)
+    {
+        // Estimate mipmap size.
+        textureSizeInBytes = textureSizeInBytes * 4 / 3;
+    }
+    return textureSizeInBytes;
+}
+
 MTLClearColor EmulatedAlphaClearColor(MTLClearColor color, MTLColorWriteMask colorMask)
 {
     MTLClearColor re = color;