Bug 1533522 - Truncate Buffer/Texture on GL_OOM. r=lsalzman

Differential Revision: https://phabricator.services.mozilla.com/D34982

--HG--
extra : moz-landing-system : lando

dom/canvas/WebGLBuffer.cpp

@@ -127,6 +127,11 @@ void WebGLBuffer::BufferData(GLenum target, uint64_t size, const void* data,
     if (error) {
       MOZ_ASSERT(error == LOCAL_GL_OUT_OF_MEMORY);
       mContext->ErrorOutOfMemory("Error from driver: 0x%04x", error);
+
+      // Truncate
+      mByteLength = 0;
+      mFetchInvalidator.InvalidateCaches();
+      mIndexCache = nullptr;
       return;
     }
   } else {

dom/canvas/WebGLRenderbuffer.cpp

@@ -181,6 +181,11 @@ void WebGLRenderbuffer::RenderbufferStorage(uint32_t samples,
   const GLenum error = DoRenderbufferStorage(samples, usage, width, height);
   if (error) {
     mContext->GenerateWarning("Unexpected error %s", EnumString(error).c_str());
+    if (error == LOCAL_GL_OUT_OF_MEMORY) {
+      // Truncate.
+      mImageInfo = {};
+      InvalidateCaches();
+    }
     return;
   }
 

dom/canvas/WebGLTexture.cpp

@@ -1029,6 +1029,13 @@ void WebGLTexture::TexParameter(TexTarget texTarget, GLenum pname,
     mContext->gl->fTexParameterf(texTarget.get(), pname, clamped.f);
 }
 
+void WebGLTexture::Truncate() {
+  for (auto& cur : mImageInfoArr) {
+    cur = {};
+  }
+  InvalidateCaches();
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_0(WebGLTexture)

dom/canvas/WebGLTexture.h

@@ -311,6 +311,7 @@ class WebGLTexture final : public nsWrapperCache,
   void PopulateMipChain(uint32_t maxLevel);
   bool IsMipAndCubeComplete(uint32_t maxLevel, bool ensureInit,
                             bool* out_initFailed) const;
+  void Truncate();
 
   bool IsCubeMap() const { return (mTarget == LOCAL_GL_TEXTURE_CUBE_MAP); }
 };

dom/canvas/WebGLTextureUpload.cpp

@@ -1095,6 +1095,7 @@ void WebGLTexture::TexStorage(TexTarget target, GLsizei levels,
 
   if (error == LOCAL_GL_OUT_OF_MEMORY) {
     mContext->ErrorOutOfMemory("Ran out of memory during texture allocation.");
+    Truncate();
     return;
   }
   if (error) {
@@ -1232,6 +1233,7 @@ void WebGLTexture::TexImage(TexImageTarget target, GLint level,
 
   if (glError == LOCAL_GL_OUT_OF_MEMORY) {
     mContext->ErrorOutOfMemory("Driver ran out of memory during upload.");
+    Truncate();
     return;
   }
 
@@ -1312,6 +1314,7 @@ void WebGLTexture::TexSubImage(TexImageTarget target, GLint level,
 
   if (glError == LOCAL_GL_OUT_OF_MEMORY) {
     mContext->ErrorOutOfMemory("Driver ran out of memory during upload.");
+    Truncate();
     return;
   }
 
@@ -1418,6 +1421,7 @@ void WebGLTexture::CompressedTexImage(TexImageTarget target, GLint level,
   mContext->OnDataAllocCall();
   if (error == LOCAL_GL_OUT_OF_MEMORY) {
     mContext->ErrorOutOfMemory("Ran out of memory during upload.");
+    Truncate();
     return;
   }
   if (error) {
@@ -1564,6 +1568,7 @@ void WebGLTexture::CompressedTexSubImage(
       blob->mPtr);
   if (error == LOCAL_GL_OUT_OF_MEMORY) {
     mContext->ErrorOutOfMemory("Ran out of memory during upload.");
+    Truncate();
     return;
   }
   if (error) {
@@ -1920,7 +1925,7 @@ static bool ValidateCopyTexImageForFeedback(const WebGLContext& webgl,
 }
 
 static bool DoCopyTexOrSubImage(WebGLContext* webgl, bool isSubImage,
-                                const WebGLTexture* tex, TexImageTarget target,
+                                WebGLTexture* const tex, const TexImageTarget target,
                                 GLint level, GLint xWithinSrc, GLint yWithinSrc,
                                 uint32_t srcTotalWidth, uint32_t srcTotalHeight,
                                 const webgl::FormatUsageInfo* srcUsage,
@@ -2009,6 +2014,7 @@ static bool DoCopyTexOrSubImage(WebGLContext* webgl, bool isSubImage,
 
   if (error == LOCAL_GL_OUT_OF_MEMORY) {
     webgl->ErrorOutOfMemory("Ran out of memory during texture copy.");
+    tex->Truncate();
     return false;
   }