Bug 1801021 - Use BigBuffer for DispatchCommands. r=gfx-reviewers,lsalzman

Using ipc::Shmem causes unbounded shmem use growth until e.g. a Worker
yields to the event loop. If a Worker never yields, Shmems sent to WebGLParent
are never released.

Specifically the manager (PCanvasManager) for WebGLParent calls
DestroySharedMemory, which sends/enqueues for WebGLChild's manager a
matching call to ShmemDestroyed. However, while WebGLChild refuses to spin its
event loop (such as a no-return WASM Worker), the ShmemDestroyed events
will just pile up. Closing e.g. the tab frees the shmems, but they accumulate
unbounded until the Worker yields to the event loop.

This is true for other users of ipc::Shmem (or RaiiShmem) as well, but
entrypoints other than DispatchCommands are rarer and can be handled
later similarly.

Differential Revision: https://phabricator.services.mozilla.com/D162946

dom/canvas/PWebGL.ipdl

@@ -8,6 +8,7 @@
 include protocol PCanvasManager;
 include "mozilla/layers/LayersMessageUtils.h";
 
+[MoveOnly] using class mozilla::ipc::BigBuffer from "mozilla/ipc/BigBuffer.h";
 using mozilla::layers::CompositableHandle from "mozilla/layers/LayersTypes.h";
 using mozilla::layers::SurfaceDescriptor from "mozilla/layers/LayersTypes.h";
 using std::string from "string";
@@ -56,7 +57,7 @@ parent:
 
   // -
 
-  async DispatchCommands(Shmem commands, uint64_t size);
+  async DispatchCommands(BigBuffer commands, uint64_t size);
   async Ping() returns (void_t ok);
   async TexImage(uint32_t level, uint32_t respecFormat, uvec3 offset,
                  PackingInfo pi, TexUnpackBlobDesc src);

dom/canvas/WebGLChild.cpp

@@ -25,29 +25,28 @@ void WebGLChild::ActorDestroy(ActorDestroyReason why) {
 
 Maybe<Range<uint8_t>> WebGLChild::AllocPendingCmdBytes(
     const size_t size, const size_t fyiAlignmentOverhead) {
-  if (!mPendingCmdsShmem) {
+  if (!mPendingCmdsShmem.Size()) {
     size_t capacity = mDefaultCmdsShmemSize;
     if (capacity < size) {
       capacity = size;
     }
 
-    auto shmem = webgl::RaiiShmem::Alloc(this, capacity);
-    if (!shmem) {
+    mPendingCmdsShmem = mozilla::ipc::BigBuffer::TryAlloc(capacity);
+    if (!mPendingCmdsShmem.Size()) {
       NS_WARNING("Failed to alloc shmem for AllocPendingCmdBytes.");
       return {};
     }
-    mPendingCmdsShmem = std::move(shmem);
     mPendingCmdsPos = 0;
     mPendingCmdsAlignmentOverhead = 0;
 
     if (kIsDebug) {
-      const auto range = mPendingCmdsShmem.ByteRange();
-      const auto initialOffset =
-          AlignmentOffset(kUniversalAlignment, range.begin().get());
+      const auto ptr = mPendingCmdsShmem.Data();
+      const auto initialOffset = AlignmentOffset(kUniversalAlignment, ptr);
       MOZ_ALWAYS_TRUE(!initialOffset);
     }
   }
-  const auto range = mPendingCmdsShmem.ByteRange();
+
+  const auto range = Range{mPendingCmdsShmem.AsSpan()};
 
   auto itr = range.begin() + mPendingCmdsPos;
   const auto offset = AlignmentOffset(kUniversalAlignment, itr.get());
@@ -66,10 +65,11 @@ Maybe<Range<uint8_t>> WebGLChild::AllocPendingCmdBytes(
 }
 
 void WebGLChild::FlushPendingCmds() {
-  if (!mPendingCmdsShmem) return;
+  if (!mPendingCmdsShmem.Size()) return;
 
   const auto byteSize = mPendingCmdsPos;
-  SendDispatchCommands(mPendingCmdsShmem.Extract(), byteSize);
+  SendDispatchCommands(std::move(mPendingCmdsShmem), byteSize);
+  mPendingCmdsShmem = {};
 
   mFlushedCmdInfo.flushes += 1;
   mFlushedCmdInfo.flushedCmdBytes += byteSize;

dom/canvas/WebGLChild.h

@@ -7,6 +7,7 @@
 #define WEBGLCHILD_H_
 
 #include "mozilla/dom/PWebGLChild.h"
+#include "mozilla/ipc/BigBuffer.h"
 #include "mozilla/WeakPtr.h"
 
 #include <string>
@@ -26,7 +27,7 @@ struct FlushedCmdInfo final {
 class WebGLChild final : public PWebGLChild, public SupportsWeakPtr {
   const WeakPtr<ClientWebGLContext> mContext;
   const size_t mDefaultCmdsShmemSize;
-  webgl::RaiiShmem mPendingCmdsShmem;
+  mozilla::ipc::BigBuffer mPendingCmdsShmem;
   size_t mPendingCmdsPos = 0;
   size_t mPendingCmdsAlignmentOverhead = 0;
   FlushedCmdInfo mFlushedCmdInfo;

dom/canvas/WebGLParent.cpp

@@ -31,20 +31,18 @@ WebGLParent::~WebGLParent() = default;
 
 using IPCResult = mozilla::ipc::IPCResult;
 
-IPCResult WebGLParent::RecvDispatchCommands(Shmem&& rawShmem,
+IPCResult WebGLParent::RecvDispatchCommands(BigBuffer&& shmem,
                                             const uint64_t cmdsByteSize) {
   AUTO_PROFILER_LABEL("WebGLParent::RecvDispatchCommands", GRAPHICS);
   if (!mHost) {
     return IPC_FAIL(this, "HostWebGLContext is not initialized.");
   }
 
-  auto shmem = webgl::RaiiShmem(this, std::move(rawShmem));
-
   const auto& gl = mHost->mContext->GL();
   const gl::GLContext::TlsScope tlsIsCurrent(gl);
 
   MOZ_ASSERT(cmdsByteSize);
-  const auto shmemBytes = shmem.ByteRange();
+  const auto shmemBytes = Range{shmem.AsSpan()};
   const auto byteSize = std::min<uint64_t>(shmemBytes.length(), cmdsByteSize);
   const auto cmdsBytes =
       Range<const uint8_t>{shmemBytes.begin(), shmemBytes.begin() + byteSize};

dom/canvas/WebGLParent.h

@@ -41,7 +41,7 @@ class WebGLParent : public PWebGLParent, public SupportsWeakPtr {
     return IPC_OK();
   }
 
-  IPCResult RecvDispatchCommands(mozilla::ipc::Shmem&&, uint64_t);
+  IPCResult RecvDispatchCommands(mozilla::ipc::BigBuffer&&, uint64_t);
   IPCResult RecvTexImage(uint32_t level, uint32_t respecFormat,
                          const uvec3& offset, const webgl::PackingInfo&,
                          webgl::TexUnpackBlobDesc&&);

ipc/glue/BigBuffer.cpp

@@ -32,18 +32,20 @@ const uint8_t* BigBuffer::Data() const {
              : reinterpret_cast<const uint8_t*>(mData.as<1>()->memory());
 }
 
-auto BigBuffer::AllocBuffer(size_t aSize) -> Storage {
+auto BigBuffer::TryAllocBuffer(size_t aSize) -> Maybe<Storage> {
   if (aSize <= kShmemThreshold) {
-    return AsVariant(UniqueFreePtr<uint8_t[]>{
-        reinterpret_cast<uint8_t*>(moz_xmalloc(aSize))});
+    auto mem = UniqueFreePtr<uint8_t[]>{
+        reinterpret_cast<uint8_t*>(malloc(aSize))};  // Fallible!
+    if (!mem) return {};
+    return Some(AsVariant(std::move(mem)));
   }
 
   RefPtr<SharedMemory> shmem = new SharedMemoryBasic();
   size_t capacity = SharedMemory::PageAlignedSize(aSize);
   if (!shmem->Create(capacity) || !shmem->Map(capacity)) {
-    NS_ABORT_OOM(capacity);
+    return {};
   }
-  return AsVariant(shmem);
+  return Some(AsVariant(shmem));
 }
 
 }  // namespace mozilla::ipc

ipc/glue/BigBuffer.h

@@ -19,6 +19,16 @@ class BigBuffer {
  public:
   static constexpr size_t kShmemThreshold = 64 * 1024;
 
+  static BigBuffer TryAlloc(const size_t aSize) {
+    auto ret = BigBuffer{};
+    auto data = TryAllocBuffer(aSize);
+    if (data) {
+      ret.mSize = aSize;
+      ret.mData = std::move(data.ref());
+    }
+    return ret;
+  }
+
   // Return a new BigBuffer which wraps no data.
   BigBuffer() : mSize(0), mData(NoData()) {}
 
@@ -89,8 +99,17 @@ class BigBuffer {
   // Empty storage which holds no data.
   static Storage NoData() { return AsVariant(UniqueFreePtr<uint8_t[]>{}); }
 
+  // Fallibly allocate a new storage of the given size.
+  static Maybe<Storage> TryAllocBuffer(size_t aSize);
+
   // Infallibly allocate a new storage of the given size.
-  static Storage AllocBuffer(size_t aSize);
+  static Storage AllocBuffer(size_t aSize) {
+    auto ret = TryAllocBuffer(aSize);
+    if (!ret) {
+      NS_ABORT_OOM(aSize);
+    }
+    return std::move(ret.ref());
+  }
 
   size_t mSize;
   Storage mData;