Bug 1595395 - Use Shmem for gfx::PaintFragment so that we can handle serializing large images. r=rhunt

We serialize an empty sized fragment if memory allocation fails rather than failing serialization to avoid crashing.
CrossProcessPaint::ReceiveFragment checks for an empty fragment, and reports an error to the caller, so this can be handled.

Differential Revision: https://phabricator.services.mozilla.com/D55018

--HG--
extra : moz-landing-system : lando

gfx/ipc/CrossProcessPaint.h

@@ -76,7 +76,7 @@ class PaintFragment final {
   PaintFragment& operator=(PaintFragment&&) = default;
 
  protected:
-  friend struct IPC::ParamTraits<PaintFragment>;
+  friend struct mozilla::ipc::IPDLParamTraits<PaintFragment>;
   friend CrossProcessPaint;
 
   typedef mozilla::ipc::ByteBuf ByteBuf;

gfx/ipc/GfxMessageUtils.h

@@ -26,6 +26,8 @@
 #include "nsRegion.h"
 #include "mozilla/Array.h"
 #include "mozilla/layers/VideoBridgeUtils.h"
+#include "mozilla/ipc/ProtocolUtils.h"
+#include "mozilla/ipc/Shmem.h"
 
 #include <stdint.h>
 
@@ -1173,23 +1175,60 @@ struct ParamTraits<mozilla::Array<T, Length>> {
   }
 };
 
+} /* namespace IPC */
+
+namespace mozilla {
+namespace ipc {
+
 template <>
-struct ParamTraits<mozilla::gfx::PaintFragment> {
+struct IPDLParamTraits<gfx::PaintFragment> {
   typedef mozilla::gfx::PaintFragment paramType;
-  static void Write(Message* aMsg, paramType&& aParam) {
+  static void Write(IPC::Message* aMsg, IProtocol* aActor, paramType&& aParam) {
+    Shmem shmem;
+    if (aParam.mSize.IsEmpty() ||
+        !aActor->AllocShmem(aParam.mRecording.mLen, SharedMemory::TYPE_BASIC,
+                            &shmem)) {
+      WriteParam(aMsg, gfx::IntSize(0, 0));
+      return;
+    }
+
+    memcpy(shmem.get<uint8_t>(), aParam.mRecording.mData,
+           aParam.mRecording.mLen);
+
     WriteParam(aMsg, aParam.mSize);
-    WriteParam(aMsg, std::move(aParam.mRecording));
+    WriteIPDLParam(aMsg, aActor, std::move(shmem));
     WriteParam(aMsg, aParam.mDependencies);
   }
 
-  static bool Read(const Message* aMsg, PickleIterator* aIter,
-                   paramType* aResult) {
-    return ReadParam(aMsg, aIter, &aResult->mSize) &&
-           ReadParam(aMsg, aIter, &aResult->mRecording) &&
-           ReadParam(aMsg, aIter, &aResult->mDependencies);
+  static bool Read(const IPC::Message* aMsg, PickleIterator* aIter,
+                   IProtocol* aActor, paramType* aResult) {
+    if (!ReadParam(aMsg, aIter, &aResult->mSize)) {
+      return false;
+    }
+    if (aResult->mSize.IsEmpty()) {
+      return true;
+    }
+    Shmem shmem;
+    if (!ReadIPDLParam(aMsg, aIter, aActor, &shmem) ||
+        !ReadParam(aMsg, aIter, &aResult->mDependencies)) {
+      aActor->DeallocShmem(shmem);
+      return false;
+    }
+
+    if (!aResult->mRecording.Allocate(shmem.Size<uint8_t>())) {
+      aResult->mSize.SizeTo(0, 0);
+      aActor->DeallocShmem(shmem);
+      return true;
+    }
+
+    memcpy(aResult->mRecording.mData, shmem.get<uint8_t>(),
+           shmem.Size<uint8_t>());
+    aActor->DeallocShmem(shmem);
+    return true;
   }
 };
 
-} /* namespace IPC */
+} /* namespace ipc */
+} /* namespace mozilla */
 
 #endif /* __GFXMESSAGEUTILS_H__ */