Bug 1145015 - Part 2: Add more checking rules for GrallocBuffer allocation. r=sotaro, r=nical

fix the assert checking.

gfx/layers/ipc/ShadowLayerUtilsGralloc.cpp

@@ -111,6 +111,10 @@ bool
 ParamTraits<MagicGrallocBufferHandle>::Read(const Message* aMsg,
                                             void** aIter, paramType* aResult)
 {
+  MOZ_ASSERT(!aResult->mGraphicBuffer.get());
+  MOZ_ASSERT(aResult->mRef.mOwner == 0);
+  MOZ_ASSERT(aResult->mRef.mKey == -1);
+
   size_t nbytes;
   const char* data;
   int owner;
@@ -126,7 +130,7 @@ ParamTraits<MagicGrallocBufferHandle>::Read(const Message* aMsg,
 
   size_t nfds = aMsg->num_fds();
   int fds[nfds];
-  bool sameProcess = (XRE_GetProcessType() == GeckoProcessType_Default);
+
   for (size_t n = 0; n < nfds; ++n) {
     FileDescriptor fd;
     if (!aMsg->ReadFileDescriptor(aIter, &fd)) {
@@ -144,7 +148,9 @@ ParamTraits<MagicGrallocBufferHandle>::Read(const Message* aMsg,
 
   aResult->mRef.mOwner = owner;
   aResult->mRef.mKey = index;
-  if (sameProcess) {
+  if (XRE_GetProcessType() == GeckoProcessType_Default) {
+    // If we are in chrome process, we can just get GraphicBuffer directly from
+    // SharedBufferManagerParent.
     aResult->mGraphicBuffer = SharedBufferManagerParent::GetGraphicBuffer(aResult->mRef);
   } else {
     // Deserialize GraphicBuffer
@@ -162,7 +168,7 @@ ParamTraits<MagicGrallocBufferHandle>::Read(const Message* aMsg,
       buffer = nullptr;
     }
 #endif
-    if(buffer.get() && !aResult->mGraphicBuffer.get()) {
+    if (buffer.get()) {
       aResult->mGraphicBuffer = buffer;
     }
   }

gfx/layers/ipc/SharedBufferManagerChild.cpp

@@ -275,7 +275,9 @@ SharedBufferManagerChild::AllocGrallocBufferNow(const IntSize& aSize,
 
 #ifdef MOZ_HAVE_SURFACEDESCRIPTORGRALLOC
   mozilla::layers::MaybeMagicGrallocBufferHandle handle;
-  SendAllocateGrallocBuffer(aSize, aFormat, aUsage, &handle);
+  if (!SendAllocateGrallocBuffer(aSize, aFormat, aUsage, &handle)) {
+    return false;
+  }
   if (handle.type() != mozilla::layers::MaybeMagicGrallocBufferHandle::TMagicGrallocBufferHandle) {
     return false;
   }
@@ -283,6 +285,7 @@ SharedBufferManagerChild::AllocGrallocBufferNow(const IntSize& aSize,
 
   {
     MutexAutoLock lock(mBufferMutex);
+    MOZ_ASSERT(mBuffers.count(handle.get_MagicGrallocBufferHandle().mRef.mKey)==0);
     mBuffers[handle.get_MagicGrallocBufferHandle().mRef.mKey] = handle.get_MagicGrallocBufferHandle().mGraphicBuffer;
   }
   return true;