Bug 599059: Always store length at the end of shared memory segments as a 32-bit value. Don't use "sizeof(size_t)" because that differs between i386 and x86_64 and causes crashes when running i386 plugins from a x86_64 host. r=cjones a=blocking-b7

ipc/glue/Shmem.cpp

@@ -370,6 +370,8 @@ Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
              SharedMemoryType aType,
              bool aProtect)
 {
+  NS_ASSERTION(aNBytes <= PR_UINT32_MAX, "Will truncate shmem segment size!");
+
   size_t pageSize = SharedMemory::SystemPageSize();
   SharedMemory* segment = nsnull;
   // |2*pageSize| is for the front and back sentinel
@@ -395,7 +397,6 @@ Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
   // initialize the segment with Shmem-internal information
   Header* header = reinterpret_cast<Header*>(frontSentinel);
   memcpy(header->mMagic, sMagic, sizeof(sMagic));
-  NS_ASSERTION(aNBytes <= PR_UINT32_MAX, "Will truncate shmem segment size!");
   header->mSize = static_cast<uint32>(aNBytes);
 
   if (aProtect)
@@ -493,11 +494,11 @@ Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
   SharedMemory *segment = nsnull;
 
   if (aType == SharedMemory::TYPE_BASIC)
-    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(size_t)),
+    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(uint32)),
                             SharedMemoryBasic::NULLHandle());
 #ifdef MOZ_HAVE_SHAREDMEMORYSYSV
   else if (aType == SharedMemory::TYPE_SYSV)
-    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(size_t)),
+    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(uint32)),
                             SharedMemorySysV::NULLHandle());
 #endif
   else
@@ -507,7 +508,7 @@ Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
   if (!segment)
     return 0;
 
-  *PtrToSize(segment) = aNBytes;
+  *PtrToSize(segment) = static_cast<uint32>(aNBytes);
 
   return segment;
 }
@@ -560,7 +561,7 @@ Shmem::OpenExisting(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
     return 0;
 
   // this is the only validity check done OPT builds
-  if (size != *PtrToSize(segment))
+  if (size != static_cast<size_t>(*PtrToSize(segment)))
     NS_RUNTIMEABORT("Alloc() segment size disagrees with OpenExisting()'s");
 
   return segment;

ipc/glue/Shmem.h

@@ -122,7 +122,7 @@ public:
     mSize(0),
     mId(aId)
   {
-    mSize = *PtrToSize(mSegment);
+    mSize = static_cast<size_t>(*PtrToSize(mSegment));
   }
 #else
   Shmem(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
@@ -274,12 +274,12 @@ private:
   void AssertInvariants() const
   { }
 
-  static size_t*
+  static uint32*
   PtrToSize(SharedMemory* aSegment)
   {
     char* endOfSegment =
       reinterpret_cast<char*>(aSegment->memory()) + aSegment->Size();
-    return reinterpret_cast<size_t*>(endOfSegment - sizeof(size_t));
+    return reinterpret_cast<uint32*>(endOfSegment - sizeof(uint32));
   }
 
 #else