Bug 831307 - Allow invalid file handles to not crash child processes. r=cjones.

dom/plugins/ipc/PluginModuleParent.cpp

@@ -970,7 +970,9 @@ PluginModuleParent::RecvBackUpXResources(const FileDescriptor& aXSocketFd)
     NS_ABORT_IF_FALSE(0 > mPluginXSocketFdDup.get(),
                       "Already backed up X resources??");
     mPluginXSocketFdDup.forget();
-    mPluginXSocketFdDup.reset(aXSocketFd.PlatformHandle());
+    if (aXSocketFd.IsValid()) {
+      mPluginXSocketFdDup.reset(aXSocketFd.PlatformHandle());
+    }
 #endif
     return true;
 }

ipc/glue/FileDescriptor.cpp

@@ -55,7 +55,7 @@ FileDescriptor::DuplicateInCurrentProcess(PlatformHandleType aHandle)
       mHandle = newHandle;
       return;
     }
-    NS_WARNING("Failed to duplicate file descriptor!");
+    NS_WARNING("Failed to duplicate file handle for current process!");
   }
 
   mHandle = INVALID_HANDLE;
@@ -75,7 +75,7 @@ FileDescriptor::CloseCurrentProcessHandle()
   if (IsValid()) {
 #ifdef XP_WIN
     if (!CloseHandle(mHandle)) {
-      NS_WARNING("Failed to close file handle!");
+      NS_WARNING("Failed to close file handle for current process!");
     }
 #else // XP_WIN
     HANDLE_EINTR(close(mHandle));
@@ -95,13 +95,16 @@ FileDescriptor::ShareTo(const FileDescriptor::IPDLPrivate&,
                         &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
       return newHandle;
     }
-    NS_WARNING("Failed to duplicate file handle!");
+    NS_WARNING("Failed to duplicate file handle for other process!");
   }
   return INVALID_HANDLE;
 #else // XP_WIN
   if (IsValid()) {
     newHandle = dup(mHandle);
-    return base::FileDescriptor(newHandle, /* auto_close */ true);
+    if (IsValid(newHandle)) {
+      return base::FileDescriptor(newHandle, /* auto_close */ true);
+    }
+    NS_WARNING("Failed to duplicate file handle for other process!");
   }
   return base::FileDescriptor();
 #endif

ipc/ipdl/ipdl/lower.py

@@ -4326,7 +4326,10 @@ class _GenerateProtocolActorCode(ipdl.ast.Visitor):
         ifread.addifstmt(StmtReturn.FALSE)
 
         ifnvalid = StmtIf(ExprNot(ExprCall(ExprSelect(tmpvar, '.', 'IsValid'))))
-        ifnvalid.addifstmt(StmtReturn.FALSE)
+        ifnvalid.addifstmt(
+            _protocolErrorBreakpoint('[' +
+                                     _actorName(self.protocol.name, self.side) +
+                                     '] Received an invalid file descriptor!'))
 
         read.addstmts([
             StmtDecl(Decl(_fdPickleType(), picklevar.name)),

netwerk/ipc/RemoteOpenFileChild.cpp

@@ -143,6 +143,11 @@ RemoteOpenFileChild::RecvFileOpened(const FileDescriptor& aFD)
 #if defined(XP_WIN) || defined(MOZ_WIDGET_COCOA)
   NS_NOTREACHED("osX and Windows shouldn't be doing IPDL here");
 #else
+  if (!aFD.IsValid()) {
+    return RecvFileDidNotOpen();
+  }
+
+  MOZ_ASSERT(!mNSPRFileDesc);
   mNSPRFileDesc = PR_AllocFileDesc(aFD.PlatformHandle(), PR_GetFileMethods());
 
   MOZ_ASSERT(mListener);
@@ -163,8 +168,10 @@ RemoteOpenFileChild::RecvFileDidNotOpen()
 #if defined(XP_WIN) || defined(MOZ_WIDGET_COCOA)
   NS_NOTREACHED("osX and Windows shouldn't be doing IPDL here");
 #else
-  MOZ_ASSERT(mListener);
+  MOZ_ASSERT(!mNSPRFileDesc);
   printf_stderr("RemoteOpenFileChild: file was not opened!\n");
+
+  MOZ_ASSERT(mListener);
   mListener->OnRemoteFileOpenComplete(NS_ERROR_FILE_NOT_FOUND);
   mListener = nullptr;     // release ref to listener