Bug 902755 - Fix deadlock in mozilla::ipc::GeckoChildProcessHost::LaunchAndWaitForProcessHandle exposed by turning on new tab page thumbnails. r=bent

ipc/glue/GeckoChildProcessHost.cpp

@@ -666,6 +666,11 @@ GeckoChildProcessHost::PerformAsyncLaunchInternal(std::vector<std::string>& aExt
   GetChannel()->CloseClientFileDescriptor();
 
 #ifdef MOZ_WIDGET_COCOA
+  if (!process) {
+    SetErrorState();
+    return false;
+  }
+
   // Wait for the child process to send us its 'task_t' data.
   const int kTimeoutMs = 10000;
 
@@ -675,17 +680,20 @@ GeckoChildProcessHost::PerformAsyncLaunchInternal(std::vector<std::string>& aExt
   if (err != KERN_SUCCESS) {
     std::string errString = StringPrintf("0x%x %s", err, mach_error_string(err));
     LOG(ERROR) << "parent WaitForMessage() failed: " << errString;
+    SetErrorState();
     return false;
   }
 
   task_t child_task = child_message.GetTranslatedPort(0);
   if (child_task == MACH_PORT_NULL) {
     LOG(ERROR) << "parent GetTranslatedPort(0) failed.";
+    SetErrorState();
     return false;
   }
 
   if (child_message.GetTranslatedPort(1) == MACH_PORT_NULL) {
     LOG(ERROR) << "parent GetTranslatedPort(1) failed.";
+    SetErrorState();
     return false;
   }
   MachPortSender parent_sender(child_message.GetTranslatedPort(1));
@@ -693,6 +701,7 @@ GeckoChildProcessHost::PerformAsyncLaunchInternal(std::vector<std::string>& aExt
   MachSendMessage parent_message(/* id= */0);
   if (!parent_message.AddDescriptor(bootstrap_port)) {
     LOG(ERROR) << "parent AddDescriptor(" << bootstrap_port << ") failed.";
+    SetErrorState();
     return false;
   }
 
@@ -700,6 +709,7 @@ GeckoChildProcessHost::PerformAsyncLaunchInternal(std::vector<std::string>& aExt
   if (err != KERN_SUCCESS) {
     std::string errString = StringPrintf("0x%x %s", err, mach_error_string(err));
     LOG(ERROR) << "parent SendMessage() failed: " << errString;
+    SetErrorState();
     return false;
   }
 #endif
@@ -760,14 +770,13 @@ GeckoChildProcessHost::PerformAsyncLaunchInternal(std::vector<std::string>& aExt
 
 #else
 #  error Sorry
-#endif
+#endif // OS_POSIX
 
   if (!process) {
-    MonitorAutoLock lock(mMonitor);
-    mProcessState = PROCESS_ERROR;
-    lock.Notify();
+    SetErrorState();
     return false;
   }
+
   // NB: on OS X, we block much longer than we need to in order to
   // reach this call, waiting for the child process's task_t.  The
   // best way to fix that is to refactor this file, hard.

ipc/glue/GeckoChildProcessHost.h

@@ -177,6 +177,12 @@ private:
   bool RunPerformAsyncLaunch(StringVector aExtraOpts=StringVector(),
 			     base::ProcessArchitecture aArch=base::GetCurrentProcessArchitecture());
 
+  inline void SetErrorState() {
+    MonitorAutoLock lock(mMonitor);
+    mProcessState = PROCESS_ERROR;
+    lock.Notify();
+  }
+
   // In between launching the subprocess and handing off its IPC
   // channel, there's a small window of time in which *we* might still
   // be the channel listener, and receive messages.  That's bad