Bug 429592 - Add a monitor thread for process hangs and crash by default if a chrome process doesn't end up back in the event loop for more than 30 seconds. By default this affects non-debug builds only. r=cjones/bent

modules/libpref/src/init/all.js

@@ -1470,6 +1470,14 @@ pref("editor.positioning.offset",            0);
 pref("dom.max_chrome_script_run_time", 20);
 pref("dom.max_script_run_time", 10);
 
+// Hang monitor timeout after which we kill the browser, in seconds
+// (0 is disabled)
+#ifndef DEBUG
+pref("hangmonitor.timeout", 30);
+#else
+pref("hangmonitor.timeout", 0);
+#endif
+
 #ifndef DEBUG
 // How long a plugin is allowed to process a synchronous IPC message
 // before we consider it "hung".

widget/src/windows/nsAppShell.cpp

@@ -46,6 +46,7 @@
 #include "nsString.h"
 #include "nsIMM32Handler.h"
 #include "mozilla/widget/AudioSession.h"
+#include "mozilla/HangMonitor.h"
 
 // For skidmark code
 #include <windows.h> 
@@ -338,11 +339,13 @@ nsAppShell::ProcessNextNativeEvent(bool mayWait)
         ::PostQuitMessage(msg.wParam);
         Exit();
       } else {
+        mozilla::HangMonitor::NotifyActivity();
         ::TranslateMessage(&msg);
         ::DispatchMessageW(&msg);
       }
     } else if (mayWait) {
       // Block and wait for any posted application message
+      mozilla::HangMonitor::Suspend();
       ::WaitMessage();
     }
   } while (!gotMessage && mayWait);

xpcom/build/nsXPComInit.cpp

@@ -139,6 +139,7 @@ extern nsresult nsStringInputStreamConstructor(nsISupports *, REFNSIID, void **)
 #include "mozilla/Services.h"
 #include "mozilla/FunctionTimer.h"
 #include "mozilla/Omnijar.h"
+#include "mozilla/HangMonitor.h"
 
 #include "nsChromeRegistry.h"
 #include "nsChromeProtocolHandler.h"
@@ -530,6 +531,8 @@ NS_InitXPCOM2(nsIServiceManager* *result,
 
     mozilla::MapsMemoryReporter::Init();
 
+    mozilla::HangMonitor::Startup();
+
     return NS_OK;
 }
 
@@ -566,6 +569,9 @@ namespace mozilla {
 nsresult
 ShutdownXPCOM(nsIServiceManager* servMgr)
 {
+    // Make sure the hang monitor is enabled for shutdown.
+    HangMonitor::NotifyActivity();
+
     NS_ENSURE_STATE(NS_IsMainThread());
 
     nsresult rv;
@@ -623,6 +629,8 @@ ShutdownXPCOM(nsIServiceManager* servMgr)
 
         NS_ProcessPendingEvents(thread);
 
+        HangMonitor::NotifyActivity();
+
         // We save the "xpcom-shutdown-loaders" observers to notify after
         // the observerservice is gone.
         if (observerService) {
@@ -732,7 +740,9 @@ ShutdownXPCOM(nsIServiceManager* servMgr)
         sExitManager = nsnull;
     }
 
-    mozilla::Omnijar::CleanUp();
+    Omnijar::CleanUp();
+
+    HangMonitor::Shutdown();
 
     NS_LogTerm();
 

xpcom/threads/HangMonitor.cpp

@@ -0,0 +1,236 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Firefox.
+ *
+ * The Initial Developer of the Original Code is
+ * the Mozilla Foundation <http://www.mozilla.org>.
+ * Portions created by the Initial Developer are Copyright (C) 2011
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#include "mozilla/HangMonitor.h"
+#include "mozilla/Monitor.h"
+#include "mozilla/Preferences.h"
+#include "nsExceptionHandler.h"
+#include "nsXULAppAPI.h"
+#include "nsThreadUtils.h"
+
+#ifdef XP_WIN
+#include <windows.h>
+#endif
+
+namespace mozilla { namespace HangMonitor {
+
+/**
+ * A flag which may be set from within a debugger to disable the hang
+ * monitor.
+ */
+volatile bool gDebugDisableHangMonitor = false;
+
+const char kHangMonitorPrefName[] = "hangmonitor.timeout";
+
+// Monitor protects gShutdown and gTimeout, but not gTimestamp which rely on
+// being atomically set by the processor; synchronization doesn't really matter
+// in this use case.
+Monitor* gMonitor;
+
+// The timeout preference, in seconds.
+PRInt32 gTimeout;
+
+PRThread* gThread;
+
+// Set when shutdown begins to signal the thread to exit immediately.
+bool gShutdown;
+
+// The timestamp of the last event notification, or PR_INTERVAL_NO_WAIT if
+// we're currently not processing events.
+volatile PRIntervalTime gTimestamp;
+
+// PrefChangedFunc
+int
+PrefChanged(const char*, void*)
+{
+  PRInt32 newval = Preferences::GetInt(kHangMonitorPrefName);
+  MonitorAutoLock lock(*gMonitor);
+  if (newval != gTimeout) {
+    gTimeout = newval;
+    lock.Notify();
+  }
+
+  return 0;
+}
+
+void
+Crash()
+{
+  if (gDebugDisableHangMonitor) {
+    return;
+  }
+
+#ifdef XP_WIN
+  if (::IsDebuggerPresent()) {
+    return;
+  }
+#endif
+
+  CrashReporter::AnnotateCrashReport(NS_LITERAL_CSTRING("Hang"),
+                                     NS_LITERAL_CSTRING("1"));
+
+  NS_RUNTIMEABORT("HangMonitor triggered");
+}
+
+void
+ThreadMain(void*)
+{
+  MonitorAutoLock lock(*gMonitor);
+
+  // In order to avoid issues with the hang monitor incorrectly triggering
+  // during a general system stop such as sleeping, the monitor thread must
+  // run twice to trigger hang protection.
+  PRIntervalTime lastTimestamp = 0;
+  int waitCount = 0;
+
+  while (true) {
+    if (gShutdown) {
+      return; // Exit the thread
+    }
+
+    // avoid rereading the volatile value in this loop
+    PRIntervalTime timestamp = gTimestamp;
+
+    PRIntervalTime now = PR_IntervalNow();
+
+    if (timestamp != PR_INTERVAL_NO_WAIT &&
+        now < timestamp) {
+      // 32-bit overflow, reset for another waiting period
+      timestamp = 1; // lowest legal PRInterval value
+    }
+
+    if (timestamp != PR_INTERVAL_NO_WAIT &&
+        timestamp == lastTimestamp &&
+        gTimeout > 0) {
+      ++waitCount;
+      if (waitCount == 2) {
+        PRInt32 delay =
+          PRInt32(PR_IntervalToSeconds(now - timestamp));
+        if (delay > gTimeout) {
+          MonitorAutoUnlock unlock(*gMonitor);
+          Crash();
+        }
+      }
+    }
+    else {
+      lastTimestamp = timestamp;
+      waitCount = 0;
+    }
+
+    PRIntervalTime timeout;
+    if (gTimeout <= 0) {
+      timeout = PR_INTERVAL_NO_TIMEOUT;
+    }
+    else {
+      timeout = PR_MillisecondsToInterval(gTimeout * 500);
+    }
+    lock.Wait(timeout);
+  }
+}
+
+void
+Startup()
+{
+  // The hang detector only runs in chrome processes. If you change this,
+  // you must also deal with the threadsafety of AnnotateCrashReport in
+  // non-chrome processes!
+  if (GeckoProcessType_Default != XRE_GetProcessType())
+    return;
+
+  NS_ASSERTION(!gMonitor, "Hang monitor already initialized");
+  gMonitor = new Monitor("HangMonitor");
+
+  Preferences::RegisterCallback(PrefChanged, kHangMonitorPrefName, NULL);
+  PrefChanged(NULL, NULL);
+
+  // Don't actually start measuring hangs until we hit the main event loop.
+  // This potentially misses a small class of really early startup hangs,
+  // but avoids dealing with some xpcshell tests and other situations which
+  // start XPCOM but don't ever start the event loop.
+  Suspend();
+
+  gThread = PR_CreateThread(PR_USER_THREAD,
+                            ThreadMain,
+                            NULL, PR_PRIORITY_LOW, PR_GLOBAL_THREAD,
+                            PR_JOINABLE_THREAD, 0);
+}
+
+void
+Shutdown()
+{
+  if (GeckoProcessType_Default != XRE_GetProcessType())
+    return;
+
+  NS_ASSERTION(gMonitor, "Hang monitor not started");
+
+  { // Scope the lock we're going to delete later
+    MonitorAutoLock lock(*gMonitor);
+    gShutdown = true;
+    lock.Notify();
+  }
+
+  // thread creation could theoretically fail
+  if (gThread) {
+    PR_JoinThread(gThread);
+    gThread = NULL;
+  }
+
+  delete gMonitor;
+  gMonitor = NULL;
+}
+
+void
+NotifyActivity()
+{
+  NS_ASSERTION(NS_IsMainThread(), "HangMonitor::Notify called from off the main thread.");
+
+  // This is not a locked activity because PRTimeStamp is a 32-bit quantity
+  // which can be read/written atomically, and we don't want to pay locking
+  // penalties here.
+  gTimestamp = PR_IntervalNow();
+}
+
+void
+Suspend()
+{
+  NS_ASSERTION(NS_IsMainThread(), "HangMonitor::Suspend called from off the main thread.");
+
+  // Because gTimestamp changes this resets the wait count.
+  gTimestamp = PR_INTERVAL_NO_WAIT;
+}
+
+} } // namespace mozilla::HangMonitor

xpcom/threads/HangMonitor.h

@@ -0,0 +1,67 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Firefox.
+ *
+ * The Initial Developer of the Original Code is
+ * the Mozilla Foundation <http://www.mozilla.org>.
+ * Portions created by the Initial Developer are Copyright (C) 2011
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef mozilla_HangMonitor_h
+#define mozilla_HangMonitor_h
+
+namespace mozilla { namespace HangMonitor {
+
+/**
+ * Start monitoring hangs. Should be called by the XPCOM startup process only.
+ */
+void Startup();
+
+/**
+ * Stop monitoring hangs and join the thread.
+ */
+void Shutdown();
+
+/**
+ * Notify the hang monitor of new activity which should reset its internal
+ * timer.
+ */
+void NotifyActivity();
+
+/**
+ * Notify the hang monitor that the browser is now idle and no detection should
+ * be done.
+ */
+void Suspend();
+
+} } // namespace mozilla::HangMonitor
+
+#endif // mozilla_HangMonitor_h

xpcom/threads/Makefile.in

@@ -47,7 +47,9 @@ XPIDL_MODULE	= xpcom_threads
 LIBRARY_NAME	= xpcomthreads_s
 GRE_MODULE	= 1
 MOZILLA_INTERNAL_API = 1
+LIBXUL_LIBRARY = 1
 
+EXPORTS_NAMESPACES = mozilla
 
 CPPSRCS		= \
 		nsEventQueue.cpp \
@@ -58,6 +60,7 @@ CPPSRCS		= \
 		nsProcessCommon.cpp \
 		nsTimerImpl.cpp \
 		TimerThread.cpp \
+		HangMonitor.cpp \
 		$(NULL)
 
 EXPORTS		= \
@@ -66,6 +69,10 @@ EXPORTS		= \
 		nsThreadUtilsInternal.h \
 		$(NULL)
 
+EXPORTS_mozilla = \
+  HangMonitor.h \
+  $(NULL)
+
 XPIDLSRCS	= \
 		nsIEventTarget.idl \
 		nsIThread.idl \

xpcom/threads/nsThread.cpp

@@ -45,6 +45,7 @@
 #include "nsCOMPtr.h"
 #include "prlog.h"
 #include "nsThreadUtilsInternal.h"
+#include "mozilla/HangMonitor.h"
 
 #define HAVE_UALARM _BSD_SOURCE || (_XOPEN_SOURCE >= 500 ||                 \
                       _XOPEN_SOURCE && _XOPEN_SOURCE_EXTENDED) &&           \
@@ -306,20 +307,7 @@ nsThread::ThreadFunc(void *arg)
 
 //-----------------------------------------------------------------------------
 
-nsThread::nsThread()
-  : mLock("nsThread.mLock")
-  , mEvents(&mEventsRoot)
-  , mPriority(PRIORITY_NORMAL)
-  , mThread(nsnull)
-  , mRunningEvent(0)
-  , mStackSize(0)
-  , mShutdownContext(nsnull)
-  , mShutdownRequired(false)
-  , mEventsAreDoomed(false)
-{
-}
-
-nsThread::nsThread(PRUint32 aStackSize)
+nsThread::nsThread(MainThreadFlag aMainThread, PRUint32 aStackSize)
   : mLock("nsThread.mLock")
   , mEvents(&mEventsRoot)
   , mPriority(PRIORITY_NORMAL)
@@ -329,6 +317,7 @@ nsThread::nsThread(PRUint32 aStackSize)
   , mShutdownContext(nsnull)
   , mShutdownRequired(false)
   , mEventsAreDoomed(false)
+  , mIsMainThread(aMainThread)
 {
 }
 
@@ -585,6 +574,9 @@ nsThread::ProcessNextEvent(bool mayWait, bool *result)
 
   NS_ENSURE_STATE(PR_GetCurrentThread() == mThread);
 
+  if (MAIN_THREAD == mIsMainThread && mayWait && !ShuttingDown())
+    HangMonitor::Suspend();
+
   bool notifyGlobalObserver = (sGlobalObserver != nsnull);
   if (notifyGlobalObserver) 
     sGlobalObserver->OnProcessNextEvent(this, mayWait && !ShuttingDown(),
@@ -615,7 +607,7 @@ nsThread::ProcessNextEvent(bool mayWait, bool *result)
 
 #ifdef NS_FUNCTION_TIMER
     char message[1024] = {'\0'};
-    if (NS_IsMainThread()) {
+    if (MAIN_THREAD == mIsMainThread) {
         mozilla::FunctionTimer::ft_snprintf(message, sizeof(message), 
                                             "@ Main Thread Event %p", (void*)event.get());
     }
@@ -628,6 +620,8 @@ nsThread::ProcessNextEvent(bool mayWait, bool *result)
 
     if (event) {
       LOG(("THRD(%p) running [%p]\n", this, event.get()));
+      if (MAIN_THREAD == mIsMainThread)
+        HangMonitor::NotifyActivity();
       event->Run();
     } else if (mayWait) {
       NS_ASSERTION(ShuttingDown(),

xpcom/threads/nsThread.h

@@ -57,8 +57,12 @@ public:
   NS_DECL_NSITHREADINTERNAL
   NS_DECL_NSISUPPORTSPRIORITY
 
-  nsThread();
-  nsThread(PRUint32 aStackSize);
+  enum MainThreadFlag {
+    MAIN_THREAD,
+    NOT_MAIN_THREAD
+  };
+
+  nsThread(MainThreadFlag aMainThread, PRUint32 aStackSize);
 
   // Initialize this as a wrapper for a new PRThread.
   nsresult Init();
@@ -147,6 +151,7 @@ private:
   bool mShutdownPending;
   // Set to true when events posted to this thread will never run.
   bool mEventsAreDoomed;
+  MainThreadFlag mIsMainThread;
 };
 
 //-----------------------------------------------------------------------------

xpcom/threads/nsThreadManager.cpp

@@ -98,7 +98,7 @@ nsThreadManager::Init()
   mLock = new Mutex("nsThreadManager.mLock");
 
   // Setup "main" thread
-  mMainThread = new nsThread();
+  mMainThread = new nsThread(nsThread::MAIN_THREAD, 0);
   if (!mMainThread)
     return NS_ERROR_OUT_OF_MEMORY;
 
@@ -224,7 +224,7 @@ nsThreadManager::GetCurrentThread()
   }
 
   // OK, that's fine.  We'll dynamically create one :-)
-  nsRefPtr<nsThread> thread = new nsThread();
+  nsRefPtr<nsThread> thread = new nsThread(nsThread::NOT_MAIN_THREAD, 0);
   if (!thread || NS_FAILED(thread->InitCurrentThread()))
     return nsnull;
 
@@ -239,7 +239,7 @@ nsThreadManager::NewThread(PRUint32 creationFlags,
   // No new threads during Shutdown
   NS_ENSURE_TRUE(mInitialized, NS_ERROR_NOT_INITIALIZED);
 
-  nsThread *thr = new nsThread(stackSize);
+  nsThread *thr = new nsThread(nsThread::NOT_MAIN_THREAD, stackSize);
   if (!thr)
     return NS_ERROR_OUT_OF_MEMORY;
   NS_ADDREF(thr);