fix bug 54792 by running assert dialog out or process on Win32. r=danm sr=brendan

xpcom/base/nsDebug.cpp

@@ -210,13 +210,39 @@ NS_COM void nsDebug::Assertion(const char* aStr, const char* aExpr,
 #if defined(_WIN32)
    if(!InDebugger())
       {
-      char msg[1200];
-      PR_snprintf(msg, sizeof(msg),
-                "%s\n\nClick Abort to exit the Application.\n"
-                "Click Retry to Debug the Application..\n"
-                "Click Ignore to continue running the Application.", buf); 
-      int code = ::MessageBox(NULL, msg, "nsDebug::Assertion",
-                     MB_ICONSTOP | MB_ABORTRETRYIGNORE);
+      DWORD code = IDRETRY;
+
+      /* Create the debug dialog out of process to avoid the crashes caused by 
+       * Windows events leaking into our event loop from an in process dialog.
+       * We do this by launching windbgdlg.exe (built in xpcom/windbgdlg).
+       * See http://bugzilla.mozilla.org/show_bug.cgi?id=54792
+       */
+      PROCESS_INFORMATION pi;
+      STARTUPINFO si;
+      char executable[MAX_PATH];
+      char* pName;
+
+      memset(&pi, 0, sizeof(pi));
+
+      memset(&si, 0, sizeof(si));
+      si.cb          = sizeof(si);
+      si.wShowWindow = SW_SHOW;
+
+      if(GetModuleFileName(NULL, executable, MAX_PATH) &&
+         NULL != (pName = strrchr(executable, '\\')) &&
+         NULL != strcpy(pName+1, "windbgdlg.exe") &&
+#ifdef DEBUG_jband
+         (printf("Launching %s\n", executable), PR_TRUE) &&
+#endif         
+         CreateProcess(executable, buf, NULL, NULL, PR_FALSE,
+                       DETACHED_PROCESS | NORMAL_PRIORITY_CLASS,
+                       NULL, NULL, &si, &pi) &&
+         WAIT_OBJECT_0 == WaitForSingleObject(pi.hProcess, INFINITE) &&
+         GetExitCodeProcess(pi.hProcess, &code))
+      {
+        CloseHandle(pi.hProcess);
+      }                         
+
       switch(code)
          {
          case IDABORT:

xpcom/glue/nsDebug.cpp

@@ -210,13 +210,39 @@ NS_COM void nsDebug::Assertion(const char* aStr, const char* aExpr,
 #if defined(_WIN32)
    if(!InDebugger())
       {
-      char msg[1200];
-      PR_snprintf(msg, sizeof(msg),
-                "%s\n\nClick Abort to exit the Application.\n"
-                "Click Retry to Debug the Application..\n"
-                "Click Ignore to continue running the Application.", buf); 
-      int code = ::MessageBox(NULL, msg, "nsDebug::Assertion",
-                     MB_ICONSTOP | MB_ABORTRETRYIGNORE);
+      DWORD code = IDRETRY;
+
+      /* Create the debug dialog out of process to avoid the crashes caused by 
+       * Windows events leaking into our event loop from an in process dialog.
+       * We do this by launching windbgdlg.exe (built in xpcom/windbgdlg).
+       * See http://bugzilla.mozilla.org/show_bug.cgi?id=54792
+       */
+      PROCESS_INFORMATION pi;
+      STARTUPINFO si;
+      char executable[MAX_PATH];
+      char* pName;
+
+      memset(&pi, 0, sizeof(pi));
+
+      memset(&si, 0, sizeof(si));
+      si.cb          = sizeof(si);
+      si.wShowWindow = SW_SHOW;
+
+      if(GetModuleFileName(NULL, executable, MAX_PATH) &&
+         NULL != (pName = strrchr(executable, '\\')) &&
+         NULL != strcpy(pName+1, "windbgdlg.exe") &&
+#ifdef DEBUG_jband
+         (printf("Launching %s\n", executable), PR_TRUE) &&
+#endif         
+         CreateProcess(executable, buf, NULL, NULL, PR_FALSE,
+                       DETACHED_PROCESS | NORMAL_PRIORITY_CLASS,
+                       NULL, NULL, &si, &pi) &&
+         WAIT_OBJECT_0 == WaitForSingleObject(pi.hProcess, INFINITE) &&
+         GetExitCodeProcess(pi.hProcess, &code))
+      {
+        CloseHandle(pi.hProcess);
+      }                         
+
       switch(code)
          {
          case IDABORT:

xpcom/makefile.win

@@ -34,6 +34,9 @@ DIRS=   typelib        \
         build          \
         appshell       \
         tools          \
+!if defined(MOZ_DEBUG)
+        windbgdlg      \
+!endif
 !if !defined(DISABLE_TESTS)
         sample         \
         tests          \

xpcom/windbgdlg/makefile.win

@@ -0,0 +1,40 @@
+#!nmake
+#
+# The contents of this file are subject to the Netscape Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/NPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is Netscape
+# Communications Corporation.  Portions created by Netscape are
+# Copyright (C) 1998 Netscape Communications Corporation. All
+# Rights Reserved.
+#
+# Contributor(s): 
+
+DEPTH=..\..
+
+include <$(DEPTH)/config/config.mak>
+
+MAKE_OBJ_TYPE	= EXE
+PROG1 = .\$(OBJDIR)\windbgdlg.exe
+
+PROGRAMS = $(PROG1) \
+           $(NULL)
+
+include <$(DEPTH)\config\rules.mak>
+
+install:: $(PROGRAMS)
+	-for %p in ($(PROGRAMS)) do $(MAKE_INSTALL) %p $(DIST)\bin
+
+clobber::
+	-for %p in ($(PROGRAMS)) do $(RM) %p $(DIST)\bin\%p
+
+$(PROG1): $(OBJDIR) windbgdlg.cpp

xpcom/windbgdlg/windbgdlg.cpp

@@ -0,0 +1,55 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ *
+ * The contents of this file are subject to the Netscape Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/NPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express oqr
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ *
+ * The Original Code is Mozilla Communicator client code, released
+ * March 31, 1998.
+ *
+ * The Initial Developer of the Original Code is Netscape
+ * Communications Corporation.  Portions created by Netscape are
+ * Copyright (C) 1999 Netscape Communications Corporation. All
+ * Rights Reserved.
+ *
+ * Contributor(s): 
+ *   John Bandhauer <jband@netscape.com>
+ *
+ * Alternatively, the contents of this file may be used under the
+ * terms of the GNU Public License (the "GPL"), in which case the
+ * provisions of the GPL are applicable instead of those above.
+ * If you wish to allow use of your version of this file only
+ * under the terms of the GPL and not to allow others to use your
+ * version of this file under the NPL, indicate your decision by
+ * deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL.  If you do not delete
+ * the provisions above, a recipient may use your version of this
+ * file under either the NPL or the GPL.
+ */
+
+/* Windows only app to show a modal debug dialog - launched by nsDebug.cpp */
+
+#include <windows.h>
+
+int WINAPI 
+WinMain(HINSTANCE  hInstance, HINSTANCE  hPrevInstance, 
+        LPSTR  lpszCmdLine, int  nCmdShow)
+{
+    static char msg[4048];
+
+    wsprintf(msg,
+             "%s\n\nClick Abort to exit the Application.\n"
+             "Click Retry to Debug the Application..\n"
+             "Click Ignore to continue running the Application.", 
+             lpszCmdLine);
+             
+    return MessageBox(NULL, msg, "nsDebug::Assertion",
+                      MB_ICONSTOP | MB_SYSTEMMODAL| 
+                      MB_ABORTRETRYIGNORE | MB_DEFBUTTON3);
+}