Bug 366803 - Improve SSL tracing, make it work in browsers, to help with

debugging bug 356470.  r=neil.williams,alexei.volkov

security/nss/lib/ssl/sslimpl.h

@@ -39,7 +39,7 @@
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
-/* $Id: sslimpl.h,v 1.53 2006-07-19 01:40:17 nelson%bolyard.com Exp $ */
+/* $Id: sslimpl.h,v 1.54 2007-01-31 04:20:26 nelson%bolyard.com Exp $ */
 
 #ifndef __sslimpl_h_
 #define __sslimpl_h_
@@ -1048,6 +1048,7 @@ const unsigned char *  preferredCipher;
 extern NSSRWLock *             ssl_global_data_lock;
 extern char                    ssl_debug;
 extern char                    ssl_trace;
+extern FILE *                  ssl_trace_iob;
 extern CERTDistNames *         ssl3_server_ca_list;
 extern PRUint32                ssl_sid_timeout;
 extern PRUint32                ssl3_sid_timeout;

security/nss/lib/ssl/sslsecur.c

@@ -37,7 +37,7 @@
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
-/* $Id: sslsecur.c,v 1.36 2006-04-20 08:46:34 nelson%bolyard.com Exp $ */
+/* $Id: sslsecur.c,v 1.37 2007-01-31 04:20:26 nelson%bolyard.com Exp $ */
 #include "cert.h"
 #include "secitem.h"
 #include "keyhi.h"
@@ -1057,13 +1057,18 @@ ssl_SecureSend(sslSocket *ss, const unsigned char *buf, int len, int flags)
 {
     int              rv		= 0;
 
+    SSL_TRC(2, ("%d: SSL[%d]: SecureSend: sending %d bytes",
+		SSL_GETPID(), ss->fd, len));
+
     if (ss->shutdownHow & ssl_SHUTDOWN_SEND) {
 	PORT_SetError(PR_SOCKET_SHUTDOWN_ERROR);
-    	return PR_FAILURE;
+    	rv = PR_FAILURE;
+	goto done;
     }
     if (flags) {
 	PORT_SetError(PR_INVALID_ARGUMENT_ERROR);
-    	return PR_FAILURE;
+    	rv = PR_FAILURE;
+	goto done;
     }
 
     ssl_GetXmitBufLock(ss);
@@ -1078,7 +1083,7 @@ ssl_SecureSend(sslSocket *ss, const unsigned char *buf, int len, int flags)
     }
     ssl_ReleaseXmitBufLock(ss);
     if (rv < 0) {
-	return rv;
+	goto done;
     }
 
     if (len > 0) 
@@ -1093,23 +1098,22 @@ ssl_SecureSend(sslSocket *ss, const unsigned char *buf, int len, int flags)
     }
     if (rv < 0) {
     	ss->writerThread = NULL;
-	return rv;
+	goto done;
     }
 
     /* Check for zero length writes after we do housekeeping so we make forward
      * progress.
      */
     if (len == 0) {
-    	return 0;
+    	rv = 0;
+	goto done;
     }
     PORT_Assert(buf != NULL);
     if (!buf) {
 	PORT_SetError(PR_INVALID_ARGUMENT_ERROR);
-    	return PR_FAILURE;
+    	rv = PR_FAILURE;
+	goto done;
     }
-    
-    SSL_TRC(2, ("%d: SSL[%d]: SecureSend: sending %d bytes",
-		SSL_GETPID(), ss->fd, len));
 
     /* Send out the data using one of these functions:
      *	ssl2_SendClear, ssl2_SendStream, ssl2_SendBlock, 
@@ -1119,6 +1123,14 @@ ssl_SecureSend(sslSocket *ss, const unsigned char *buf, int len, int flags)
     rv = (*ss->sec.send)(ss, buf, len, flags);
     ssl_ReleaseXmitBufLock(ss);
     ss->writerThread = NULL;
+done:
+    if (rv < 0) {
+	SSL_TRC(2, ("%d: SSL[%d]: SecureSend: returning %d count, error %d",
+		    SSL_GETPID(), ss->fd, rv, PORT_GetError()));
+    } else {
+	SSL_TRC(2, ("%d: SSL[%d]: SecureSend: returning %d count",
+		    SSL_GETPID(), ss->fd, rv));
+    }
     return rv;
 }
 

security/nss/lib/ssl/sslsock.c

@@ -40,7 +40,7 @@
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
-/* $Id: sslsock.c,v 1.50 2006-10-02 21:15:46 julien.pierre.bugs%sun.com Exp $ */
+/* $Id: sslsock.c,v 1.51 2007-01-31 04:20:26 nelson%bolyard.com Exp $ */
 #include "seccomon.h"
 #include "cert.h"
 #include "keyhi.h"
@@ -186,6 +186,7 @@ PRBool			ssl_force_locks;  	/* implicitly PR_FALSE */
 int                     ssl_lock_readers	= 1;	/* default true. */
 char                    ssl_debug;
 char                    ssl_trace;
+FILE *                  ssl_trace_iob;
 char lockStatus[] = "Locks are ENABLED.  ";
 #define LOCKSTATUS_OFFSET 10 /* offset of ENABLED */
 
@@ -2080,6 +2081,13 @@ ssl_NewSocket(PRBool makeLocks)
 	char * ev;
 	firsttime = 0;
 #ifdef DEBUG
+	ev = getenv("SSLDEBUGFILE");
+	if (ev && ev[0]) {
+	    ssl_trace_iob = fopen(ev, "w");
+	}
+	if (!ssl_trace_iob) {
+	    ssl_trace_iob = stderr;
+	}
 #ifdef TRACE
 	ev = getenv("SSLTRACE");
 	if (ev && ev[0]) {

security/nss/lib/ssl/ssltrace.c

@@ -36,7 +36,7 @@
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
-/* $Id: ssltrace.c,v 1.3 2004-04-27 23:04:39 gerv%gerv.net Exp $ */
+/* $Id: ssltrace.c,v 1.4 2007-01-31 04:20:26 nelson%bolyard.com Exp $ */
 #include <stdarg.h>
 #include "cert.h"
 #include "ssl.h"
@@ -262,11 +262,15 @@ void
 ssl_Trace(const char *format, ... )
 {
     char buf[2000]; 
-
     va_list args;
-    va_start(args, format);
-    PR_vsnprintf(buf, sizeof(buf), format, args);
-    va_end(args);
-    puts(buf);
+
+    if (ssl_trace_iob) {
+	va_start(args, format);
+	PR_vsnprintf(buf, sizeof(buf), format, args);
+	va_end(args);
+
+	fputs(buf,  ssl_trace_iob);
+	fputs("\n", ssl_trace_iob);
+    }
 }
 #endif