Bug 340359, SSL Server stalls on v3 hello using TLS hello extensions

r=darin

security/manager/ssl/src/nsNSSIOLayer.cpp

@@ -182,6 +182,7 @@ nsNSSSocketInfo::nsNSSSocketInfo()
     mCanceled(PR_FALSE),
     mHasCleartextPhase(PR_FALSE),
     mHandshakeInProgress(PR_FALSE),
+    mHandshakeStartTime(0),
     mPort(0),
     mCAChain(nsnull)
 {
@@ -459,6 +460,27 @@ nsresult nsNSSSocketInfo::SetSSLStatus(nsISSLStatus *aSSLStatus)
   return NS_OK;
 }
 
+void nsNSSSocketInfo::SetHandshakeInProgress(PRBool aIsIn)
+{
+  mHandshakeInProgress = aIsIn;
+
+  if (mHandshakeInProgress && !mHandshakeStartTime)
+  {
+    mHandshakeStartTime = PR_IntervalNow();
+  }
+}
+
+#define HANDSHAKE_TIMEOUT_SECONDS 8
+
+PRBool nsNSSSocketInfo::HandshakeTimeout()
+{
+  if (!mHandshakeInProgress)
+    return PR_FALSE;
+
+  return ((PRIntervalTime)(PR_IntervalNow() - mHandshakeStartTime)
+          > PR_SecondsToInterval(HANDSHAKE_TIMEOUT_SECONDS));
+}
+
 void nsSSLIOLayerHelpers::Cleanup()
 {
   if (mTLSIntolerantSites) {
@@ -1077,6 +1099,18 @@ nsSSLThread::checkHandshake(PRInt32 bytesTransfered, PRFileDesc* ssl_layer_fd, n
   // there are enough broken servers out there that such a gross work-around
   // is necessary.  :(
 
+  // Additional comment added in August 2006:
+  // When we begun to use TLS hello extensions, we encountered a new class of
+  // broken server, which simply stall for a very long time.
+  // We would like to shorten the timeout, but limit this shorter timeout 
+  // to the handshake phase.
+  // When we arrive here for the first time (for a given socket),
+  // we know the connection is established, and the application code
+  // tried the first read or write. This triggers the beginning of the
+  // SSL handshake phase at the SSL FD level.
+  // We'll make a note of the current time,
+  // and use this to measure the elapsed time since handshake begin.
+
   PRBool handleHandshakeResultNow;
   socketInfo->GetHandshakePending(&handleHandshakeResultNow);
 

security/manager/ssl/src/nsNSSIOLayer.h

@@ -156,8 +156,9 @@ public:
   void SetHasCleartextPhase(PRBool aHasCleartextPhase);
   PRBool GetHasCleartextPhase();
   
-  void SetHandshakeInProgress(PRBool aIsIn) { mHandshakeInProgress = aIsIn; }
+  void SetHandshakeInProgress(PRBool aIsIn);
   PRBool GetHandshakeInProgress() { return mHandshakeInProgress; }
+  PRBool HandshakeTimeout();
 
   nsresult RememberCAChain(CERTCertList *aCertList);
 
@@ -176,6 +177,7 @@ protected:
   PRPackedBool mCanceled;
   PRPackedBool mHasCleartextPhase;
   PRPackedBool mHandshakeInProgress;
+  PRIntervalTime mHandshakeStartTime;
   PRInt32 mPort;
   nsXPIDLCString mHostName;
   CERTCertList *mCAChain;

security/manager/ssl/src/nsSSLThread.cpp

@@ -220,6 +220,7 @@ PRInt16 nsSSLThread::requestPoll(nsNSSSocketInfo *si, PRInt16 in_flags, PRInt16
   *out_flags = 0;
 
   PRBool want_sleep_and_wakeup_on_any_socket_activity = PR_FALSE;
+  PRBool handshake_timeout = PR_FALSE;
   
   {
     nsAutoLock threadLock(ssl_thread_singleton->mMutex);
@@ -302,6 +303,8 @@ PRInt16 nsSSLThread::requestPoll(nsNSSSocketInfo *si, PRInt16 in_flags, PRInt16
         
         case nsSSLSocketThreadData::ssl_idle:
         {
+          handshake_timeout = si->HandshakeTimeout();
+
           if (si != ssl_thread_singleton->mBusySocket)
           {
             // Some other socket is currently busy on the SSL thread.
@@ -327,6 +330,18 @@ PRInt16 nsSSLThread::requestPoll(nsNSSSocketInfo *si, PRInt16 in_flags, PRInt16
           break;
       }
     }
+    else
+    {
+      handshake_timeout = si->HandshakeTimeout();
+    }
+
+    if (handshake_timeout)
+    {
+      NS_ASSERTION(in_flags & PR_POLL_EXCEPT, "nsSSLThread::requestPoll handshake timeout, but caller did not poll for EXCEPT");
+
+      *out_flags |= PR_POLL_EXCEPT;
+      return in_flags;
+    }
   }
 
   if (want_sleep_and_wakeup_on_any_socket_activity)
@@ -463,6 +478,14 @@ PRInt32 nsSSLThread::requestRead(nsNSSSocketInfo *si, void *buf, PRInt32 amount)
     {
       some_other_socket_is_busy = PR_TRUE;
     }
+
+    if (!this_socket_is_busy && si->HandshakeTimeout())
+    {
+      restoreOriginalSocket_locked(si);
+      PR_SetError(PR_CONNECT_RESET_ERROR, 0);
+      checkHandshake(-1, si->mFd->lower, si);
+      return -1;
+    }
   }
 
   switch (my_ssl_state)
@@ -665,6 +688,14 @@ PRInt32 nsSSLThread::requestWrite(nsNSSSocketInfo *si, const void *buf, PRInt32
     {
       some_other_socket_is_busy = PR_TRUE;
     }
+
+    if (!this_socket_is_busy && si->HandshakeTimeout())
+    {
+      restoreOriginalSocket_locked(si);
+      PR_SetError(PR_CONNECT_RESET_ERROR, 0);
+      checkHandshake(-1, si->mFd->lower, si);
+      return -1;
+    }
   }
 
   switch (my_ssl_state)