fixes bug 122675 "socket transport doesn't cancel properly if busy resolving

or connecting" r=gordon, sr=mscott

netwerk/base/src/nsSocketTransport.cpp

@@ -414,6 +414,14 @@ nsresult nsSocketTransport::Process(PRInt16 aSelectFlags)
         done = PR_TRUE; // nothing to process
 
     while (!done) {
+        //
+        // Check for failure on any socket requests
+        //
+        if (mReadRequest && mReadRequest->HasFailed())
+            mReadRequest->GetStatus(&mStatus);
+        else if (mWriteRequest && mWriteRequest->HasFailed())
+            mWriteRequest->GetStatus(&mStatus);
+
         //
         // If an error has occurred then move into the error state...
         //
@@ -495,7 +503,7 @@ nsresult nsSocketTransport::Process(PRInt16 aSelectFlags)
             // Send status message
             // only send a status if doResolveHost is going to do some
             // resolution
-            if (mStatus != NS_OK)
+            if (mStatus == NS_BASE_STREAM_WOULD_BLOCK)
                 OnStatus_Locked(NS_NET_STATUS_RESOLVING_HOST);
             break;
 
@@ -636,6 +644,12 @@ nsresult nsSocketTransport::doResolveHost(void)
     LOG(("nsSocketTransport: Entering doResolveHost() [host=%s:%d this=%x].\n", 
         mHostName, mPort, this));
 
+    // 
+    // The hostname is being resolved...
+    //
+    if (mDNSRequest)
+        return NS_BASE_STREAM_WOULD_BLOCK;
+
     //
     // The hostname has not been resolved yet...
     //
@@ -1036,7 +1050,6 @@ nsSocketTransport::doReadWrite(PRInt16 aSelectFlags)
     if (mReadRequest) {
         if (mReadRequest->IsCanceled() || (mBytesExpected == 0)) {
             LOG(("nsSocketTransport: [this=%x] completing read request due to cancelation\n", this));
-            mSelectFlags &= ~PR_POLL_READ;
             mReadRequest->GetStatus(&readStatus);
             CompleteAsyncRead();
             if (NS_FAILED(readStatus))
@@ -1078,7 +1091,6 @@ nsSocketTransport::doReadWrite(PRInt16 aSelectFlags)
     if (mWriteRequest) {
         if (mWriteRequest->IsCanceled()) {
             LOG(("nsSocketTransport: [this=%x] completing write request due to cancelation\n", this));
-            mSelectFlags &= ~PR_POLL_WRITE;
             mWriteRequest->GetStatus(&writeStatus);
             CompleteAsyncWrite();
             if (NS_FAILED(writeStatus))

netwerk/base/src/nsSocketTransport.h

@@ -415,6 +415,7 @@ public:
     PRBool IsInitialized() { return mStartFired; }
     PRBool IsSuspended() { return mSuspendCount > 0; }
     PRBool IsCanceled() { return mCanceled; }
+    PRBool HasFailed() { return NS_FAILED(mStatus); }
 
     void SetTransport(nsSocketTransport *);
     void SetObserver(nsIRequestObserver *obs) { mObserver = obs; }

netwerk/dns/src/nsDnsService.cpp

@@ -825,6 +825,12 @@ nsDNSLookup::DoSyncLookup()
     PRStatus status;
     nsresult rv = NS_OK;
 
+    // must not hold the service lock while resolving, otherwise we could
+    // end up locking up the DNS service and anyone who accesses it until
+    // this function returns, which could be a while if the DNS server is
+    // unreachable. XXX is this thread-safe?
+    nsDNSService::Unlock();
+
     status = PR_GetIPNodeByName(mHostName,
                                 PR_AF_INET6,
                                 PR_AI_DEFAULT,
@@ -832,6 +838,8 @@ nsDNSLookup::DoSyncLookup()
                                 PR_NETDB_BUF_SIZE,
                                 &(mHostEntry.hostEnt));
 
+    nsDNSService::Lock();
+
     if (PR_SUCCESS != status)
         rv = NS_ERROR_UNKNOWN_HOST;