bug 599164 pipeline with type and state r=honzab

--HG--
extra : rebase_source : b060665a1726eab3e3ef0ab0d94c34cfea5c1460

content/base/src/nsXMLHttpRequest.cpp

@@ -2779,6 +2779,10 @@ nsXMLHttpRequest::Send(nsIVariant *aBody)
     }
   }
 
+  // Blocking gets are common enough out of XHR that we should mark
+  // the channel slow by default for pipeline purposes
+  AddLoadFlags(mChannel, nsIRequest::INHIBIT_PIPELINE);
+
   if (!IsSystemXHR()) {
     // Always create a nsCORSListenerProxy here even if it's
     // a same-origin request right now, since it could be redirected.

content/html/content/src/nsHTMLMediaElement.cpp

@@ -57,6 +57,8 @@
 #include "nsThreadUtils.h"
 #include "nsIThreadInternal.h"
 #include "nsContentUtils.h"
+#include "nsIRequest.h"
+
 #include "nsFrameManager.h"
 #include "nsIScriptSecurityManager.h"
 #include "nsIXPConnect.h"
@@ -2850,6 +2852,13 @@ void nsHTMLMediaElement::SetRequestHeaders(nsIHttpChannel* aChannel)
   // Send Accept header for video and audio types only (Bug 489071)
   SetAcceptHeader(aChannel);
 
+  // Media elements are likely candidates for HTTP Pipeline head of line
+  // blocking problems, so disable pipelines.
+  nsLoadFlags loadflags;
+  aChannel->GetLoadFlags(&loadflags);
+  loadflags |= nsIRequest::INHIBIT_PIPELINE;
+  aChannel->SetLoadFlags(loadflags);
+
   // Apache doesn't send Content-Length when gzip transfer encoding is used,
   // which prevents us from estimating the video length (if explicit Content-Duration
   // and a length spec in the container are not present either) and from seeking.

modules/libpref/src/init/all.js

@@ -806,7 +806,13 @@ pref("network.http.pipelining.ssl"  , false); // disable pipelining over SSL
 pref("network.http.proxy.pipelining", false);
 
 // Max number of requests in the pipeline
-pref("network.http.pipelining.maxrequests" , 4);
+pref("network.http.pipelining.maxrequests" , 32);
+
+// An optimistic request is one pipelined when policy might allow a new
+// connection instead
+pref("network.http.pipelining.max-optimistic-requests" , 4);
+
+pref("network.http.pipelining.aggressive", false);
 
 // Prompt for 307 redirects
 pref("network.http.prompt-temp-redirect", true);

netwerk/base/public/nsIRequest.idl

@@ -159,6 +159,13 @@ interface nsIRequest : nsISupports
      * The following flags control the flow of data into the cache.
      */
 
+    /**
+     *  This flag prevents loading of the request with an HTTP pipeline.
+     *  Generally this is because the resource is expected to take a
+     *  while to load and may cause head of line blocking problems.
+     */
+    const unsigned long INHIBIT_PIPELINE = 1 << 6;
+
     /**
      * This flag prevents caching of any kind.  It does not, however, prevent
      * cached content from being used to satisfy this request.

netwerk/protocol/http/SpdySession.cpp

@@ -2064,8 +2064,7 @@ SpdySession::SetSSLConnectFailed()
 bool
 SpdySession::IsDone()
 {
-  NS_ABORT_IF_FALSE(false, "SpdySession::IsDone()");
+  return !mStreamTransactionHash.Count();
-  return false;
 }
 
 nsresult
@@ -2151,12 +2150,10 @@ SpdySession::AddTransaction(nsAHttpTransaction *)
   return NS_ERROR_NOT_IMPLEMENTED;
 }
 
-PRUint16
+PRUint32
-SpdySession::PipelineDepthAvailable()
+SpdySession::PipelineDepth()
 {
-  // any attempt at pipelining will be turned into parallelism
+  return IsDone() ? 0 : 1;
-
-  return 0;
 }
 
 nsresult

netwerk/protocol/http/nsAHttpTransaction.h

@@ -63,6 +63,8 @@ class nsAHttpTransaction : public nsISupports
 public:
     // called by the connection when it takes ownership of the transaction.
     virtual void SetConnection(nsAHttpConnection *) = 0;
+
+    // used to obtain the connection associated with this transaction
     virtual nsAHttpConnection *Connection() = 0;
 
     // called by the connection to get security callbacks to set on the
@@ -121,14 +123,38 @@ public:
     // return NS_ERROR_NOT_IMPLEMENTED
     virtual nsresult AddTransaction(nsAHttpTransaction *transaction) = 0;
     
-    // called to count the number of sub transactions that can be added
+    // The total length of the outstanding pipeline comprised of transacations
-    virtual PRUint16 PipelineDepthAvailable() = 0;
+    // and sub-transactions.
+    virtual PRUint32 PipelineDepth() = 0;
 
     // Used to inform the connection that it is being used in a pipelined
     // context. That may influence the handling of some errors.
-    // The value is the pipeline position.
+    // The value is the pipeline position (> 1).
     virtual nsresult SetPipelinePosition(PRInt32) = 0;
     virtual PRInt32  PipelinePosition() = 0;
+
+    // Every transaction is classified into one of the types below. When using
+    // HTTP pipelines, only transactions with the same type appear on the same
+    // pipeline.
+    enum Classifier  {
+        // Transactions that expect a short 304 (no-content) response
+        CLASS_REVALIDATION,
+
+        // Transactions for content expected to be CSS or JS
+        CLASS_SCRIPT,
+
+        // Transactions for content expected to be an image
+        CLASS_IMAGE,
+
+        // Transactions that cannot involve a pipeline 
+        CLASS_SOLO,
+
+        // Transactions that do not fit any of the other categories. HTML
+        // is normally GENERAL.
+        CLASS_GENERAL,
+
+        CLASS_MAX
+    };
 };
 
 #define NS_DECL_NSAHTTPTRANSACTION \
@@ -150,7 +176,7 @@ public:
     PRUint32 Http1xTransactionCount();                                  \
     nsresult TakeSubTransactions(nsTArray<nsRefPtr<nsAHttpTransaction> > &outTransactions); \
     nsresult AddTransaction(nsAHttpTransaction *);                      \
-    PRUint16 PipelineDepthAvailable();                                  \
+    PRUint32 PipelineDepth();                                           \
     nsresult SetPipelinePosition(PRInt32);                              \
     PRInt32  PipelinePosition();
 

netwerk/protocol/http/nsHttp.h

@@ -140,9 +140,6 @@ typedef PRUint8 nsHttpVersion;
 // some default values
 //-----------------------------------------------------------------------------
 
-// hard upper limit on the number of requests that can be pipelined
-#define NS_HTTP_MAX_PIPELINED_REQUESTS 8 
-
 #define NS_HTTP_DEFAULT_PORT  80
 #define NS_HTTPS_DEFAULT_PORT 443
 

netwerk/protocol/http/nsHttpChannel.cpp

@@ -502,16 +502,17 @@ nsHttpChannel::SetupTransaction()
     if (mCaps & NS_HTTP_ALLOW_PIPELINING) {
         //
         // disable pipelining if:
-        //   (1) pipelining has been explicitly disabled
+        //   (1) pipelining has been disabled by config
-        //   (2) request corresponds to a top-level document load (link click)
+        //   (2) pipelining has been disabled by connection mgr info
-        //   (3) request method is non-idempotent
+        //   (3) request corresponds to a top-level document load (link click)
+        //   (4) request method is non-idempotent
+        //   (5) request is marked slow (e.g XHR)
         //
-        // XXX does the toplevel document check really belong here?  or, should
+        if (!mAllowPipelining ||
-        //     we push it out entirely to necko consumers?
+           (mLoadFlags & (LOAD_INITIAL_DOCUMENT_URI | INHIBIT_PIPELINE)) ||
-        //
-        if (!mAllowPipelining || (mLoadFlags & LOAD_INITIAL_DOCUMENT_URI) ||
             !(mRequestHead.Method() == nsHttp::Get ||
               mRequestHead.Method() == nsHttp::Head ||
+              mRequestHead.Method() == nsHttp::Options ||
               mRequestHead.Method() == nsHttp::Propfind ||
               mRequestHead.Method() == nsHttp::Proppatch)) {
             LOG(("  pipelining disallowed\n"));
@@ -1772,7 +1773,10 @@ nsHttpChannel::EnsureAssocReq()
                    endofmethod - method)) {
         LOG(("  Assoc-Req failure Method %s", method));
         if (mConnectionInfo)
-            mConnectionInfo->BanPipelining();
+            gHttpHandler->ConnMgr()->
+                PipelineFeedbackInfo(mConnectionInfo,
+                                     nsHttpConnectionMgr::RedCorruptedContent,
+                                     nsnull, 0);
 
         nsCOMPtr<nsIConsoleService> consoleService =
             do_GetService(NS_CONSOLESERVICE_CONTRACTID);
@@ -1802,7 +1806,10 @@ nsHttpChannel::EnsureAssocReq()
     if (!equals) {
         LOG(("  Assoc-Req failure URL %s", assoc_val));
         if (mConnectionInfo)
-            mConnectionInfo->BanPipelining();
+            gHttpHandler->ConnMgr()->
+                PipelineFeedbackInfo(mConnectionInfo,
+                                     nsHttpConnectionMgr::RedCorruptedContent,
+                                     nsnull, 0);
 
         nsCOMPtr<nsIConsoleService> consoleService =
             do_GetService(NS_CONSOLESERVICE_CONTRACTID);

netwerk/protocol/http/nsHttpConnection.cpp

@@ -72,8 +72,6 @@ using namespace mozilla::net;
 
 nsHttpConnection::nsHttpConnection()
     : mTransaction(nsnull)
-    , mLastReadTime(0)
-    , mIdleTimeout(0)
     , mConsiderReusedAfterInterval(0)
     , mConsiderReusedAfterEpoch(0)
     , mCurrentBytesRead(0)
@@ -88,6 +86,7 @@ nsHttpConnection::nsHttpConnection()
     , mIdleMonitoring(false)
     , mProxyConnectInProgress(false)
     , mHttp1xTransactionCount(0)
+    , mClassification(nsAHttpTransaction::CLASS_GENERAL)
     , mNPNComplete(false)
     , mSetupNPNCalled(false)
     , mUsingSpdy(false)
@@ -142,21 +141,25 @@ nsHttpConnection::Init(nsHttpConnectionInfo *info,
                        nsIAsyncInputStream *instream,
                        nsIAsyncOutputStream *outstream,
                        nsIInterfaceRequestor *callbacks,
-                       nsIEventTarget *callbackTarget)
+                       nsIEventTarget *callbackTarget,
+                       PRIntervalTime rtt)
 {
     NS_ABORT_IF_FALSE(transport && instream && outstream,
                       "invalid socket information");
     LOG(("nsHttpConnection::Init [this=%p "
-         "transport=%p instream=%p outstream=%p]\n",
+         "transport=%p instream=%p outstream=%p rtt=%d]\n",
-         this, transport, instream, outstream));
+         this, transport, instream, outstream,
+         PR_IntervalToMilliseconds(rtt)));
 
     NS_ENSURE_ARG_POINTER(info);
     NS_ENSURE_TRUE(!mConnInfo, NS_ERROR_ALREADY_INITIALIZED);
 
     mConnInfo = info;
-    mSupportsPipelining = mConnInfo->SupportsPipelining();
-    mMaxHangTime = PR_SecondsToInterval(maxHangTime);
     mLastReadTime = PR_IntervalNow();
+    mSupportsPipelining =
+        gHttpHandler->ConnMgr()->SupportsPipelining(mConnInfo);
+    mRtt = rtt;
+    mMaxHangTime = PR_SecondsToInterval(maxHangTime);
 
     mSocketTransport = transport;
     mSocketIn = instream;
@@ -340,6 +343,9 @@ nsHttpConnection::Activate(nsAHttpTransaction *trans, PRUint8 caps, PRInt32 pri)
     NS_ENSURE_ARG_POINTER(trans);
     NS_ENSURE_TRUE(!mTransaction, NS_ERROR_IN_PROGRESS);
 
+    // reset the read timers to wash away any idle time
+    mLastReadTime = PR_IntervalNow();
+
     // Update security callbacks
     nsCOMPtr<nsIInterfaceRequestor> callbacks;
     nsCOMPtr<nsIEventTarget>        callbackTarget;
@@ -537,7 +543,7 @@ nsHttpConnection::CanReuse()
         canReuse = mSpdySession->CanReuse();
     else
         canReuse = IsKeepAlive();
-    
+
     canReuse = canReuse && (IdleTime() < mIdleTimeout) && IsAlive();
 
     // An idle persistent connection should not have data waiting to be read
@@ -622,21 +628,19 @@ nsHttpConnection::SupportsPipelining(nsHttpResponseHead *responseHead)
     if (mUsingSpdy)
         return false;
 
-    // XXX there should be a strict mode available that disables this
-    // blacklisting.
-
     // assuming connection is HTTP/1.1 with keep-alive enabled
     if (mConnInfo->UsingHttpProxy() && !mConnInfo->UsingSSL()) {
         // XXX check for bad proxy servers...
         return true;
     }
 
-    // XXX what about checking for a Via header? (transparent proxies)
-
     // check for bad origin servers
     const char *val = responseHead->PeekHeader(nsHttp::Server);
+
+    // If there is no server header we will assume it should not be banned
+    // as facebook and some other prominent sites do this
     if (!val)
-        return false; // no header, no love
+        return true;
 
     // The blacklist is indexed by the first character. All of these servers are
     // known to return their identifier as the first thing in the server string,
@@ -663,6 +667,8 @@ nsHttpConnection::SupportsPipelining(nsHttpResponseHead *responseHead)
         for (int i = 0; bad_servers[index][i] != nsnull; i++) {
             if (!PL_strncmp (val, bad_servers[index][i], strlen (bad_servers[index][i]))) {
                 LOG(("looks like this server does not support pipelining"));
+                gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+                    mConnInfo, nsHttpConnectionMgr::RedBannedServer, this , 0);
                 return false;
             }
         }
@@ -718,11 +724,19 @@ nsHttpConnection::OnHeadersAvailable(nsAHttpTransaction *trans,
             mKeepAlive = true;
         else
             mKeepAlive = false;
+        
+        // We need at least version 1.1 to use pipelines
+        gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+            mConnInfo, nsHttpConnectionMgr::RedVersionTooLow, this, 0);
     }
     else {
         // HTTP/1.1 connections are by default persistent
-        if (val && !PL_strcasecmp(val, "close")) 
+        if (val && !PL_strcasecmp(val, "close")) {
             mKeepAlive = false;
+            // persistent connections are required for pipelining to work
+            gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+                mConnInfo, nsHttpConnectionMgr::BadExplicitClose, this, 0);
+        }
         else {
             mKeepAlive = true;
 
@@ -741,7 +755,26 @@ nsHttpConnection::OnHeadersAvailable(nsAHttpTransaction *trans,
     // and also read it back. It is possible the ci status is
     // locked to false if pipelining has been banned on this ci due to
     // some kind of observed flaky behavior
-    mSupportsPipelining = mConnInfo->SetSupportsPipelining(mSupportsPipelining);
+    if (mSupportsPipelining) {
+        // report the pipelining-compatible header to the connection manager
+        // as positive feedback. This will undo 1 penalty point the host
+        // may have accumulated in the past.
+
+        gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+            mConnInfo, nsHttpConnectionMgr::NeutralExpectedOK, this, 0);
+
+        mSupportsPipelining =
+            gHttpHandler->ConnMgr()->SupportsPipelining(mConnInfo);
+    }
+
+    // If this connection is reserved for revalidations and we are
+    // receiving a document that failed revalidation then switch the
+    // classification to general to avoid pipelining more revalidations behind
+    // it.
+    if (mClassification == nsAHttpTransaction::CLASS_REVALIDATION &&
+        responseHead->Status() != 304) {
+        mClassification = nsAHttpTransaction::CLASS_GENERAL;
+    }
     
     // if this connection is persistent, then the server may send a "Keep-Alive"
     // header specifying the maximum number of times the connection can be
@@ -758,13 +791,13 @@ nsHttpConnection::OnHeadersAvailable(nsAHttpTransaction *trans,
             if (cp)
                 mIdleTimeout = PR_SecondsToInterval((PRUint32) atoi(cp + 8));
             else
-                mIdleTimeout = gHttpHandler->IdleTimeout();
+                mIdleTimeout = gHttpHandler->SpdyTimeout();
         }
         else {
             mIdleTimeout = gHttpHandler->SpdyTimeout();
         }
         
-        LOG(("Connection can be reused [this=%x idle-timeout=%usec]\n",
+        LOG(("Connection can be reused [this=%p idle-timeout=%usec]\n",
              this, PR_IntervalToSeconds(mIdleTimeout)));
     }
 
@@ -939,6 +972,14 @@ nsHttpConnection::ResumeRecv()
 
     NS_ASSERTION(PR_GetCurrentThread() == gSocketThread, "wrong thread");
 
+    // the mLastReadTime timestamp is used for finding slowish readers
+    // and can be pretty sensitive. For that reason we actually reset it
+    // when we ask to read (resume recv()) so that when we get called back
+    // with actual read data in OnSocketReadable() we are only measuring
+    // the latency between those two acts and not all the processing that
+    // may get done before the ResumeRecv() call
+    mLastReadTime = PR_IntervalNow();
+
     if (mSocketIn)
         return mSocketIn->AsyncWait(this, 0, 0, nsnull);
 
@@ -1061,7 +1102,8 @@ nsHttpConnection::OnReadSegment(const char *buf,
 nsresult
 nsHttpConnection::OnSocketWritable()
 {
-    LOG(("nsHttpConnection::OnSocketWritable [this=%x]\n", this));
+    LOG(("nsHttpConnection::OnSocketWritable [this=%p] host=%s\n",
+         this, mConnInfo->Host()));
 
     nsresult rv;
     PRUint32 n;
@@ -1141,7 +1183,7 @@ nsHttpConnection::OnSocketWritable()
                                             nsISocketTransport::STATUS_WAITING_FOR,
                                             LL_ZERO);
 
-            rv = mSocketIn->AsyncWait(this, 0, 0, nsnull); // start reading
+            rv = ResumeRecv(); // start reading
             again = false;
         }
         // write more to the socket until error or end-of-request...
@@ -1180,14 +1222,41 @@ nsHttpConnection::OnSocketReadable()
     LOG(("nsHttpConnection::OnSocketReadable [this=%x]\n", this));
 
     PRIntervalTime now = PR_IntervalNow();
+    PRIntervalTime delta = now - mLastReadTime;
 
-    if (mKeepAliveMask && ((now - mLastReadTime) >= mMaxHangTime)) {
+    if (mKeepAliveMask && (delta >= mMaxHangTime)) {
         LOG(("max hang time exceeded!\n"));
         // give the handler a chance to create a new persistent connection to
         // this host if we've been busy for too long.
         mKeepAliveMask = false;
         gHttpHandler->ProcessPendingQ(mConnInfo);
     }
+
+    // Look for data being sent in bursts with large pauses. If the pauses
+    // are caused by server bottlenecks such as think-time, disk i/o, or
+    // cpu exhaustion (as opposed to network latency) then we generate negative
+    // pipelining feedback to prevent head of line problems
+    
+    // Reduce the estimate of the time since last read by up to 1 RTT to
+    // accommodate exhausted sender TCP congestion windows or minor I/O delays.
+
+    if (delta > mRtt)
+        delta -= mRtt;
+    else
+        delta = 0;
+
+    const PRIntervalTime k400ms  = PR_MillisecondsToInterval(400);
+    const PRIntervalTime k1200ms = PR_MillisecondsToInterval(1200);
+
+    if (delta > k1200ms) {
+        gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+            mConnInfo, nsHttpConnectionMgr::BadSlowReadMajor, this, 0);
+    }
+    else if (delta > k400ms) {
+        gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+            mConnInfo, nsHttpConnectionMgr::BadSlowReadMinor, this, 0);
+    }
+
     mLastReadTime = now;
 
     nsresult rv;
@@ -1209,7 +1278,7 @@ nsHttpConnection::OnSocketReadable()
             if (NS_FAILED(mSocketInCondition)) {
                 // continue waiting for the socket if necessary...
                 if (mSocketInCondition == NS_BASE_STREAM_WOULD_BLOCK)
-                    rv = mSocketIn->AsyncWait(this, 0, 0, nsnull);
+                    rv = ResumeRecv();
                 else
                     rv = mSocketInCondition;
                 again = false;

netwerk/protocol/http/nsHttpConnection.h

@@ -47,6 +47,7 @@
 #include "nsAutoPtr.h"
 #include "prinrval.h"
 #include "SpdySession.h"
+#include "mozilla/TimeStamp.h"
 
 #include "nsIStreamListener.h"
 #include "nsISocketTransport.h"
@@ -92,7 +93,7 @@ public:
     nsresult Init(nsHttpConnectionInfo *info, PRUint16 maxHangTime,
                   nsISocketTransport *, nsIAsyncInputStream *,
                   nsIAsyncOutputStream *, nsIInterfaceRequestor *,
-                  nsIEventTarget *);
+                  nsIEventTarget *, PRIntervalTime);
 
     // Activate causes the given transaction to be processed on this
     // connection.  It fails if there is already an existing transaction unless
@@ -105,7 +106,7 @@ public:
     //-------------------------------------------------------------------------
     // XXX document when these are ok to call
 
-    bool     SupportsPipelining() { return mSupportsPipelining; }
+    bool     SupportsPipelining() { return mSupportsPipelining && IsKeepAlive(); }
     bool     IsKeepAlive() { return mUsingSpdy ||
                                     (mKeepAliveMask && mKeepAlive); }
     bool     CanReuse();   // can this connection be reused?
@@ -168,6 +169,12 @@ public:
     // When the connection is active this is called every 15 seconds
     void  ReadTimeoutTick(PRIntervalTime now);
 
+    nsAHttpTransaction::Classifier Classification() { return mClassification; }
+    void Classify(nsAHttpTransaction::Classifier newclass)
+    {
+        mClassification = newclass;
+    }
+
 private:
     // called to cause the underlying socket to start speaking SSL
     nsresult ProxyStartSSL();
@@ -217,7 +224,7 @@ private:
 
     nsRefPtr<nsHttpConnectionInfo> mConnInfo;
 
-    PRUint32                        mLastReadTime;
+    PRIntervalTime                  mLastReadTime;
     PRIntervalTime                  mMaxHangTime;    // max download time before dropping keep-alive status
     PRIntervalTime                  mIdleTimeout;    // value of keep-alive: timeout=
     PRIntervalTime                  mConsiderReusedAfterInterval;
@@ -228,6 +235,8 @@ private:
 
     nsRefPtr<nsIAsyncInputStream>   mInputOverflow;
 
+    PRIntervalTime                  mRtt;
+
     bool                            mKeepAlive;
     bool                            mKeepAliveMask;
     bool                            mSupportsPipelining;
@@ -241,6 +250,8 @@ private:
     // excludes spdy transactions.
     PRUint32                        mHttp1xTransactionCount;
 
+    nsAHttpTransaction::Classifier  mClassification;
+
     // SPDY related
     bool                            mNPNComplete;
     bool                            mSetupNPNCalled;

netwerk/protocol/http/nsHttpConnectionInfo.cpp

@@ -100,27 +100,6 @@ nsHttpConnectionInfo::Clone() const
     return clone;
 }
 
-bool
-nsHttpConnectionInfo::SupportsPipelining()
-{
-    return mSupportsPipelining;
-}
-
-bool
-nsHttpConnectionInfo::SetSupportsPipelining(bool support)
-{
-    if (!mBannedPipelining)
-        mSupportsPipelining = support;
-    return mSupportsPipelining;
-}
-
-void
-nsHttpConnectionInfo::BanPipelining()
-{
-    mBannedPipelining = true;
-    mSupportsPipelining = false;
-}
-
 bool
 nsHttpConnectionInfo::ShouldForceConnectMethod()
 {

netwerk/protocol/http/nsHttpConnectionInfo.h

@@ -60,8 +60,6 @@ public:
         : mRef(0)
         , mProxyInfo(proxyInfo)
         , mUsingSSL(usingSSL)
-        , mSupportsPipelining(false)
-        , mBannedPipelining(false)
     {
         LOG(("Creating nsHttpConnectionInfo @%x\n", this));
 
@@ -131,10 +129,6 @@ public:
     bool          ShouldForceConnectMethod();
     const nsCString &GetHost() { return mHost; }
 
-    bool          SupportsPipelining();
-    bool          SetSupportsPipelining(bool support);
-    void          BanPipelining();
-
 private:
     nsrefcnt               mRef;
     nsCString              mHashKey;
@@ -143,8 +137,6 @@ private:
     nsCOMPtr<nsProxyInfo>  mProxyInfo;
     bool                   mUsingHttpProxy;
     bool                   mUsingSSL;
-    bool                   mSupportsPipelining;
-    bool                   mBannedPipelining;
 };
 
 #endif // nsHttpConnectionInfo_h__

netwerk/protocol/http/nsHttpConnectionMgr.h

@@ -49,6 +49,7 @@
 #include "nsAutoPtr.h"
 #include "mozilla/ReentrantMonitor.h"
 #include "nsISocketTransportService.h"
+#include "mozilla/TimeStamp.h"
 
 #include "nsIObserver.h"
 #include "nsITimer.h"
@@ -72,7 +73,8 @@ public:
         MAX_PERSISTENT_CONNECTIONS_PER_HOST,
         MAX_PERSISTENT_CONNECTIONS_PER_PROXY,
         MAX_REQUEST_DELAY,
-        MAX_PIPELINED_REQUESTS
+        MAX_PIPELINED_REQUESTS,
+        MAX_OPTIMISTIC_PIPELINED_REQUESTS
     };
 
     //-------------------------------------------------------------------------
@@ -87,7 +89,8 @@ public:
                   PRUint16 maxPersistentConnectionsPerHost,
                   PRUint16 maxPersistentConnectionsPerProxy,
                   PRUint16 maxRequestDelay,
-                  PRUint16 maxPipelinedRequests);
+                  PRUint16 maxPipelinedRequests,
+                  PRUint16 maxOptimisticPipelinedRequests);
     nsresult Shutdown();
 
     //-------------------------------------------------------------------------
@@ -138,17 +141,73 @@ public:
     void ReportSpdyAlternateProtocol(nsHttpConnection *);
     void RemoveSpdyAlternateProtocol(nsACString &key);
 
+    // Pipielining Interfaces and Datatypes
+
+    const static PRUint32 kPipelineInfoTypeMask = 0xffff0000;
+    const static PRUint32 kPipelineInfoIDMask   = ~kPipelineInfoTypeMask;
+
+    const static PRUint32 kPipelineInfoTypeRed     = 0x00010000;
+    const static PRUint32 kPipelineInfoTypeBad     = 0x00020000;
+    const static PRUint32 kPipelineInfoTypeNeutral = 0x00040000;
+    const static PRUint32 kPipelineInfoTypeGood    = 0x00080000;
+
+    enum PipelineFeedbackInfoType
+    {
+        // Used when an HTTP response less than 1.1 is received
+        RedVersionTooLow = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0001,
+
+        // Used when a HTTP Server response header that is on the banned from
+        // pipelining list is received
+        RedBannedServer = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0002,
+    
+        // Used when a response is terminated early, or when it fails an
+        // integrity check such as assoc-req or md5
+        RedCorruptedContent = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0004,
+
+        // Used when a pipeline is only partly satisfied - for instance if the
+        // server closed the connection after responding to the first
+        // request but left some requests unprocessed.
+        RedCanceledPipeline = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0005,
+
+        // Used when a connection that we expected to stay persistently open
+        // was closed by the server. Not used when simply timed out.
+        BadExplicitClose = kPipelineInfoTypeBad | 0x0003,
+
+        // Used when there is a gap of around 400 - 1200ms in between data being
+        // read from the server
+        BadSlowReadMinor = kPipelineInfoTypeBad | 0x0006,
+
+        // Used when there is a gap of > 1200ms in between data being
+        // read from the server
+        BadSlowReadMajor = kPipelineInfoTypeBad | 0x0007,
+
+        // Used when a response is received that is not framed with either chunked
+        // encoding or a complete content length.
+        BadInsufficientFraming = kPipelineInfoTypeBad | 0x0008,
+
+        // Used when a response is received that has headers that appear to support
+        // pipelining.
+        NeutralExpectedOK = kPipelineInfoTypeNeutral | 0x0009,
+
+        // Used when a response is received successfully to a pipelined request.
+        GoodCompletedOK = kPipelineInfoTypeGood | 0x000A
+    };
+    
+    // called to provide information relevant to the pipelining manager
+    // may be called from any thread
+    void     PipelineFeedbackInfo(nsHttpConnectionInfo *,
+                                  PipelineFeedbackInfoType info,
+                                  nsHttpConnection *,
+                                  PRUint32);
+
     //-------------------------------------------------------------------------
     // NOTE: functions below may be called only on the socket thread.
     //-------------------------------------------------------------------------
 
-    // removes the next transactions for the specified connection from the
-    // pending transaction queue.
-    void AddTransactionToPipeline(nsHttpPipeline *);
-
     // called to force the transaction queue to be processed once more, giving
     // preference to the specified connection.
     nsresult ProcessPendingQ(nsHttpConnectionInfo *);
+    bool     ProcessPendingQForEntry(nsHttpConnectionInfo *);
 
     // This is used to force an idle connection to be closed and removed from
     // the idle connection list. It is called when the idle connection detects
@@ -161,29 +220,40 @@ public:
     void ReportSpdyConnection(nsHttpConnection *, bool usingSpdy);
 
     
-    // Similar to ProcessPendingQ, but only considers adding transactions to
+    bool     SupportsPipelining(nsHttpConnectionInfo *);
-    // existing connections
+
-    bool     ProcessPipelinePendingQForCI(nsHttpConnectionInfo *);
 private:
     virtual ~nsHttpConnectionMgr();
-    class nsHalfOpenSocket;
+
+    enum PipeliningState {
+        // Host has proven itself pipeline capable through past experience and
+        // large pipeline depths are allowed on multiple connections.
+        PS_GREEN,
+
+        // Not enough information is available yet with this host to be certain
+        // of pipeline capability. Small pipelines on a single connection are
+        // allowed in order to decide whether or not to proceed to green.
+        PS_YELLOW,
+
+        // One or more bad events has happened that indicate that pipelining
+        // to this host (or a particular type of transaction with this host)
+        // is a bad idea. Pipelining is not currently allowed, but time and
+        // other positive experiences will eventually allow it to try again.
+        PS_RED
+    };
     
+    class nsHalfOpenSocket;
+
     // nsConnectionEntry
     //
     // mCT maps connection info hash key to nsConnectionEntry object, which
     // contains list of active and idle connections as well as the list of
     // pending transactions.
     //
-    struct nsConnectionEntry
+    class nsConnectionEntry
     {
-        nsConnectionEntry(nsHttpConnectionInfo *ci)
+    public:
-          : mConnInfo(ci),
+        nsConnectionEntry(nsHttpConnectionInfo *ci);
-            mUsingSpdy(false),
-            mTestedSpdy(false),
-            mSpdyPreferred(false)
-        {
-            NS_ADDREF(mConnInfo);
-        }
         ~nsConnectionEntry();
 
         nsHttpConnectionInfo        *mConnInfo;
@@ -192,6 +262,54 @@ private:
         nsTArray<nsHttpConnection*>  mIdleConns;   // idle persistent connections
         nsTArray<nsHalfOpenSocket*>  mHalfOpens;
 
+        // Pipeline depths for various states
+        const static PRUint32 kPipelineUnlimited  = 1024; // fully open - extended green
+        const static PRUint32 kPipelineOpen       = 6;    // 6 on each conn - normal green
+        const static PRUint32 kPipelineRestricted = 2;    // 2 on just 1 conn in yellow
+        
+        nsHttpConnectionMgr::PipeliningState PipelineState();
+        void OnPipelineFeedbackInfo(
+            nsHttpConnectionMgr::PipelineFeedbackInfoType info,
+            nsHttpConnection *, PRUint32);
+        bool SupportsPipelining();
+        PRUint32 MaxPipelineDepth(nsAHttpTransaction::Classifier classification);
+        void CreditPenalty();
+
+        nsHttpConnectionMgr::PipeliningState mPipelineState;
+
+        void SetYellowConnection(nsHttpConnection *);
+        void OnYellowComplete();
+        PRUint32                  mYellowGoodEvents;
+        PRUint32                  mYellowBadEvents;
+        nsHttpConnection         *mYellowConnection;
+
+        // initialGreenDepth is the max depth of a pipeline when you first
+        // transition to green. Normally this is kPipelineOpen, but it can
+        // be kPipelineUnlimited in aggressive mode.
+        PRUint32                  mInitialGreenDepth;
+
+        // greenDepth is the current max allowed depth of a pipeline when
+        // in the green state. Normally this starts as kPipelineOpen and
+        // grows to kPipelineUnlimited after a pipeline of depth 3 has been
+        // successfully transacted.
+        PRUint32                  mGreenDepth;
+
+        // pipeliningPenalty is the current amount of penalty points this host
+        // entry has earned for participating in events that are not conducive
+        // to good pipelines - such as head of line blocking, canceled pipelines,
+        // etc.. penalties are paid back either through elapsed time or simply
+        // healthy transactions. Having penalty points means that this host is
+        // not currently eligible for pipelines.
+        PRInt16                   mPipeliningPenalty;
+
+        // some penalty points only apply to particular classifications of
+        // transactions - this allows a server that perhaps has head of line
+        // blocking problems on CGI queries to still serve JS pipelined.
+        PRInt16                   mPipeliningClassPenalty[nsAHttpTransaction::CLASS_MAX];
+
+        // for calculating penalty repair credits
+        mozilla::TimeStamp        mLastCreditTime;
+
         // Spdy sometimes resolves the address in the socket manager in order
         // to re-coalesce sharded HTTP hosts. The dotted decimal address is
         // combined with the Anonymous flag from the connection information
@@ -276,6 +394,9 @@ private:
         nsCOMPtr<nsIAsyncOutputStream> mStreamOut;
         nsCOMPtr<nsIAsyncInputStream>  mStreamIn;
 
+        mozilla::TimeStamp             mPrimarySynStarted;
+        mozilla::TimeStamp             mBackupSynStarted;
+
         // for syn retry
         nsCOMPtr<nsITimer>             mSynTimer;
         nsCOMPtr<nsISocketTransport>   mBackupTransport;
@@ -300,7 +421,7 @@ private:
     PRUint16 mMaxPersistConnsPerProxy;
     PRUint16 mMaxRequestDelay; // in seconds
     PRUint16 mMaxPipelinedRequests;
-
+    PRUint16 mMaxOptimisticPipelinedRequests;
     bool mIsShuttingDown;
 
     //-------------------------------------------------------------------------
@@ -314,13 +435,18 @@ private:
     static PLDHashOperator PurgeExcessIdleConnectionsCB(const nsACString &, nsAutoPtr<nsConnectionEntry> &, void *);
     static PLDHashOperator ClosePersistentConnectionsCB(const nsACString &, nsAutoPtr<nsConnectionEntry> &, void *);
     bool     ProcessPendingQForEntry(nsConnectionEntry *);
+    bool     IsUnderPressure(nsConnectionEntry *ent,
+                             nsHttpTransaction::Classifier classification);
     bool     AtActiveConnectionLimit(nsConnectionEntry *, PRUint8 caps);
-    bool     ProcessPipelinePendingQForEntry(nsConnectionEntry *);
+    nsresult TryDispatchTransaction(nsConnectionEntry *ent,
-    void     GetConnection(nsConnectionEntry *, nsHttpTransaction *,
+                                    bool onlyReusedConnection,
-                           bool, nsHttpConnection **);
+                                    nsHttpTransaction *trans);
-    nsresult DispatchTransaction(nsConnectionEntry *, nsHttpTransaction *,
+    nsresult DispatchTransaction(nsConnectionEntry *,
-                                 PRUint8 caps, nsHttpConnection *);
+                                 nsHttpTransaction *,
-    bool     BuildPipeline(nsConnectionEntry *, nsAHttpTransaction *, nsHttpPipeline **);
+                                 nsHttpConnection *);
+    nsresult BuildPipeline(nsConnectionEntry *,
+                           nsAHttpTransaction *,
+                           nsHttpPipeline **);
     nsresult ProcessNewTransaction(nsHttpTransaction *);
     nsresult EnsureSocketThreadTargetIfOnline();
     void     ClosePersistentConnections(nsConnectionEntry *ent);
@@ -329,6 +455,13 @@ private:
     void     StartedConnect();
     void     RecvdConnect();
 
+    bool     MakeNewConnection(nsConnectionEntry *ent,
+                               nsHttpTransaction *trans);
+    bool     AddToShortestPipeline(nsConnectionEntry *ent,
+                                   nsHttpTransaction *trans,
+                                   nsHttpTransaction::Classifier classification,
+                                   PRUint16 depthLimit);
+
     // Manage the preferred spdy connection entry for this address
     nsConnectionEntry *GetSpdyPreferredEnt(nsConnectionEntry *aOriginalEntry);
     void               RemoveSpdyPreferredEnt(nsACString &aDottedDecimal);
@@ -401,6 +534,7 @@ private:
     void OnMsgReclaimConnection    (PRInt32, void *);
     void OnMsgUpdateParam          (PRInt32, void *);
     void OnMsgClosePersistentConnections (PRInt32, void *);
+    void OnMsgProcessFeedback      (PRInt32, void *);
 
     // Total number of active connections in all of the ConnectionEntry objects
     // that are accessed from mCT connection table.

netwerk/protocol/http/nsHttpHandler.cpp

@@ -185,7 +185,9 @@ nsHttpHandler::nsHttpHandler()
     , mMaxConnectionsPerServer(8)
     , mMaxPersistentConnectionsPerServer(2)
     , mMaxPersistentConnectionsPerProxy(4)
-    , mMaxPipelinedRequests(2)
+    , mMaxPipelinedRequests(32)
+    , mMaxOptimisticPipelinedRequests(4)
+    , mPipelineAggressive(false)
     , mRedirectionLimit(10)
     , mPhishyUserPassLength(1)
     , mQoSBits(0x00)
@@ -364,7 +366,8 @@ nsHttpHandler::InitConnectionMgr()
                         mMaxPersistentConnectionsPerServer,
                         mMaxPersistentConnectionsPerProxy,
                         mMaxRequestDelay,
-                        mMaxPipelinedRequests);
+                        mMaxPipelinedRequests,
+                        mMaxOptimisticPipelinedRequests);
     return rv;
 }
 
@@ -1016,13 +1019,31 @@ nsHttpHandler::PrefsChanged(nsIPrefBranch *prefs, const char *pref)
     if (PREF_CHANGED(HTTP_PREF("pipelining.maxrequests"))) {
         rv = prefs->GetIntPref(HTTP_PREF("pipelining.maxrequests"), &val);
         if (NS_SUCCEEDED(rv)) {
-            mMaxPipelinedRequests = clamped(val, 1, NS_HTTP_MAX_PIPELINED_REQUESTS);
+            mMaxPipelinedRequests = clamped(val, 1, 0xffff);
             if (mConnMgr)
                 mConnMgr->UpdateParam(nsHttpConnectionMgr::MAX_PIPELINED_REQUESTS,
                                       mMaxPipelinedRequests);
         }
     }
 
+    if (PREF_CHANGED(HTTP_PREF("pipelining.max-optimistic-requests"))) {
+        rv = prefs->
+            GetIntPref(HTTP_PREF("pipelining.max-optimistic-requests"), &val);
+        if (NS_SUCCEEDED(rv)) {
+            mMaxOptimisticPipelinedRequests = clamped(val, 1, 0xffff);
+            if (mConnMgr)
+                mConnMgr->UpdateParam
+                    (nsHttpConnectionMgr::MAX_OPTIMISTIC_PIPELINED_REQUESTS,
+                     mMaxOptimisticPipelinedRequests);
+        }
+    }
+
+    if (PREF_CHANGED(HTTP_PREF("pipelining.aggressive"))) {
+        rv = prefs->GetBoolPref(HTTP_PREF("pipelining.aggressive"), &cVar);
+        if (NS_SUCCEEDED(rv))
+            mPipelineAggressive = cVar;
+    }
+
     if (PREF_CHANGED(HTTP_PREF("pipelining.ssl"))) {
         rv = prefs->GetBoolPref(HTTP_PREF("pipelining.ssl"), &cVar);
         if (NS_SUCCEEDED(rv))

netwerk/protocol/http/nsHttpHandler.h

@@ -232,6 +232,8 @@ public:
     static nsresult GenerateHostPort(const nsCString& host, PRInt32 port,
                                      nsCString& hostLine);
 
+    bool GetPipelineAggressive()     { return mPipelineAggressive; }
+
 private:
 
     //
@@ -288,7 +290,9 @@ private:
     PRUint8  mMaxConnectionsPerServer;
     PRUint8  mMaxPersistentConnectionsPerServer;
     PRUint8  mMaxPersistentConnectionsPerProxy;
-    PRUint8  mMaxPipelinedRequests;
+    PRUint16 mMaxPipelinedRequests;
+    PRUint16 mMaxOptimisticPipelinedRequests;
+    bool     mPipelineAggressive;
 
     PRUint8  mRedirectionLimit;
 

netwerk/protocol/http/nsHttpPipeline.cpp

@@ -92,9 +92,8 @@ private:
 // nsHttpPipeline <public>
 //-----------------------------------------------------------------------------
 
-nsHttpPipeline::nsHttpPipeline(PRUint16 maxPipelineDepth)
+nsHttpPipeline::nsHttpPipeline()
-    : mMaxPipelineDepth(maxPipelineDepth)
+    : mConnection(nsnull)
-    , mConnection(nsnull)
     , mStatus(NS_OK)
     , mRequestIsPartial(false)
     , mResponseIsPartial(false)
@@ -131,7 +130,7 @@ nsHttpPipeline::AddTransaction(nsAHttpTransaction *trans)
 
     NS_ADDREF(trans);
     mRequestQ.AppendElement(trans);
-    PRInt32 qlen = mRequestQ.Length();
+    PRUint32 qlen = PipelineDepth();
     
     if (qlen != 1) {
         trans->SetPipelinePosition(qlen);
@@ -142,34 +141,20 @@ nsHttpPipeline::AddTransaction(nsAHttpTransaction *trans)
         trans->SetPipelinePosition(0);
     }
 
-    if (mConnection && !mClosed) {
+    // trans->SetConnection() needs to be updated to point back at
-        trans->SetConnection(this);
+    // the pipeline object.
-        if (qlen == 1)
+    trans->SetConnection(this);
-            mConnection->ResumeSend();
+
-    }
+    if (mConnection && !mClosed && mRequestQ.Length() == 1)
+        mConnection->ResumeSend();
 
     return NS_OK;
 }
 
-PRUint16
+PRUint32
-nsHttpPipeline::PipelineDepthAvailable()
+nsHttpPipeline::PipelineDepth()
 {
-    PRUint16 currentTransactions = mRequestQ.Length() + mResponseQ.Length();
+    return mRequestQ.Length() + mResponseQ.Length();
-
-    // Check to see if there are too many transactions currently in use.
-    if (currentTransactions >= mMaxPipelineDepth)
-        return 0;
-
-    // Check to see if this connection is being used by a non-pipelineable
-    // transaction already.
-    nsAHttpTransaction *trans = Request(0);
-    if (!trans)
-        trans = Response(0);
-    if (trans && !(trans->Caps() & NS_HTTP_ALLOW_PIPELINING))
-        return 0;
-
-    // There is still some room available.
-    return mMaxPipelineDepth - currentTransactions;
 }
 
 nsresult
@@ -223,7 +208,7 @@ nsHttpPipeline::OnHeadersAvailable(nsAHttpTransaction *trans,
 
     NS_ABORT_IF_FALSE(ci, "no connection info");
     
-    bool pipeliningBefore = ci->SupportsPipelining();
+    bool pipeliningBefore = gHttpHandler->ConnMgr()->SupportsPipelining(ci);
     
     // trans has now received its response headers; forward to the real connection
     nsresult rv = mConnection->OnHeadersAvailable(trans,
@@ -231,10 +216,10 @@ nsHttpPipeline::OnHeadersAvailable(nsAHttpTransaction *trans,
                                                   responseHead,
                                                   reset);
     
-    if (!pipeliningBefore && ci->SupportsPipelining())
+    if (!pipeliningBefore && gHttpHandler->ConnMgr()->SupportsPipelining(ci))
         // The received headers have expanded the eligible
         // pipeline depth for this connection
-        gHttpHandler->ConnMgr()->ProcessPipelinePendingQForCI(ci);
+        gHttpHandler->ConnMgr()->ProcessPendingQForEntry(ci);
 
     return rv;
 }
@@ -261,7 +246,7 @@ nsHttpPipeline::CloseTransaction(nsAHttpTransaction *trans, nsresult reason)
     LOG(("nsHttpPipeline::CloseTransaction [this=%x trans=%x reason=%x]\n",
         this, trans, reason));
 
-    NS_ASSERTION(PR_GetCurrentThread() == gSocketThread, "wrong thread");
+    NS_ABORT_IF_FALSE(PR_GetCurrentThread() == gSocketThread, "wrong thread");
     NS_ASSERTION(NS_FAILED(reason), "expecting failure code");
 
     // the specified transaction is to be closed with the given "reason"
@@ -303,7 +288,11 @@ nsHttpPipeline::CloseTransaction(nsAHttpTransaction *trans, nsresult reason)
 void
 nsHttpPipeline::GetConnectionInfo(nsHttpConnectionInfo **result)
 {
-    NS_ASSERTION(mConnection, "no connection");
+    if (!mConnection) {
+        *result = nsnull;
+        return;
+    }
+
     mConnection->GetConnectionInfo(result);
 }
 
@@ -341,7 +330,7 @@ nsHttpPipeline::PushBack(const char *data, PRUint32 length)
 {
     LOG(("nsHttpPipeline::PushBack [this=%x len=%u]\n", this, length));
     
-    NS_ASSERTION(PR_GetCurrentThread() == gSocketThread, "wrong thread");
+    NS_ABORT_IF_FALSE(PR_GetCurrentThread() == gSocketThread, "wrong thread");
     NS_ASSERTION(mPushBackLen == 0, "push back buffer already has data!");
 
     // If we have no chance for a pipeline (e.g. due to an Upgrade)
@@ -471,7 +460,7 @@ nsHttpPipeline::TakeSubTransactions(
 }
 
 //-----------------------------------------------------------------------------
-// nsHttpPipeline::nsAHttpConnection
+// nsHttpPipeline::nsAHttpTransaction
 //-----------------------------------------------------------------------------
 
 void
@@ -483,10 +472,6 @@ nsHttpPipeline::SetConnection(nsAHttpConnection *conn)
     NS_ASSERTION(!mConnection, "already have a connection");
 
     NS_IF_ADDREF(mConnection = conn);
-
-    PRInt32 i, count = mRequestQ.Length();
-    for (i=0; i<count; ++i)
-        Request(i)->SetConnection(this);
 }
 
 nsAHttpConnection *
@@ -759,7 +744,10 @@ nsHttpPipeline::WriteSegments(nsAHttpSegmentWriter *writer,
 
             // ask the connection manager to add additional transactions
             // to our pipeline.
-            gHttpHandler->ConnMgr()->AddTransactionToPipeline(this);
+            nsRefPtr<nsHttpConnectionInfo> ci;
+            GetConnectionInfo(getter_AddRefs(ci));
+            if (ci)
+                gHttpHandler->ConnMgr()->ProcessPendingQForEntry(ci);
         }
         else
             mResponseIsPartial = true;
@@ -819,6 +807,13 @@ nsHttpPipeline::Close(nsresult reason)
     mRequestQ.Clear();
 
     trans = Response(0);
+    
+    nsRefPtr<nsHttpConnectionInfo> ci;
+    GetConnectionInfo(getter_AddRefs(ci));
+    if (ci && (trans || count))
+        gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+            ci, nsHttpConnectionMgr::RedCanceledPipeline, nsnull, 0);
+
     if (trans) {
         // The current transaction can be restarted via reset
         // if the response has not started to arrive and the reason
@@ -879,6 +874,13 @@ nsHttpPipeline::FillSendBuf()
     while ((trans = Request(0)) != nsnull) {
         avail = trans->Available();
         if (avail) {
+            // if there is already a response in the responseq then this
+            // new data comprises a pipeline. Update the transaction in the
+            // response queue to reflect that if necessary. We are now sending
+            // out a request while we haven't received all responses.
+            nsAHttpTransaction *response = Response(0);
+            if (response && !response->PipelinePosition())
+                response->SetPipelinePosition(1);
             rv = trans->ReadSegments(this, avail, &n);
             if (NS_FAILED(rv)) return rv;
             
@@ -909,6 +911,10 @@ nsHttpPipeline::FillSendBuf()
                                          NS_NET_STATUS_WAITING_FOR,
                                          mSendingToProgress);
             }
+
+            // It would be good to re-enable data read handlers via ResumeRecv()
+            // except the read handler code can be synchronously dispatched on
+            // the stack.
         }
         else
             mRequestIsPartial = true;

netwerk/protocol/http/nsHttpPipeline.h

@@ -57,7 +57,7 @@ public:
     NS_DECL_NSAHTTPTRANSACTION
     NS_DECL_NSAHTTPSEGMENTREADER
 
-    nsHttpPipeline(PRUint16 maxPipelineDepth);
+    nsHttpPipeline();
     virtual ~nsHttpPipeline();
 
 private:
@@ -82,7 +82,6 @@ private:
         return mResponseQ[i];
     }
 
-    PRUint16                      mMaxPipelineDepth;
     nsAHttpConnection            *mConnection;
     nsTArray<nsAHttpTransaction*> mRequestQ;  // array of transactions
     nsTArray<nsAHttpTransaction*> mResponseQ; // array of transactions

netwerk/protocol/http/nsHttpTransaction.cpp

@@ -119,6 +119,7 @@ nsHttpTransaction::nsHttpTransaction()
     , mPriority(0)
     , mRestartCount(0)
     , mCaps(0)
+    , mClassification(CLASS_GENERAL)
     , mPipelinePosition(0)
     , mClosed(false)
     , mConnected(false)
@@ -150,6 +151,40 @@ nsHttpTransaction::~nsHttpTransaction()
     delete mChunkedDecoder;
 }
 
+nsHttpTransaction::Classifier
+nsHttpTransaction::Classify()
+{
+    if (!(mCaps & NS_HTTP_ALLOW_PIPELINING))
+        return (mClassification = CLASS_SOLO);
+
+    if (mRequestHead->PeekHeader(nsHttp::If_Modified_Since) ||
+        mRequestHead->PeekHeader(nsHttp::If_None_Match))
+        return (mClassification = CLASS_REVALIDATION);
+
+    const char *accept = mRequestHead->PeekHeader(nsHttp::Accept);
+    if (accept && !PL_strncmp(accept, "image/", 6))
+        return (mClassification = CLASS_IMAGE);
+
+    if (accept && !PL_strncmp(accept, "text/css", 8))
+        return (mClassification = CLASS_SCRIPT);
+
+    mClassification = CLASS_GENERAL;
+
+    PRInt32 queryPos = mRequestHead->RequestURI().FindChar('?');
+    if (queryPos == kNotFound) {
+        if (StringEndsWith(mRequestHead->RequestURI(),
+                           NS_LITERAL_CSTRING(".js")))
+            mClassification = CLASS_SCRIPT;
+    }
+    else if (queryPos >= 3 &&
+             Substring(mRequestHead->RequestURI(), queryPos - 3, 3).
+             EqualsLiteral(".js")) {
+        mClassification = CLASS_SCRIPT;
+    }
+
+    return mClassification;
+}
+
 nsresult
 nsHttpTransaction::Init(PRUint8 caps,
                         nsHttpConnectionInfo *cinfo,
@@ -301,6 +336,8 @@ nsHttpTransaction::Init(PRUint8 caps,
                      nsIOService::gDefaultSegmentCount);
     if (NS_FAILED(rv)) return rv;
 
+    Classify();
+
     NS_ADDREF(*responseBody = mPipeIn);
     return NS_OK;
 }
@@ -661,9 +698,15 @@ nsHttpTransaction::Close(nsresult reason)
     // mReceivedData == FALSE.  (see bug 203057 for more info.)
     //
     if (reason == NS_ERROR_NET_RESET || reason == NS_OK) {
-        if (!mReceivedData && (!mSentData || connReused)) {
+        if (!mReceivedData && (!mSentData || connReused || mPipelinePosition)) {
             // if restarting fails, then we must proceed to close the pipe,
             // which will notify the channel that the transaction failed.
+            
+            if (mPipelinePosition) {
+                gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+                    mConnInfo, nsHttpConnectionMgr::RedCanceledPipeline,
+                    nsnull, 0);
+            }
             if (NS_SUCCEEDED(Restart()))
                 return;
         }
@@ -671,6 +714,21 @@ nsHttpTransaction::Close(nsresult reason)
 
     bool relConn = true;
     if (NS_SUCCEEDED(reason)) {
+        if (!mResponseIsComplete) {
+            // The response has not been delimited with a high-confidence
+            // algorithm like Content-Length or Chunked Encoding. We
+            // need to use a strong framing mechanism to pipeline.
+            gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+                mConnInfo, nsHttpConnectionMgr::BadInsufficientFraming,
+                nsnull, mClassification);
+        }
+        else if (mPipelinePosition) {
+            // report this success as feedback
+            gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+                mConnInfo, nsHttpConnectionMgr::GoodCompletedOK,
+                nsnull, mPipelinePosition);
+        }
+
         // the server has not sent the final \r\n terminating the header
         // section, and there may still be a header line unparsed.  let's make
         // sure we parse the remaining header line, and then hopefully, the
@@ -717,10 +775,10 @@ nsHttpTransaction::AddTransaction(nsAHttpTransaction *trans)
     return NS_ERROR_NOT_IMPLEMENTED;
 }
 
-PRUint16
+PRUint32
-nsHttpTransaction::PipelineDepthAvailable()
+nsHttpTransaction::PipelineDepth()
 {
-    return 0;
+    return IsDone() ? 0 : 1;
 }
 
 nsresult
@@ -871,6 +929,9 @@ nsHttpTransaction::ParseLineSegment(char *segment, PRUint32 len)
             nsresult rv = ParseLine(mLineBuf.BeginWriting());
             mLineBuf.Truncate();
             if (NS_FAILED(rv)) {
+                gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+                    mConnInfo, nsHttpConnectionMgr::RedCorruptedContent,
+                    nsnull, 0);
                 return rv;
             }
         }
@@ -1065,6 +1126,10 @@ nsHttpTransaction::HandleContentStart()
             break;
         }
         mConnection->SetLastTransactionExpectedNoContent(mNoContent);
+        if (mInvalidResponseBytesRead)
+            gHttpHandler->ConnMgr()->PipelineFeedbackInfo(
+                mConnInfo, nsHttpConnectionMgr::BadInsufficientFraming,
+                nsnull, mClassification);
 
         if (mNoContent)
             mContentLength = 0;

netwerk/protocol/http/nsHttpTransaction.h

@@ -134,6 +134,7 @@ public:
     PRInt32    Priority()                 { return mPriority; }
 
     const TimingStruct& Timings() const { return mTimings; }
+    enum Classifier Classification() { return mClassification; }
 
 private:
     nsresult Restart();
@@ -147,6 +148,8 @@ private:
     nsresult ProcessData(char *, PRUint32, PRUint32 *);
     void     DeleteSelfOnConsumerThread();
 
+    Classifier Classify();
+
     static NS_METHOD ReadRequestSegment(nsIInputStream *, void *, const char *,
                                         PRUint32, PRUint32, PRUint32 *);
     static NS_METHOD WritePipeSegment(nsIOutputStream *, void *, char *,
@@ -199,6 +202,7 @@ private:
 
     PRUint16                        mRestartCount;        // the number of times this transaction has been restarted
     PRUint8                         mCaps;
+    enum Classifier                 mClassification;
     PRInt32                         mPipelinePosition;
 
     // state flags, all logically boolean, but not packed together into a