diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index fa023eb6a334..9a602c3da484 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -1519,6 +1519,27 @@ pref("network.http.redirection-limit", 20); pref("network.http.accept-encoding", "gzip, deflate"); pref("network.http.accept-encoding.secure", "gzip, deflate, br"); +pref("network.http.pipelining" , false); +pref("network.http.pipelining.ssl" , false); // disable pipelining over SSL +pref("network.http.pipelining.abtest", false); +pref("network.http.proxy.pipelining", false); + +// Max number of requests in the pipeline +pref("network.http.pipelining.maxrequests" , 32); + +// An optimistic request is one pipelined when policy might allow a new +// connection instead +pref("network.http.pipelining.max-optimistic-requests" , 4); + +pref("network.http.pipelining.aggressive", false); +pref("network.http.pipelining.maxsize" , 300000); +pref("network.http.pipelining.reschedule-on-timeout", true); +pref("network.http.pipelining.reschedule-timeout", 1500); + +// The read-timeout is a ms timer that causes the transaction to be completely +// restarted without pipelining. +pref("network.http.pipelining.read-timeout", 30000); + // Prompt for redirects resulting in unsafe HTTP requests pref("network.http.prompt-temp-redirect", false); diff --git a/netwerk/ipc/NeckoChannelParams.ipdlh b/netwerk/ipc/NeckoChannelParams.ipdlh index 9bddf817f7cc..f29c5302f5f2 100644 --- a/netwerk/ipc/NeckoChannelParams.ipdlh +++ b/netwerk/ipc/NeckoChannelParams.ipdlh @@ -108,6 +108,7 @@ struct HttpChannelOpenArgs int16_t priority; uint32_t classOfService; uint8_t redirectionLimit; + bool allowPipelining; bool allowSTS; uint32_t thirdPartyFlags; bool resumeAt; diff --git a/netwerk/protocol/http/ConnectionDiagnostics.cpp b/netwerk/protocol/http/ConnectionDiagnostics.cpp index 9ecf5d537fe4..c70a32eceaba 100644 --- a/netwerk/protocol/http/ConnectionDiagnostics.cpp +++ b/netwerk/protocol/http/ConnectionDiagnostics.cpp @@ -65,8 +65,14 @@ nsHttpConnectionMgr::OnMsgPrintDiagnostics(int32_t, ARefBase *) ent->mCoalescingKeys.Length()); mLogData.AppendPrintf(" Spdy using = %d, preferred = %d\n", ent->mUsingSpdy, ent->mInPreferredHash); + mLogData.AppendPrintf(" pipelinestate = %d penalty = %d\n", + ent->mPipelineState, ent->mPipeliningPenalty); uint32_t i; + for (i = 0; i < nsAHttpTransaction::CLASS_MAX; ++i) { + mLogData.AppendPrintf(" pipeline per class penalty 0x%x %d\n", + i, ent->mPipeliningClassPenalty[i]); + } for (i = 0; i < ent->mActiveConns.Length(); ++i) { mLogData.AppendPrintf(" :: Active Connection #%u\n", i); ent->mActiveConns[i]->PrintDiagnostics(mLogData); @@ -146,6 +152,9 @@ nsHttpConnection::PrintDiagnostics(nsCString &log) log.AppendPrintf(" idlemonitoring = %d transactionCount=%d\n", mIdleMonitoring, mHttp1xTransactionCount); + log.AppendPrintf(" supports pipeline = %d classification = 0x%x\n", + mSupportsPipelining, mClassification); + if (mSpdySession) mSpdySession->PrintDiagnostics(log); } @@ -198,6 +207,7 @@ nsHttpTransaction::PrintDiagnostics(nsCString &log) log.AppendPrintf(" caps = 0x%x\n", mCaps); log.AppendPrintf(" priority = %d\n", mPriority); log.AppendPrintf(" restart count = %u\n", mRestartCount); + log.AppendPrintf(" classification = 0x%x\n", mClassification); } } // namespace net diff --git a/netwerk/protocol/http/Http2Push.cpp b/netwerk/protocol/http/Http2Push.cpp index 30555b9e30b2..5a3d01a80a64 100644 --- a/netwerk/protocol/http/Http2Push.cpp +++ b/netwerk/protocol/http/Http2Push.cpp @@ -467,6 +467,30 @@ Http2PushTransactionBuffer::Close(nsresult reason) mIsDone = true; } +nsresult +Http2PushTransactionBuffer::AddTransaction(nsAHttpTransaction *trans) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +uint32_t +Http2PushTransactionBuffer::PipelineDepth() +{ + return 0; +} + +nsresult +Http2PushTransactionBuffer::SetPipelinePosition(int32_t position) +{ + return NS_OK; +} + +int32_t +Http2PushTransactionBuffer::PipelinePosition() +{ + return 1; +} + nsresult Http2PushTransactionBuffer::GetBufferedData(char *buf, uint32_t count, diff --git a/netwerk/protocol/http/Http2Session.cpp b/netwerk/protocol/http/Http2Session.cpp index 4213c9319489..a3aaa760e2b0 100644 --- a/netwerk/protocol/http/Http2Session.cpp +++ b/netwerk/protocol/http/Http2Session.cpp @@ -41,7 +41,9 @@ namespace mozilla { namespace net { -// Http2Session has multiple inheritance of things that implement nsISupports +// Http2Session has multiple inheritance of things that implement +// nsISupports, so this magic is taken from nsHttpPipeline that +// implements some of the same abstract classes. NS_IMPL_ADDREF(Http2Session) NS_IMPL_RELEASE(Http2Session) NS_INTERFACE_MAP_BEGIN(Http2Session) @@ -3809,6 +3811,21 @@ Http2Session::TakeHttpConnection() return nullptr; } +uint32_t +Http2Session::CancelPipeline(nsresult reason) +{ + // we don't pipeline inside http/2, so this isn't an issue + return 0; +} + +nsAHttpTransaction::Classifier +Http2Session::Classification() +{ + if (!mConnection) + return nsAHttpTransaction::CLASS_GENERAL; + return mConnection->Classification(); +} + void Http2Session::GetSecurityCallbacks(nsIInterfaceRequestor **aOut) { @@ -3908,6 +3925,42 @@ Http2Session::TakeSubTransactions( return NS_OK; } +nsresult +Http2Session::AddTransaction(nsAHttpTransaction *) +{ + // This API is meant for pipelining, Http2Session's should be + // extended with AddStream() + + MOZ_ASSERT(false, + "Http2Session::AddTransaction() should not be called"); + + return NS_ERROR_NOT_IMPLEMENTED; +} + +uint32_t +Http2Session::PipelineDepth() +{ + return IsDone() ? 0 : 1; +} + +nsresult +Http2Session::SetPipelinePosition(int32_t position) +{ + // This API is meant for pipelining, Http2Session's should be + // extended with AddStream() + + MOZ_ASSERT(false, + "Http2Session::SetPipelinePosition() should not be called"); + + return NS_ERROR_NOT_IMPLEMENTED; +} + +int32_t +Http2Session::PipelinePosition() +{ + return 0; +} + //----------------------------------------------------------------------------- // Pass through methods of nsAHttpConnection //----------------------------------------------------------------------------- diff --git a/netwerk/protocol/http/HttpBaseChannel.cpp b/netwerk/protocol/http/HttpBaseChannel.cpp index 1d0c7b01c027..976cacfe69d6 100644 --- a/netwerk/protocol/http/HttpBaseChannel.cpp +++ b/netwerk/protocol/http/HttpBaseChannel.cpp @@ -162,6 +162,7 @@ HttpBaseChannel::HttpBaseChannel() , mWasOpened(false) , mRequestObserversCalled(false) , mResponseHeadersModified(false) + , mAllowPipelining(true) , mAllowSTS(true) , mThirdPartyFlags(0) , mUploadStreamHasHeaders(false) @@ -1953,7 +1954,7 @@ NS_IMETHODIMP HttpBaseChannel::GetAllowPipelining(bool *value) { NS_ENSURE_ARG_POINTER(value); - *value = false; + *value = mAllowPipelining; return NS_OK; } @@ -1961,7 +1962,8 @@ NS_IMETHODIMP HttpBaseChannel::SetAllowPipelining(bool value) { ENSURE_CALLED_BEFORE_CONNECT(); - // nop + + mAllowPipelining = value; return NS_OK; } @@ -3230,7 +3232,8 @@ HttpBaseChannel::SetupReplacementChannel(nsIURI *newURI, // convey the referrer if one was used for this channel to the next one if (mReferrer) httpChannel->SetReferrerWithPolicy(mReferrer, mReferrerPolicy); - // convey the mAllowSTS flags + // convey the mAllowPipelining and mAllowSTS flags + httpChannel->SetAllowPipelining(mAllowPipelining); httpChannel->SetAllowSTS(mAllowSTS); // convey the new redirection limit // make sure we don't underflow diff --git a/netwerk/protocol/http/HttpBaseChannel.h b/netwerk/protocol/http/HttpBaseChannel.h index 0d447fa7202e..6dfd07769271 100644 --- a/netwerk/protocol/http/HttpBaseChannel.h +++ b/netwerk/protocol/http/HttpBaseChannel.h @@ -179,8 +179,8 @@ public: NS_IMETHOD GetOriginalResponseHeader(const nsACString &aHeader, nsIHttpHeaderVisitor *aVisitor) override; NS_IMETHOD VisitOriginalResponseHeaders(nsIHttpHeaderVisitor *aVisitor) override; - NS_IMETHOD GetAllowPipelining(bool *value) override; // deprecated - NS_IMETHOD SetAllowPipelining(bool value) override; // deprecated + NS_IMETHOD GetAllowPipelining(bool *value) override; + NS_IMETHOD SetAllowPipelining(bool value) override; NS_IMETHOD GetAllowSTS(bool *value) override; NS_IMETHOD SetAllowSTS(bool value) override; NS_IMETHOD GetRedirectionLimit(uint32_t *value) override; @@ -487,6 +487,7 @@ protected: // if 1 all "http-on-{opening|modify|etc}-request" observers have been called uint32_t mRequestObserversCalled : 1; uint32_t mResponseHeadersModified : 1; + uint32_t mAllowPipelining : 1; uint32_t mAllowSTS : 1; uint32_t mThirdPartyFlags : 3; uint32_t mUploadStreamHasHeaders : 1; diff --git a/netwerk/protocol/http/HttpChannelChild.cpp b/netwerk/protocol/http/HttpChannelChild.cpp index 064ad40f0177..2be2079c4ec7 100644 --- a/netwerk/protocol/http/HttpChannelChild.cpp +++ b/netwerk/protocol/http/HttpChannelChild.cpp @@ -2190,6 +2190,7 @@ HttpChannelChild::ContinueAsyncOpen() openArgs.priority() = mPriority; openArgs.classOfService() = mClassOfService; openArgs.redirectionLimit() = mRedirectionLimit; + openArgs.allowPipelining() = mAllowPipelining; openArgs.allowSTS() = mAllowSTS; openArgs.thirdPartyFlags() = mThirdPartyFlags; openArgs.resumeAt() = mSendResumeAt; diff --git a/netwerk/protocol/http/HttpChannelParent.cpp b/netwerk/protocol/http/HttpChannelParent.cpp index b925da6fe795..5ce15e713703 100644 --- a/netwerk/protocol/http/HttpChannelParent.cpp +++ b/netwerk/protocol/http/HttpChannelParent.cpp @@ -120,7 +120,7 @@ HttpChannelParent::Init(const HttpChannelCreationArgs& aArgs) a.loadFlags(), a.requestHeaders(), a.requestMethod(), a.uploadStream(), a.uploadStreamHasHeaders(), a.priority(), a.classOfService(), - a.redirectionLimit(), a.allowSTS(), + a.redirectionLimit(), a.allowPipelining(), a.allowSTS(), a.thirdPartyFlags(), a.resumeAt(), a.startPos(), a.entityID(), a.chooseApplicationCache(), a.appCacheClientID(), a.allowSpdy(), a.allowAltSvc(), a.beConservative(), @@ -305,6 +305,7 @@ HttpChannelParent::DoAsyncOpen( const URIParams& aURI, const int16_t& priority, const uint32_t& classOfService, const uint8_t& redirectionLimit, + const bool& allowPipelining, const bool& allowSTS, const uint32_t& thirdPartyFlags, const bool& doResumeAt, @@ -525,6 +526,7 @@ HttpChannelParent::DoAsyncOpen( const URIParams& aURI, mChannel->SetClassFlags(classOfService); } mChannel->SetRedirectionLimit(redirectionLimit); + mChannel->SetAllowPipelining(allowPipelining); mChannel->SetAllowSTS(allowSTS); mChannel->SetThirdPartyFlags(thirdPartyFlags); mChannel->SetAllowSpdy(allowSpdy); diff --git a/netwerk/protocol/http/HttpChannelParent.h b/netwerk/protocol/http/HttpChannelParent.h index 7229f18054da..53c2086a60aa 100644 --- a/netwerk/protocol/http/HttpChannelParent.h +++ b/netwerk/protocol/http/HttpChannelParent.h @@ -122,6 +122,7 @@ protected: const int16_t& priority, const uint32_t& classOfService, const uint8_t& redirectionLimit, + const bool& allowPipelining, const bool& allowSTS, const uint32_t& thirdPartyFlags, const bool& doResumeAt, diff --git a/netwerk/protocol/http/NullHttpTransaction.cpp b/netwerk/protocol/http/NullHttpTransaction.cpp index 23e6b1303323..965ffcc2c51f 100644 --- a/netwerk/protocol/http/NullHttpTransaction.cpp +++ b/netwerk/protocol/http/NullHttpTransaction.cpp @@ -304,6 +304,30 @@ NullHttpTransaction::ConnectionInfo() return mConnectionInfo; } +nsresult +NullHttpTransaction::AddTransaction(nsAHttpTransaction *trans) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +uint32_t +NullHttpTransaction::PipelineDepth() +{ + return 0; +} + +nsresult +NullHttpTransaction::SetPipelinePosition(int32_t position) +{ + return NS_OK; +} + +int32_t +NullHttpTransaction::PipelinePosition() +{ + return 1; +} + } // namespace net } // namespace mozilla diff --git a/netwerk/protocol/http/TunnelUtils.cpp b/netwerk/protocol/http/TunnelUtils.cpp index 1f511246946a..e98897c49b4d 100644 --- a/netwerk/protocol/http/TunnelUtils.cpp +++ b/netwerk/protocol/http/TunnelUtils.cpp @@ -689,6 +689,59 @@ TLSFilterTransaction::SetProxiedTransaction(nsAHttpTransaction *aTrans) return NS_OK; } +// AddTransaction is for adding pipelined subtransactions +nsresult +TLSFilterTransaction::AddTransaction(nsAHttpTransaction *aTrans) +{ + LOG(("TLSFilterTransaction::AddTransaction passing on subtransaction " + "[this=%p] aTrans=%p ,mTransaction=%p\n", this, aTrans, mTransaction.get())); + + if (!mTransaction) { + return NS_ERROR_FAILURE; + } + + return mTransaction->AddTransaction(aTrans); +} + +uint32_t +TLSFilterTransaction::PipelineDepth() +{ + if (!mTransaction) { + return 0; + } + + return mTransaction->PipelineDepth(); +} + +nsresult +TLSFilterTransaction::SetPipelinePosition(int32_t aPosition) +{ + if (!mTransaction) { + return NS_OK; + } + + return mTransaction->SetPipelinePosition(aPosition); +} + +int32_t +TLSFilterTransaction::PipelinePosition() +{ + if (!mTransaction) { + return 1; + } + + return mTransaction->PipelinePosition(); +} + +nsHttpPipeline * +TLSFilterTransaction::QueryPipeline() +{ + if (!mTransaction) { + return nullptr; + } + return mTransaction->QueryPipeline(); +} + bool TLSFilterTransaction::IsNullTransaction() { diff --git a/netwerk/protocol/http/TunnelUtils.h b/netwerk/protocol/http/TunnelUtils.h index 725530b5b689..20cfaf7ee9c7 100644 --- a/netwerk/protocol/http/TunnelUtils.h +++ b/netwerk/protocol/http/TunnelUtils.h @@ -128,6 +128,7 @@ public: nsIAsyncOutputStream **outSocketOut); // nsAHttpTransaction overloads + nsHttpPipeline *QueryPipeline() override; bool IsNullTransaction() override; NullHttpTransaction *QueryNullTransaction() override; nsHttpTransaction *QueryHttpTransaction() override; diff --git a/netwerk/protocol/http/moz.build b/netwerk/protocol/http/moz.build index 93623575c055..72a15c2d1a23 100644 --- a/netwerk/protocol/http/moz.build +++ b/netwerk/protocol/http/moz.build @@ -88,6 +88,7 @@ UNIFIED_SOURCES += [ 'nsHttpDigestAuth.cpp', 'nsHttpHeaderArray.cpp', 'nsHttpNTLMAuth.cpp', + 'nsHttpPipeline.cpp', 'nsHttpRequestHead.cpp', 'nsHttpResponseHead.cpp', 'nsHttpTransaction.cpp', diff --git a/netwerk/protocol/http/nsAHttpConnection.h b/netwerk/protocol/http/nsAHttpConnection.h index 974103205ae9..1775a2c680d6 100644 --- a/netwerk/protocol/http/nsAHttpConnection.h +++ b/netwerk/protocol/http/nsAHttpConnection.h @@ -62,8 +62,8 @@ public: // After a connection has had ResumeSend() called by a transaction, // and it is ready to write to the network it may need to know the // transaction that has data to write. This is only an issue for - // multiplexed protocols like SPDY - h1 - // implicitly has this information in a 1:1 relationship with the + // multiplexed protocols like SPDY - plain HTTP or pipelined HTTP + // implicitly have this information in a 1:1 relationship with the // transaction(s) they manage. virtual void TransactionHasDataToWrite(nsAHttpTransaction *) { @@ -112,7 +112,7 @@ public: virtual void DontReuse() = 0; // called by a transaction when the transaction reads more from the socket - // than it should have (eg. containing part of the next response). + // than it should have (eg. containing part of the next pipelined response). virtual nsresult PushBack(const char *data, uint32_t length) = 0; // Used to determine if the connection wants read events even though @@ -133,6 +133,14 @@ public: // references or ownership. virtual nsISocketTransport *Transport() = 0; + // Cancel and reschedule transactions deeper than the current response. + // Returns the number of canceled transactions. + virtual uint32_t CancelPipeline(nsresult originalReason) = 0; + + // Read and write class of transaction that is carried on this connection + virtual nsAHttpTransaction::Classifier Classification() = 0; + virtual void Classify(nsAHttpTransaction::Classifier newclass) = 0; + // The number of transaction bytes written out on this HTTP Connection, does // not count CONNECT tunnel setup virtual int64_t BytesWritten() = 0; @@ -158,6 +166,8 @@ NS_DEFINE_STATIC_IID_ACCESSOR(nsAHttpConnection, NS_AHTTPCONNECTION_IID) void DontReuse() override; \ nsresult PushBack(const char *, uint32_t) override; \ already_AddRefed TakeHttpConnection() override; \ + uint32_t CancelPipeline(nsresult originalReason) override; \ + nsAHttpTransaction::Classifier Classification() override; \ /* \ Thes methods below have automatic definitions that just forward the \ function to a lower level connection object \ @@ -232,6 +242,12 @@ NS_DEFINE_STATIC_IID_ACCESSOR(nsAHttpConnection, NS_AHTTPCONNECTION_IID) if (fwdObject) \ (fwdObject)->SetLastTransactionExpectedNoContent(val); \ } \ + void Classify(nsAHttpTransaction::Classifier newclass) \ + override \ + { \ + if (fwdObject) \ + (fwdObject)->Classify(newclass); \ + } \ int64_t BytesWritten() override \ { return fwdObject ? (fwdObject)->BytesWritten() : 0; } \ void SetSecurityCallbacks(nsIInterfaceRequestor* aCallbacks) \ diff --git a/netwerk/protocol/http/nsAHttpTransaction.h b/netwerk/protocol/http/nsAHttpTransaction.h index 8d15872d2d3a..df998699af12 100644 --- a/netwerk/protocol/http/nsAHttpTransaction.h +++ b/netwerk/protocol/http/nsAHttpTransaction.h @@ -19,6 +19,7 @@ class nsAHttpConnection; class nsAHttpSegmentReader; class nsAHttpSegmentWriter; class nsHttpTransaction; +class nsHttpPipeline; class nsHttpRequestHead; class nsHttpConnectionInfo; class NullHttpTransaction; @@ -99,12 +100,12 @@ public: virtual nsHttpRequestHead *RequestHead() = 0; // determine the number of real http/1.x transactions on this - // abstract object. Pipelines had multiple, SPDY has 0, + // abstract object. Pipelines may have multiple, SPDY has 0, // normal http transactions have 1. virtual uint32_t Http1xTransactionCount() = 0; // called to remove the unused sub transactions from an object that can - // handle multiple transactions simultaneously (i.e. h2). + // handle multiple transactions simultaneously (i.e. pipelines or spdy). // // Returns NS_ERROR_NOT_IMPLEMENTED if the object does not implement // sub-transactions. @@ -115,12 +116,32 @@ public: virtual nsresult TakeSubTransactions( nsTArray > &outTransactions) = 0; + // called to add a sub-transaction in the case of pipelined transactions + // classes that do not implement sub transactions + // return NS_ERROR_NOT_IMPLEMENTED + virtual nsresult AddTransaction(nsAHttpTransaction *transaction) = 0; + + // The total length of the outstanding pipeline comprised of transacations + // and sub-transactions. + virtual uint32_t PipelineDepth() = 0; + + // Used to inform the connection that it is being used in a pipelined + // context. That may influence the handling of some errors. + // The value is the pipeline position (> 1). + virtual nsresult SetPipelinePosition(int32_t) = 0; + virtual int32_t PipelinePosition() = 0; + // Occasionally the abstract interface has to give way to base implementations - // to respect differences between spdy, h2, etc.. + // to respect differences between spdy, pipelines, etc.. // These Query* (and IsNullTransaction()) functions provide a way to do // that without using xpcom or rtti. Any calling code that can't deal with // a null response from one of them probably shouldn't be using nsAHttpTransaction + // If we used rtti this would be the result of doing + // dynamic_cast(this).. i.e. it can be nullptr for + // non pipeline implementations of nsAHttpTransaction + virtual nsHttpPipeline *QueryPipeline() { return nullptr; } + // equivalent to !!dynamic_cast(this) // A null transaction is expected to return BASE_STREAM_CLOSED on all of // its IO functions all the time. @@ -147,6 +168,29 @@ public: virtual bool ResponseTimeoutEnabled() const; virtual PRIntervalTime ResponseTimeout(); + // Every transaction is classified into one of the types below. When using + // HTTP pipelines, only transactions with the same type appear on the same + // pipeline. + enum Classifier { + // Transactions that expect a short 304 (no-content) response + CLASS_REVALIDATION, + + // Transactions for content expected to be CSS or JS + CLASS_SCRIPT, + + // Transactions for content expected to be an image + CLASS_IMAGE, + + // Transactions that cannot involve a pipeline + CLASS_SOLO, + + // Transactions that do not fit any of the other categories. HTML + // is normally GENERAL. + CLASS_GENERAL, + + CLASS_MAX + }; + // conceptually the security info is part of the connection, but sometimes // in the case of TLS tunneled within TLS the transaction might present // a more specific security info that cannot be represented as a layer in @@ -201,7 +245,11 @@ NS_DEFINE_STATIC_IID_ACCESSOR(nsAHttpTransaction, NS_AHTTPTRANSACTION_IID) void SetProxyConnectFailed() override; \ virtual nsHttpRequestHead *RequestHead() override; \ uint32_t Http1xTransactionCount() override; \ - nsresult TakeSubTransactions(nsTArray > &outTransactions) override; + nsresult TakeSubTransactions(nsTArray > &outTransactions) override; \ + nsresult AddTransaction(nsAHttpTransaction *) override; \ + uint32_t PipelineDepth() override; \ + nsresult SetPipelinePosition(int32_t) override; \ + int32_t PipelinePosition() override; //----------------------------------------------------------------------------- // nsAHttpSegmentReader diff --git a/netwerk/protocol/http/nsHttp.h b/netwerk/protocol/http/nsHttp.h index 252609e2e6f6..1bbfb0b2e0e0 100644 --- a/netwerk/protocol/http/nsHttp.h +++ b/netwerk/protocol/http/nsHttp.h @@ -51,7 +51,7 @@ typedef uint8_t nsHttpVersion; //----------------------------------------------------------------------------- #define NS_HTTP_ALLOW_KEEPALIVE (1<<0) -// NS_HTTP_ALLOW_PIPELINING (1<<1) removed +#define NS_HTTP_ALLOW_PIPELINING (1<<1) // a transaction with this caps flag will continue to own the connection, // preventing it from being reclaimed, even after the transaction completes. diff --git a/netwerk/protocol/http/nsHttpChannel.cpp b/netwerk/protocol/http/nsHttpChannel.cpp index 021186609410..bd9d11e157d8 100644 --- a/netwerk/protocol/http/nsHttpChannel.cpp +++ b/netwerk/protocol/http/nsHttpChannel.cpp @@ -798,6 +798,24 @@ nsHttpChannel::SetupTransactionRequestContext() mTransaction->SetRequestContext(rc); } +static bool +SafeForPipelining(nsHttpRequestHead::ParsedMethodType method, + const nsCString &methodString) +{ + if (method == nsHttpRequestHead::kMethod_Get || + method == nsHttpRequestHead::kMethod_Head || + method == nsHttpRequestHead::kMethod_Options) { + return true; + } + + if (method != nsHttpRequestHead::kMethod_Custom) { + return false; + } + + return (!strcmp(methodString.get(), "PROPFIND") || + !strcmp(methodString.get(), "PROPPATCH")); +} + nsresult nsHttpChannel::SetupTransaction() { @@ -808,6 +826,24 @@ nsHttpChannel::SetupTransaction() nsresult rv; mUsedNetwork = 1; + if (mCaps & NS_HTTP_ALLOW_PIPELINING) { + // + // disable pipelining if: + // (1) pipelining has been disabled by config + // (2) pipelining has been disabled by connection mgr info + // (3) request corresponds to a top-level document load (link click) + // (4) request method is non-idempotent + // (5) request is marked slow (e.g XHR) + // + nsAutoCString method; + mRequestHead.Method(method); + if (!mAllowPipelining || + (mLoadFlags & (LOAD_INITIAL_DOCUMENT_URI | INHIBIT_PIPELINE)) || + !SafeForPipelining(mRequestHead.ParsedMethod(), method)) { + LOG((" pipelining disallowed\n")); + mCaps &= ~NS_HTTP_ALLOW_PIPELINING; + } + } if (!mAllowSpdy) { mCaps |= NS_HTTP_DISALLOW_SPDY; @@ -955,6 +991,7 @@ nsHttpChannel::SetupTransaction() nsHttp::Upgrade.get(), true); mCaps |= NS_HTTP_STICKY_CONNECTION; + mCaps &= ~NS_HTTP_ALLOW_PIPELINING; mCaps &= ~NS_HTTP_ALLOW_KEEPALIVE; } @@ -1367,6 +1404,10 @@ nsHttpChannel::CallOnStartRequest() } } + rv = EnsureAssocReq(); + if (NS_FAILED(rv)) + return rv; + // if this channel is for a download, close off access to the cache. if (mCacheEntry && mChannelIsForDownload) { mCacheEntry->AsyncDoom(nullptr); @@ -2877,6 +2918,120 @@ nsHttpChannel::HandleAsyncAbort() } +nsresult +nsHttpChannel::EnsureAssocReq() +{ + // Confirm Assoc-Req response header on pipelined transactions + // per draft-nottingham-http-pipeline-01.txt + // of the form: GET http://blah.com/foo/bar?qv + // return NS_OK as long as we don't find a violation + // (i.e. no header is ok, as are malformed headers, as are + // transactions that have not been pipelined (unless those have been + // opted in via pragma)) + + if (!mResponseHead) + return NS_OK; + + nsAutoCString assoc_val; + if (NS_FAILED(mResponseHead->GetHeader(nsHttp::Assoc_Req, assoc_val))) { + return NS_OK; + } + + if (!mTransaction || !mURI) + return NS_OK; + + if (!mTransaction->PipelinePosition()) { + // "Pragma: X-Verify-Assoc-Req" can be used to verify even non pipelined + // transactions. It is used by test harness. + + nsAutoCString pragma_val; + mResponseHead->GetHeader(nsHttp::Pragma, pragma_val); + if (pragma_val.IsEmpty() || + !nsHttp::FindToken(pragma_val.get(), "X-Verify-Assoc-Req", + HTTP_HEADER_VALUE_SEPS)) + return NS_OK; + } + + char *method = net_FindCharNotInSet(assoc_val.get(), HTTP_LWS); + if (!method) + return NS_OK; + + bool equals; + char *endofmethod; + + char * assoc_valChar = nullptr; + endofmethod = net_FindCharInSet(method, HTTP_LWS); + if (endofmethod) + assoc_valChar = net_FindCharNotInSet(endofmethod, HTTP_LWS); + if (!assoc_valChar) + return NS_OK; + + // check the method + nsAutoCString methodHead; + mRequestHead.Method(methodHead); + if ((((int32_t)methodHead.Length()) != (endofmethod - method)) || + PL_strncmp(method, + methodHead.get(), + endofmethod - method)) { + LOG((" Assoc-Req failure Method %s", method)); + if (mConnectionInfo) + gHttpHandler->ConnMgr()-> + PipelineFeedbackInfo(mConnectionInfo, + nsHttpConnectionMgr::RedCorruptedContent, + nullptr, 0); + + nsCOMPtr consoleService = + do_GetService(NS_CONSOLESERVICE_CONTRACTID); + if (consoleService) { + nsAutoString message + (NS_LITERAL_STRING("Failed Assoc-Req. Received ")); + nsAutoCString assoc_req; + mResponseHead->GetHeader(nsHttp::Assoc_Req, assoc_req); + AppendASCIItoUTF16(assoc_req, message); + message += NS_LITERAL_STRING(" expected method "); + AppendASCIItoUTF16(methodHead, message); + consoleService->LogStringMessage(message.get()); + } + + if (gHttpHandler->EnforceAssocReq()) + return NS_ERROR_CORRUPTED_CONTENT; + return NS_OK; + } + + // check the URL + nsCOMPtr assoc_url; + if (NS_FAILED(NS_NewURI(getter_AddRefs(assoc_url), assoc_valChar)) || + !assoc_url) + return NS_OK; + + mURI->Equals(assoc_url, &equals); + if (!equals) { + LOG((" Assoc-Req failure URL %s", assoc_valChar)); + if (mConnectionInfo) + gHttpHandler->ConnMgr()-> + PipelineFeedbackInfo(mConnectionInfo, + nsHttpConnectionMgr::RedCorruptedContent, + nullptr, 0); + + nsCOMPtr consoleService = + do_GetService(NS_CONSOLESERVICE_CONTRACTID); + if (consoleService) { + nsAutoString message + (NS_LITERAL_STRING("Failed Assoc-Req. Received ")); + nsAutoCString assoc_req; + mResponseHead->GetHeader(nsHttp::Assoc_Req, assoc_req); + AppendASCIItoUTF16(assoc_req, message); + message += NS_LITERAL_STRING(" expected URL "); + AppendASCIItoUTF16(mSpec.get(), message); + consoleService->LogStringMessage(message.get()); + } + + if (gHttpHandler->EnforceAssocReq()) + return NS_ERROR_CORRUPTED_CONTENT; + } + return NS_OK; +} + //----------------------------------------------------------------------------- // nsHttpChannel //----------------------------------------------------------------------------- @@ -3156,7 +3311,7 @@ nsHttpChannel::ProcessNotModified() // case of bug 716840, a sign of the server having previously corrupted // our cache with a bad response. Take the minor step here of just dooming // that cache entry so there is a fighting chance of getting things on the - // right track. + // right track as well as disabling pipelining for that host. nsAutoCString lastModifiedCached; nsAutoCString lastModified304; @@ -3174,6 +3329,11 @@ nsHttpChannel::ProcessNotModified() lastModifiedCached.get(), lastModified304.get())); mCacheEntry->AsyncDoom(nullptr); + if (mConnectionInfo) + gHttpHandler->ConnMgr()-> + PipelineFeedbackInfo(mConnectionInfo, + nsHttpConnectionMgr::RedCorruptedContent, + nullptr, 0); Telemetry::Accumulate(Telemetry::CACHE_LM_INCONSISTENT, true); } @@ -5982,6 +6142,13 @@ nsHttpChannel::BeginConnect() if (!mTimingEnabled) mAsyncOpenTime = TimeStamp(); + // when proxying only use the pipeline bit if ProxyPipelining() allows it. + if (!mConnectionInfo->UsingConnect() && mConnectionInfo->UsingHttpProxy()) { + mCaps &= ~NS_HTTP_ALLOW_PIPELINING; + if (gHttpHandler->ProxyPipelining()) + mCaps |= NS_HTTP_ALLOW_PIPELINING; + } + // if this somehow fails we can go on without it gHttpHandler->AddConnectionHeader(&mRequestHead, mCaps); @@ -6013,7 +6180,7 @@ nsHttpChannel::BeginConnect() // - If "Connection: close" is set as a request header, then do not bother // trying to establish a keep-alive connection. if (mRequestHead.HasHeaderValue(nsHttp::Connection, "close")) - mCaps &= ~(NS_HTTP_ALLOW_KEEPALIVE); + mCaps &= ~(NS_HTTP_ALLOW_KEEPALIVE | NS_HTTP_ALLOW_PIPELINING); if (gHttpHandler->CriticalRequestPrioritization()) { if (mClassOfService & nsIClassOfService::Leader) { @@ -6031,6 +6198,7 @@ nsHttpChannel::BeginConnect() gHttpHandler->ConnMgr()->ClearAltServiceMappings(); gHttpHandler->ConnMgr()->DoShiftReloadConnectionCleanup(mConnectionInfo); } + mCaps &= ~NS_HTTP_ALLOW_PIPELINING; } // We may have been cancelled already, either by on-modify-request @@ -6702,6 +6870,14 @@ nsHttpChannel::OnStopRequest(nsIRequest *request, nsISupports *ctxt, nsresult st LOG((" connection is not persistent, not reusing it")); conn = nullptr; } + // We do not use a sticky connection in case of a nsHttpPipeline as + // well (see bug 1337826). This is a quick fix, because + // nsHttpPipeline is turned off by default. + RefPtr tranConn = do_QueryObject(conn); + if (tranConn && tranConn->QueryPipeline()) { + LOG(("Do not use this connection, it is a nsHttpPipeline.")); + conn = nullptr; + } } RefPtr stickyConn; @@ -7499,8 +7675,9 @@ nsHttpChannel::DoAuthRetry(nsAHttpConnection *conn) seekable->Seek(nsISeekableStream::NS_SEEK_SET, 0); } - // set sticky connection flag + // set sticky connection flag and disable pipelining. mCaps |= NS_HTTP_STICKY_CONNECTION; + mCaps &= ~NS_HTTP_ALLOW_PIPELINING; // and create a new one... rv = SetupTransaction(); diff --git a/netwerk/protocol/http/nsHttpConnection.cpp b/netwerk/protocol/http/nsHttpConnection.cpp index 80d4a8587c19..af1f6f8b1c18 100644 --- a/netwerk/protocol/http/nsHttpConnection.cpp +++ b/netwerk/protocol/http/nsHttpConnection.cpp @@ -22,6 +22,7 @@ #include "mozilla/Telemetry.h" #include "nsHttpConnection.h" #include "nsHttpHandler.h" +#include "nsHttpPipeline.h" #include "nsHttpRequestHead.h" #include "nsHttpResponseHead.h" #include "nsIOService.h" @@ -58,6 +59,7 @@ nsHttpConnection::nsHttpConnection() , mKeepAlive(true) // assume to keep-alive by default , mKeepAliveMask(true) , mDontReuse(false) + , mSupportsPipelining(false) // assume low-grade server , mIsReused(false) , mCompletedProxyConnect(false) , mLastTransactionExpectedNoContent(false) @@ -69,6 +71,7 @@ nsHttpConnection::nsHttpConnection() , mTrafficStamp(false) , mHttp1xTransactionCount(0) , mRemainingConnectionUses(0xffffffff) + , mClassification(nsAHttpTransaction::CLASS_GENERAL) , mNPNComplete(false) , mSetupSSLCalled(false) , mUsingSpdyVersion(0) @@ -138,6 +141,8 @@ nsHttpConnection::Init(nsHttpConnectionInfo *info, mConnectedTransport = connectedTransport; mConnInfo = info; mLastWriteTime = mLastReadTime = PR_IntervalNow(); + mSupportsPipelining = + gHttpHandler->ConnMgr()->SupportsPipelining(mConnInfo); mRtt = rtt; mMaxHangTime = PR_SecondsToInterval(maxHangTime); @@ -273,7 +278,7 @@ nsHttpConnection::StartSpdy(uint8_t spdyVersion) // a server goaway was generated). mIsReused = true; - // If mTransaction is a muxed object it might represent + // If mTransaction is a pipeline object it might represent // several requests. If so, we need to unpack that and // pack them all into a new spdy session. @@ -320,6 +325,7 @@ nsHttpConnection::StartSpdy(uint8_t spdyVersion) "rv[0x%" PRIx32 "]", this, static_cast(rv))); } + mSupportsPipelining = false; // don't use http/1 pipelines with spdy mIdleTimeout = gHttpHandler->SpdyTimeout(); if (!mTLSFilter) { @@ -864,24 +870,38 @@ nsHttpConnection::DontReuse() mSpdySession->DontReuse(); } +// Checked by the Connection Manager before scheduling a pipelined transaction +bool +nsHttpConnection::SupportsPipelining() +{ + if (mTransaction && + mTransaction->PipelineDepth() >= mRemainingConnectionUses) { + LOG(("nsHttpConnection::SupportsPipelining this=%p deny pipeline " + "because current depth %d exceeds max remaining uses %d\n", + this, mTransaction->PipelineDepth(), mRemainingConnectionUses)); + return false; + } + return mSupportsPipelining && IsKeepAlive() && !mDontReuse; +} + bool nsHttpConnection::CanReuse() { - if (mDontReuse || !mRemainingConnectionUses) { + if (mDontReuse) return false; - } - if ((mTransaction ? (mTransaction->IsDone() ? 0 : 1) : 0) >= + if ((mTransaction ? mTransaction->PipelineDepth() : 0) >= mRemainingConnectionUses) { return false; } bool canReuse; - if (mSpdySession) { + + if (mSpdySession) canReuse = mSpdySession->CanReuse(); - } else { + else canReuse = IsKeepAlive(); - } + canReuse = canReuse && (IdleTime() < mIdleTimeout) && IsAlive(); // An idle persistent connection should not have data waiting to be read @@ -960,6 +980,64 @@ nsHttpConnection::IsAlive() return alive; } +bool +nsHttpConnection::SupportsPipelining(nsHttpResponseHead *responseHead) +{ + // SPDY supports infinite parallelism, so no need to pipeline. + if (mUsingSpdyVersion) + return false; + + // assuming connection is HTTP/1.1 with keep-alive enabled + if (mConnInfo->UsingHttpProxy() && !mConnInfo->UsingConnect()) { + // XXX check for bad proxy servers... + return true; + } + + // check for bad origin servers + nsAutoCString val; + responseHead->GetHeader(nsHttp::Server, val); + + // If there is no server header we will assume it should not be banned + // as facebook and some other prominent sites do this + if (val.IsEmpty()) + return true; + + // The blacklist is indexed by the first character. All of these servers are + // known to return their identifier as the first thing in the server string, + // so we can do a leading match. + + static const char *bad_servers[26][6] = { + { nullptr }, { nullptr }, { nullptr }, { nullptr }, // a - d + { "EFAServer/", nullptr }, // e + { nullptr }, { nullptr }, { nullptr }, { nullptr }, // f - i + { nullptr }, { nullptr }, { nullptr }, // j - l + { "Microsoft-IIS/4.", "Microsoft-IIS/5.", nullptr }, // m + { "Netscape-Enterprise/3.", "Netscape-Enterprise/4.", + "Netscape-Enterprise/5.", "Netscape-Enterprise/6.", nullptr }, // n + { nullptr }, { nullptr }, { nullptr }, { nullptr }, // o - r + { nullptr }, { nullptr }, { nullptr }, { nullptr }, // s - v + { "WebLogic 3.", "WebLogic 4.","WebLogic 5.", "WebLogic 6.", + "Winstone Servlet Engine v0.", nullptr }, // w + { nullptr }, { nullptr }, { nullptr } // x - z + }; + + int index = val.get()[0] - 'A'; // the whole table begins with capital letters + if ((index >= 0) && (index <= 25)) + { + for (int i = 0; bad_servers[index][i] != nullptr; i++) { + if (val.Equals(bad_servers[index][i])) { + LOG(("looks like this server does not support pipelining")); + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::RedBannedServer, this , 0); + return false; + } + } + } + + // ok, let's allow pipelining to this server + return true; +} + //---------------------------------------------------------------------------- // nsHttpConnection::nsAHttpConnection compatible methods //---------------------------------------------------------------------------- @@ -1017,6 +1095,9 @@ nsHttpConnection::OnHeadersAvailable(nsAHttpTransaction *trans, explicitKeepAlive = false; } + // reset to default (the server may have changed since we last checked) + mSupportsPipelining = false; + if ((responseHead->Version() < NS_HTTP_VERSION_1_1) || (requestHead->Version() < NS_HTTP_VERSION_1_1)) { // HTTP/1.0 connections are by default NOT persistent @@ -1024,13 +1105,62 @@ nsHttpConnection::OnHeadersAvailable(nsAHttpTransaction *trans, mKeepAlive = true; else mKeepAlive = false; + + // We need at least version 1.1 to use pipelines + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::RedVersionTooLow, this, 0); } else { // HTTP/1.1 connections are by default persistent - mKeepAlive = !explicitClose; + if (explicitClose) { + mKeepAlive = false; + + // persistent connections are required for pipelining to work - if + // this close was not pre-announced then generate the negative + // BadExplicitClose feedback + if (mRemainingConnectionUses > 1) + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::BadExplicitClose, this, 0); + } + else { + mKeepAlive = true; + + // Do not support pipelining when we are establishing + // an SSL tunnel though an HTTP proxy. Pipelining support + // determination must be based on comunication with the + // target server in this case. See bug 422016 for futher + // details. + if (!mProxyConnectStream) + mSupportsPipelining = SupportsPipelining(responseHead); + } } mKeepAliveMask = mKeepAlive; + // Update the pipelining status in the connection info object + // and also read it back. It is possible the ci status is + // locked to false if pipelining has been banned on this ci due to + // some kind of observed flaky behavior + if (mSupportsPipelining) { + // report the pipelining-compatible header to the connection manager + // as positive feedback. This will undo 1 penalty point the host + // may have accumulated in the past. + + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::NeutralExpectedOK, this, 0); + + mSupportsPipelining = + gHttpHandler->ConnMgr()->SupportsPipelining(mConnInfo); + } + + // If this connection is reserved for revalidations and we are + // receiving a document that failed revalidation then switch the + // classification to general to avoid pipelining more revalidations behind + // it. + if (mClassification == nsAHttpTransaction::CLASS_REVALIDATION && + responseStatus != 304) { + mClassification = nsAHttpTransaction::CLASS_GENERAL; + } + // if this connection is persistent, then the server may send a "Keep-Alive" // header specifying the maximum number of times the connection can be // reused as well as the maximum amount of time the connection can be idle @@ -1256,7 +1386,67 @@ nsHttpConnection::ReadTimeoutTick(PRIntervalTime now) nextTickAfter = std::max(nextTickAfter, 1U); } - return nextTickAfter; + if (!gHttpHandler->GetPipelineRescheduleOnTimeout()) + return nextTickAfter; + + PRIntervalTime delta = now - mLastReadTime; + + // we replicate some of the checks both here and in OnSocketReadable() as + // they will be discovered under different conditions. The ones here + // will generally be discovered if we are totally hung and OSR does + // not get called at all, however OSR discovers them with lower latency + // if the issue is just very slow (but not stalled) reading. + // + // Right now we only take action if pipelining is involved, but this would + // be the place to add general read timeout handling if it is desired. + + uint32_t pipelineDepth = mTransaction->PipelineDepth(); + if (pipelineDepth > 1) { + // if we have pipelines outstanding (not just an idle connection) + // then get a fairly quick tick + nextTickAfter = 1; + } + + if (delta >= gHttpHandler->GetPipelineRescheduleTimeout() && + pipelineDepth > 1) { + + // this just reschedules blocked transactions. no transaction + // is aborted completely. + LOG(("cancelling pipeline due to a %ums stall - depth %d\n", + PR_IntervalToMilliseconds(delta), pipelineDepth)); + + nsHttpPipeline *pipeline = mTransaction->QueryPipeline(); + MOZ_ASSERT(pipeline, "pipelinedepth > 1 without pipeline"); + // code this defensively for the moment and check for null in opt build + // This will reschedule blocked members of the pipeline, but the + // blocking transaction (i.e. response 0) will not be changed. + if (pipeline) { + pipeline->CancelPipeline(NS_ERROR_NET_TIMEOUT); + LOG(("Rescheduling the head of line blocked members of a pipeline " + "because reschedule-timeout idle interval exceeded")); + } + } + + if (delta < gHttpHandler->GetPipelineTimeout()) + return nextTickAfter; + + if (pipelineDepth <= 1 && !mTransaction->PipelinePosition()) + return nextTickAfter; + + // nothing has transpired on this pipelined socket for many + // seconds. Call that a total stall and close the transaction. + // There is a chance the transaction will be restarted again + // depending on its state.. that will come back araound + // without pipelining on, so this won't loop. + + LOG(("canceling transaction stalled for %ums on a pipeline " + "of depth %d and scheduled originally at pos %d\n", + PR_IntervalToMilliseconds(delta), + pipelineDepth, mTransaction->PipelinePosition())); + + // This will also close the connection + CloseTransaction(mTransaction, NS_ERROR_NET_TIMEOUT); + return UINT32_MAX; } void @@ -1746,8 +1936,49 @@ nsHttpConnection::OnSocketReadable() gHttpHandler->ProcessPendingQ(mConnInfo); } + // Look for data being sent in bursts with large pauses. If the pauses + // are caused by server bottlenecks such as think-time, disk i/o, or + // cpu exhaustion (as opposed to network latency) then we generate negative + // pipelining feedback to prevent head of line problems + // Reduce the estimate of the time since last read by up to 1 RTT to // accommodate exhausted sender TCP congestion windows or minor I/O delays. + + if (delta > mRtt) + delta -= mRtt; + else + delta = 0; + + static const PRIntervalTime k400ms = PR_MillisecondsToInterval(400); + + if (delta >= (mRtt + gHttpHandler->GetPipelineRescheduleTimeout())) { + LOG(("Read delta ms of %u causing slow read major " + "event and pipeline cancellation", + PR_IntervalToMilliseconds(delta))); + + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::BadSlowReadMajor, this, 0); + + if (gHttpHandler->GetPipelineRescheduleOnTimeout() && + mTransaction->PipelineDepth() > 1) { + nsHttpPipeline *pipeline = mTransaction->QueryPipeline(); + MOZ_ASSERT(pipeline, "pipelinedepth > 1 without pipeline"); + // code this defensively for the moment and check for null + // This will reschedule blocked members of the pipeline, but the + // blocking transaction (i.e. response 0) will not be changed. + if (pipeline) { + pipeline->CancelPipeline(NS_ERROR_NET_TIMEOUT); + LOG(("Rescheduling the head of line blocked members of a " + "pipeline because reschedule-timeout idle interval " + "exceeded")); + } + } + } + else if (delta > k400ms) { + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::BadSlowReadMinor, this, 0); + } + mLastReadTime = now; nsresult rv; diff --git a/netwerk/protocol/http/nsHttpConnection.h b/netwerk/protocol/http/nsHttpConnection.h index d7b9981f771a..1f8500d7552a 100644 --- a/netwerk/protocol/http/nsHttpConnection.h +++ b/netwerk/protocol/http/nsHttpConnection.h @@ -82,6 +82,7 @@ public: //------------------------------------------------------------------------- // XXX document when these are ok to call + bool SupportsPipelining(); bool IsKeepAlive() { return mUsingSpdyVersion || (mKeepAliveMask && mKeepAlive); @@ -176,6 +177,12 @@ public: // should move from short-lived (fast-detect) to long-lived. static void UpdateTCPKeepalive(nsITimer *aTimer, void *aClosure); + nsAHttpTransaction::Classifier Classification() { return mClassification; } + void Classify(nsAHttpTransaction::Classifier newclass) + { + mClassification = newclass; + } + // When the connection is active this is called every second void ReadTimeoutTick(); @@ -231,6 +238,7 @@ private: PRIntervalTime IdleTime(); bool IsAlive(); + bool SupportsPipelining(nsHttpResponseHead *); // Makes certain the SSL handshake is complete and NPN negotiation // has had a chance to happen @@ -300,6 +308,7 @@ private: bool mKeepAlive; bool mKeepAliveMask; bool mDontReuse; + bool mSupportsPipelining; bool mIsReused; bool mCompletedProxyConnect; bool mLastTransactionExpectedNoContent; @@ -322,6 +331,8 @@ private: // on this persistent connection. uint32_t mRemainingConnectionUses; + nsAHttpTransaction::Classifier mClassification; + // SPDY related bool mNPNComplete; bool mSetupSSLCalled; diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.cpp b/netwerk/protocol/http/nsHttpConnectionMgr.cpp index a161f6b5865d..59b406317a2a 100644 --- a/netwerk/protocol/http/nsHttpConnectionMgr.cpp +++ b/netwerk/protocol/http/nsHttpConnectionMgr.cpp @@ -14,6 +14,7 @@ #include "nsHttpConnectionMgr.h" #include "nsHttpConnection.h" +#include "nsHttpPipeline.h" #include "nsHttpHandler.h" #include "nsIHttpChannelInternal.h" #include "nsNetCID.h" @@ -122,7 +123,9 @@ nsresult nsHttpConnectionMgr::Init(uint16_t maxConns, uint16_t maxPersistConnsPerHost, uint16_t maxPersistConnsPerProxy, - uint16_t maxRequestDelay) + uint16_t maxRequestDelay, + uint16_t maxPipelinedRequests, + uint16_t maxOptimisticPipelinedRequests) { LOG(("nsHttpConnectionMgr::Init\n")); @@ -133,6 +136,8 @@ nsHttpConnectionMgr::Init(uint16_t maxConns, mMaxPersistConnsPerHost = maxPersistConnsPerHost; mMaxPersistConnsPerProxy = maxPersistConnsPerProxy; mMaxRequestDelay = maxRequestDelay; + mMaxPipelinedRequests = maxPipelinedRequests; + mMaxOptimisticPipelinedRequests = maxOptimisticPipelinedRequests; mIsShuttingDown = false; } @@ -933,6 +938,113 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsHttpConnectionInfo *ci) return false; } +bool +nsHttpConnectionMgr::SupportsPipelining(nsHttpConnectionInfo *ci) +{ + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + nsConnectionEntry *ent = mCT.Get(ci->HashKey()); + if (ent) + return ent->SupportsPipelining(); + return false; +} + +// nsHttpPipelineFeedback used to hold references across events + +class nsHttpPipelineFeedback : public ARefBase +{ +public: + nsHttpPipelineFeedback(nsHttpConnectionInfo *ci, + nsHttpConnectionMgr::PipelineFeedbackInfoType info, + nsHttpConnection *conn, uint32_t data) + : mConnInfo(ci) + , mConn(conn) + , mInfo(info) + , mData(data) + { + } + + + RefPtr mConnInfo; + RefPtr mConn; + nsHttpConnectionMgr::PipelineFeedbackInfoType mInfo; + uint32_t mData; +private: + ~nsHttpPipelineFeedback() {} + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(nsHttpPipelineFeedback) +}; + +void +nsHttpConnectionMgr::PipelineFeedbackInfo(nsHttpConnectionInfo *ci, + PipelineFeedbackInfoType info, + nsHttpConnection *conn, + uint32_t data) +{ + if (!ci) + return; + + // Post this to the socket thread if we are not running there already + if (PR_GetCurrentThread() != gSocketThread) { + RefPtr fb = + new nsHttpPipelineFeedback(ci, info, conn, data); + PostEvent(&nsHttpConnectionMgr::OnMsgProcessFeedback, 0, fb); + return; + } + + nsConnectionEntry *ent = mCT.Get(ci->HashKey()); + if (ent) + ent->OnPipelineFeedbackInfo(info, conn, data); +} + +void +nsHttpConnectionMgr::ReportFailedToProcess(nsIURI *uri) +{ + MOZ_ASSERT(uri); + + nsAutoCString host; + int32_t port = -1; + nsAutoCString username; + bool usingSSL = false; + bool isHttp = false; + + nsresult rv = uri->SchemeIs("https", &usingSSL); + if (NS_SUCCEEDED(rv) && usingSSL) + isHttp = true; + if (NS_SUCCEEDED(rv) && !isHttp) + rv = uri->SchemeIs("http", &isHttp); + if (NS_SUCCEEDED(rv)) + rv = uri->GetAsciiHost(host); + if (NS_SUCCEEDED(rv)) + rv = uri->GetPort(&port); + if (NS_SUCCEEDED(rv)) + uri->GetUsername(username); + if (NS_FAILED(rv) || !isHttp || host.IsEmpty()) + return; + + // report the event for all the permutations of anonymous and + // private versions of this host + RefPtr ci = + new nsHttpConnectionInfo(host, port, EmptyCString(), username, nullptr, + OriginAttributes(), usingSSL); + ci->SetAnonymous(false); + ci->SetPrivate(false); + PipelineFeedbackInfo(ci, RedCorruptedContent, nullptr, 0); + + ci = ci->Clone(); + ci->SetAnonymous(false); + ci->SetPrivate(true); + PipelineFeedbackInfo(ci, RedCorruptedContent, nullptr, 0); + + ci = ci->Clone(); + ci->SetAnonymous(true); + ci->SetPrivate(false); + PipelineFeedbackInfo(ci, RedCorruptedContent, nullptr, 0); + + ci = ci->Clone(); + ci->SetAnonymous(true); + ci->SetPrivate(true); + PipelineFeedbackInfo(ci, RedCorruptedContent, nullptr, 0); +} // we're at the active connection limit if any one of the following conditions is true: // (1) at max-connections @@ -1195,6 +1307,140 @@ nsHttpConnectionMgr::MakeNewConnection(nsConnectionEntry *ent, return NS_OK; } +bool +nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, + nsHttpTransaction *trans, + nsHttpTransaction::Classifier classification, + uint16_t depthLimit) +{ + if (classification == nsAHttpTransaction::CLASS_SOLO) + return false; + + uint32_t maxdepth = ent->MaxPipelineDepth(classification); + if (maxdepth == 0) { + ent->CreditPenalty(); + maxdepth = ent->MaxPipelineDepth(classification); + } + + if (ent->PipelineState() == PS_RED) + return false; + + if (ent->PipelineState() == PS_YELLOW && ent->mYellowConnection) + return false; + + // The maximum depth of a pipeline in yellow is 1 pipeline of + // depth 2 for entire CI. When that transaction completes successfully + // we transition to green and that expands the allowed depth + // to any number of pipelines of up to depth 4. When a transaction + // queued at position 3 or deeper succeeds we open it all the way + // up to depths limited only by configuration. The staggered start + // in green is simply because a successful yellow test of depth=2 + // might really just be a race condition (i.e. depth=1 from the + // server's point of view), while depth=3 is a stronger indicator - + // keeping the pipelines to a modest depth during that period limits + // the damage if something is going to go wrong. + + maxdepth = std::min(maxdepth, depthLimit); + + if (maxdepth < 2) + return false; + + nsAHttpTransaction *activeTrans; + + nsHttpConnection *bestConn = nullptr; + uint32_t activeCount = ent->mActiveConns.Length(); + uint32_t bestConnLength = 0; + uint32_t connLength; + + for (uint32_t i = 0; i < activeCount; ++i) { + nsHttpConnection *conn = ent->mActiveConns[i]; + if (!conn->SupportsPipelining()) + continue; + + if (conn->Classification() != classification) + continue; + + activeTrans = conn->Transaction(); + if (!activeTrans || + activeTrans->IsDone() || + NS_FAILED(activeTrans->Status())) + continue; + + connLength = activeTrans->PipelineDepth(); + + if (maxdepth <= connLength) + continue; + + if (!bestConn || (connLength < bestConnLength)) { + bestConn = conn; + bestConnLength = connLength; + } + } + + if (!bestConn) + return false; + + activeTrans = bestConn->Transaction(); + nsresult rv = activeTrans->AddTransaction(trans); + if (NS_FAILED(rv)) + return false; + + LOG((" scheduling trans %p on pipeline at position %d\n", + trans, trans->PipelinePosition())); + + if ((ent->PipelineState() == PS_YELLOW) && (trans->PipelinePosition() > 1)) + ent->SetYellowConnection(bestConn); + + if (!trans->GetPendingTime().IsNull()) { + if (trans->UsesPipelining()) + AccumulateTimeDelta( + Telemetry::TRANSACTION_WAIT_TIME_HTTP_PIPELINES, + trans->GetPendingTime(), TimeStamp::Now()); + else + AccumulateTimeDelta( + Telemetry::TRANSACTION_WAIT_TIME_HTTP, + trans->GetPendingTime(), TimeStamp::Now()); + trans->SetPendingTime(false); + } + return true; +} + +bool +nsHttpConnectionMgr::IsUnderPressure(nsConnectionEntry *ent, + nsHttpTransaction::Classifier classification) +{ + // A connection entry is declared to be "under pressure" if most of the + // allowed parallel connections are already used up. In that case we want to + // favor existing pipelines over more parallelism so as to reserve any + // unused parallel connections for types that don't have existing pipelines. + // + // The definition of connection pressure is a pretty liberal one here - that + // is why we are using the more restrictive maxPersist* counters. + // + // Pipelines are also favored when the requested classification is already + // using 3 or more of the connections. Failure to do this could result in + // one class (e.g. images) establishing self replenishing queues on all the + // connections that would starve the other transaction types. + + int32_t currentConns = ent->mActiveConns.Length(); + int32_t maxConns = + (ent->mConnInfo->UsingHttpProxy() && !ent->mConnInfo->UsingConnect()) ? + mMaxPersistConnsPerProxy : mMaxPersistConnsPerHost; + + // Leave room for at least 3 distinct types to operate concurrently, + // this satisfies the typical {html, js/css, img} page. + if (currentConns >= (maxConns - 2)) + return true; /* prefer pipeline */ + + int32_t sameClass = 0; + for (int32_t i = 0; i < currentConns; ++i) + if (classification == ent->mActiveConns[i]->Classification()) + if (++sameClass == 3) + return true; /* prefer pipeline */ + + return false; /* normal behavior */ +} + // returns OK if a connection is found for the transaction // and the transaction is started. // returns ERROR_NOT_AVAILABLE if no connection can be found and it @@ -1215,8 +1461,13 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, onlyReusedConnection, ent->mActiveConns.Length(), ent->mIdleConns.Length())); + nsHttpTransaction::Classifier classification = trans->Classification(); uint32_t caps = trans->Caps(); + // no keep-alive means no pipelines either + if (!(caps & NS_HTTP_ALLOW_KEEPALIVE)) + caps = caps & ~NS_HTTP_ALLOW_PIPELINING; + // 0 - If this should use spdy then dispatch it post haste. // 1 - If there is connection pressure then see if we can pipeline this on // a connection of a matching type instead of using a new conn @@ -1231,6 +1482,7 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, // global limits // 6 - no connection is available - queue it + bool attemptedOptimisticPipeline = !(caps & NS_HTTP_ALLOW_PIPELINING); RefPtr unusedSpdyPersistentConnection; // step 0 @@ -1276,7 +1528,15 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, // step 1 // If connection pressure, then we want to favor pipelining of any kind - // h1 pipelining has been removed + if (IsUnderPressure(ent, classification) && !attemptedOptimisticPipeline) { + attemptedOptimisticPipeline = true; + if (AddToShortestPipeline(ent, trans, + classification, + mMaxOptimisticPipelinedRequests)) { + LOG((" dispatched step 1 trans=%p\n", trans)); + return NS_OK; + } + } // Subject most transactions at high parallelism to rate pacing. // It will only be actually submitted to the @@ -1340,7 +1600,18 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, // step 3 // consider pipelining scripts and revalidations - // h1 pipelining has been removed + if (!attemptedOptimisticPipeline && + (classification == nsHttpTransaction::CLASS_REVALIDATION || + classification == nsHttpTransaction::CLASS_SCRIPT)) { + // Assignation kept here for documentation purpose; Never read after + attemptedOptimisticPipeline = true; + if (AddToShortestPipeline(ent, trans, + classification, + mMaxOptimisticPipelinedRequests)) { + LOG((" dispatched step 3 (pipeline) trans=%p\n", trans)); + return NS_OK; + } + } // step 4 if (!onlyReusedConnection) { @@ -1365,7 +1636,14 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, } // step 5 - // previously pipelined anything here if allowed but h1 pipelining has been removed + if (caps & NS_HTTP_ALLOW_PIPELINING) { + if (AddToShortestPipeline(ent, trans, + classification, + mMaxPipelinedRequests)) { + LOG((" dispatched step 5 trans=%p\n", trans)); + return NS_OK; + } + } // step 6 if (unusedSpdyPersistentConnection) { @@ -1394,7 +1672,7 @@ nsHttpConnectionMgr::DispatchTransaction(nsConnectionEntry *ent, // It is possible for a rate-paced transaction to be dispatched independent // of the token bucket when the amount of parallelization has changed or - // when a muxed connection (e.g. h2) becomes available. + // when a muxed connection (e.g. spdy or pipelines) becomes available. trans->CancelPacing(NS_OK); if (conn->UsingSpdy()) { @@ -1415,11 +1693,20 @@ nsHttpConnectionMgr::DispatchTransaction(nsConnectionEntry *ent, MOZ_ASSERT(conn && !conn->Transaction(), "DispatchTranaction() on non spdy active connection"); + if (!(caps & NS_HTTP_ALLOW_PIPELINING)) + conn->Classify(nsAHttpTransaction::CLASS_SOLO); + else + conn->Classify(trans->Classification()); + rv = DispatchAbstractTransaction(ent, trans, caps, conn, priority); if (NS_SUCCEEDED(rv) && !trans->GetPendingTime().IsNull()) { - AccumulateTimeDelta(Telemetry::TRANSACTION_WAIT_TIME_HTTP, - trans->GetPendingTime(), TimeStamp::Now()); + if (trans->UsesPipelining()) + AccumulateTimeDelta(Telemetry::TRANSACTION_WAIT_TIME_HTTP_PIPELINES, + trans->GetPendingTime(), TimeStamp::Now()); + else + AccumulateTimeDelta(Telemetry::TRANSACTION_WAIT_TIME_HTTP, + trans->GetPendingTime(), TimeStamp::Now()); trans->SetPendingTime(false); } return rv; @@ -1430,7 +1717,7 @@ nsHttpConnectionMgr::DispatchTransaction(nsConnectionEntry *ent, // // thin wrapper around a real connection, used to keep track of references // to the connection to determine when the connection may be reused. the -// transaction owns a reference to this handle. this extra +// transaction (or pipeline) owns a reference to this handle. this extra // layer of indirection greatly simplifies consumer code, avoiding the // need for consumer code to know when to give the connection back to the // connection manager. @@ -1474,14 +1761,33 @@ nsHttpConnectionMgr::DispatchAbstractTransaction(nsConnectionEntry *ent, nsHttpConnection *conn, int32_t priority) { - nsresult rv; MOZ_ASSERT(!conn->UsingSpdy(), "Spdy Must Not Use DispatchAbstractTransaction"); LOG(("nsHttpConnectionMgr::DispatchAbstractTransaction " "[ci=%s trans=%p caps=%x conn=%p]\n", ent->mConnInfo->HashKey().get(), aTrans, caps, conn)); - RefPtr transaction(aTrans); + /* Use pipeline datastructure even if connection does not currently qualify + to pipeline this transaction because a different pipeline-eligible + transaction might be placed on the active connection. Make an exception + for CLASS_SOLO as that connection will never pipeline until it goes + quiescent */ + + RefPtr transaction; + nsresult rv; + if (conn->Classification() != nsAHttpTransaction::CLASS_SOLO) { + LOG((" using pipeline datastructure.\n")); + RefPtr pipeline; + rv = BuildPipeline(ent, aTrans, getter_AddRefs(pipeline)); + if (!NS_SUCCEEDED(rv)) + return rv; + transaction = pipeline; + } + else { + LOG((" not using pipeline datastructure due to class solo.\n")); + transaction = aTrans; + } + RefPtr handle = new ConnectionHandle(conn); // give the transaction the indirect reference to the connection. @@ -1491,6 +1797,8 @@ nsHttpConnectionMgr::DispatchAbstractTransaction(nsConnectionEntry *ent, if (NS_FAILED(rv)) { LOG((" conn->Activate failed [rv=%" PRIx32 "]\n", static_cast(rv))); ent->mActiveConns.RemoveElement(conn); + if (conn == ent->mYellowConnection) + ent->OnYellowComplete(); DecrementActiveConnCount(conn); ConditionallyStopTimeoutTick(); @@ -1500,9 +1808,33 @@ nsHttpConnectionMgr::DispatchAbstractTransaction(nsConnectionEntry *ent, handle->Reset(); // destroy the connection } + // As transaction goes out of scope it will drop the last refernece to the + // pipeline if activation failed, in which case this will destroy + // the pipeline, which will cause each the transactions owned by the + // pipeline to be restarted. + return rv; } +nsresult +nsHttpConnectionMgr::BuildPipeline(nsConnectionEntry *ent, + nsAHttpTransaction *firstTrans, + nsHttpPipeline **result) +{ + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + /* form a pipeline here even if nothing is pending so that we + can stream-feed it as new transactions arrive */ + + /* the first transaction can go in unconditionally - 1 transaction + on a nsHttpPipeline object is not a real HTTP pipeline */ + + RefPtr pipeline = new nsHttpPipeline(); + pipeline->AddTransaction(firstTrans); + pipeline.forget(result); + return NS_OK; +} + void nsHttpConnectionMgr::ReportProxyTelemetry(nsConnectionEntry *ent) { @@ -2111,9 +2443,11 @@ nsHttpConnectionMgr::OnMsgPruneDeadConnections(int32_t, ARefBase *) ConditionallyStopPruneDeadConnectionsTimer(); } - // If this entry is empty, we have too many entries busy then - // we can clean it up and restart - if (mCT.Count() > 125 && + // If this entry is empty, we have too many entries, and this + // doesn't represent some painfully determined red condition, then + // we can clean it up and restart from yellow. + if (ent->PipelineState() != PS_RED && + mCT.Count() > 125 && ent->mIdleConns.Length() == 0 && ent->mActiveConns.Length() == 0 && ent->mHalfOpens.Length() == 0 && @@ -2277,6 +2611,9 @@ nsHttpConnectionMgr::OnMsgReclaimConnection(int32_t, ARefBase *param) } if (ent->mActiveConns.RemoveElement(conn)) { + if (conn == ent->mYellowConnection) { + ent->OnYellowComplete(); + } DecrementActiveConnCount(conn); ConditionallyStopTimeoutTick(); } @@ -2359,6 +2696,12 @@ nsHttpConnectionMgr::OnMsgUpdateParam(int32_t inParam, ARefBase *) case MAX_REQUEST_DELAY: mMaxRequestDelay = value; break; + case MAX_PIPELINED_REQUESTS: + mMaxPipelinedRequests = value; + break; + case MAX_OPTIMISTIC_PIPELINED_REQUESTS: + mMaxOptimisticPipelinedRequests = value; + break; default: NS_NOTREACHED("unexpected parameter name"); } @@ -2371,6 +2714,14 @@ nsHttpConnectionMgr::nsConnectionEntry::~nsConnectionEntry() gHttpHandler->ConnMgr()->RemovePreferredHash(this); } +void +nsHttpConnectionMgr::OnMsgProcessFeedback(int32_t, ARefBase *param) +{ + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + nsHttpPipelineFeedback *fb = static_cast(param); + PipelineFeedbackInfo(fb->mConnInfo, fb->mInfo, fb->mConn, fb->mData); +} + // Read Timeout Tick handlers void @@ -3059,10 +3410,12 @@ nsHalfOpenSocket::OnOutputStreamReady(nsIAsyncOutputStream *out) trans = mTransaction; } else { trans = new NullHttpTransaction(mEnt->mConnInfo, - callbacks, mCaps); + callbacks, + mCaps & ~NS_HTTP_ALLOW_PIPELINING); } gHttpHandler->ConnMgr()->AddActiveConn(conn, mEnt); + conn->Classify(nsAHttpTransaction::CLASS_SOLO); rv = gHttpHandler->ConnMgr()-> DispatchAbstractTransaction(mEnt, trans, mCaps, conn, 0); } else { @@ -3192,12 +3545,35 @@ ConnectionHandle::TakeHttpConnection() return mConn.forget(); } +uint32_t +ConnectionHandle::CancelPipeline(nsresult reason) +{ + // no pipeline to cancel + return 0; +} + +nsAHttpTransaction::Classifier +ConnectionHandle::Classification() +{ + if (mConn) + return mConn->Classification(); + + LOG(("ConnectionHandle::Classification this=%p " + "has null mConn using CLASS_SOLO default", this)); + return nsAHttpTransaction::CLASS_SOLO; +} // nsConnectionEntry nsHttpConnectionMgr:: nsConnectionEntry::nsConnectionEntry(nsHttpConnectionInfo *ci) : mConnInfo(ci) + , mPipelineState(PS_YELLOW) + , mYellowGoodEvents(0) + , mYellowBadEvents(0) + , mYellowConnection(nullptr) + , mGreenDepth(kPipelineOpen) + , mPipeliningPenalty(0) , mUsingSpdy(false) , mInPreferredHash(false) , mPreferIPv4(false) @@ -3205,6 +3581,12 @@ nsConnectionEntry::nsConnectionEntry(nsHttpConnectionInfo *ci) , mUsedForConnection(false) { MOZ_COUNT_CTOR(nsConnectionEntry); + if (gHttpHandler->GetPipelineAggressive()) { + mGreenDepth = kPipelineUnlimited; + mPipelineState = PS_GREEN; + } + mInitialGreenDepth = mGreenDepth; + memset(mPipeliningClassPenalty, 0, sizeof(int16_t) * nsAHttpTransaction::CLASS_MAX); } bool @@ -3218,6 +3600,238 @@ nsHttpConnectionMgr::nsConnectionEntry::AvailableForDispatchNow() GetSpdyPreferredConn(this) ? true : false; } +bool +nsHttpConnectionMgr::nsConnectionEntry::SupportsPipelining() +{ + return mPipelineState != nsHttpConnectionMgr::PS_RED; +} + +nsHttpConnectionMgr::PipeliningState +nsHttpConnectionMgr::nsConnectionEntry::PipelineState() +{ + return mPipelineState; +} + +void +nsHttpConnectionMgr:: +nsConnectionEntry::OnPipelineFeedbackInfo( + nsHttpConnectionMgr::PipelineFeedbackInfoType info, + nsHttpConnection *conn, + uint32_t data) +{ + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + if (mPipelineState == PS_YELLOW) { + if (info & kPipelineInfoTypeBad) + mYellowBadEvents++; + else if (info & (kPipelineInfoTypeNeutral | kPipelineInfoTypeGood)) + mYellowGoodEvents++; + } + + if (mPipelineState == PS_GREEN && info == GoodCompletedOK) { + int32_t depth = data; + LOG(("Transaction completed at pipeline depth of %d. Host = %s\n", + depth, mConnInfo->Origin())); + + if (depth >= 3) + mGreenDepth = kPipelineUnlimited; + } + + nsAHttpTransaction::Classifier classification; + if (conn) + classification = conn->Classification(); + else if (info == BadInsufficientFraming || + info == BadUnexpectedLarge) + classification = (nsAHttpTransaction::Classifier) data; + else + classification = nsAHttpTransaction::CLASS_SOLO; + + if (gHttpHandler->GetPipelineAggressive() && + info & kPipelineInfoTypeBad && + info != BadExplicitClose && + info != RedVersionTooLow && + info != RedBannedServer && + info != RedCorruptedContent && + info != BadInsufficientFraming) { + LOG(("minor negative feedback ignored " + "because of pipeline aggressive mode")); + } + else if (info & kPipelineInfoTypeBad) { + if ((info & kPipelineInfoTypeRed) && (mPipelineState != PS_RED)) { + LOG(("transition to red from %d. Host = %s.\n", + mPipelineState, mConnInfo->Origin())); + mPipelineState = PS_RED; + mPipeliningPenalty = 0; + } + + if (mLastCreditTime.IsNull()) + mLastCreditTime = TimeStamp::Now(); + + // Red* events impact the host globally via mPipeliningPenalty, while + // Bad* events impact the per class penalty. + + // The individual penalties should be < 16bit-signed-maxint - 25000 + // (approx 7500). Penalties are paid-off either when something promising + // happens (a successful transaction, or promising headers) or when + // time goes by at a rate of 1 penalty point every 16 seconds. + + switch (info) { + case RedVersionTooLow: + mPipeliningPenalty += 1000; + break; + case RedBannedServer: + mPipeliningPenalty += 7000; + break; + case RedCorruptedContent: + mPipeliningPenalty += 7000; + break; + case RedCanceledPipeline: + mPipeliningPenalty += 60; + break; + case BadExplicitClose: + mPipeliningClassPenalty[classification] += 250; + break; + case BadSlowReadMinor: + mPipeliningClassPenalty[classification] += 5; + break; + case BadSlowReadMajor: + mPipeliningClassPenalty[classification] += 25; + break; + case BadInsufficientFraming: + mPipeliningClassPenalty[classification] += 7000; + break; + case BadUnexpectedLarge: + mPipeliningClassPenalty[classification] += 120; + break; + + default: + MOZ_ASSERT(false, "Unknown Bad/Red Pipeline Feedback Event"); + } + + const int16_t kPenalty = 25000; + mPipeliningPenalty = std::min(mPipeliningPenalty, kPenalty); + mPipeliningClassPenalty[classification] = + std::min(mPipeliningClassPenalty[classification], kPenalty); + + LOG(("Assessing red penalty to %s class %d for event %d. " + "Penalty now %d, throttle[%d] = %d\n", mConnInfo->Origin(), + classification, info, mPipeliningPenalty, classification, + mPipeliningClassPenalty[classification])); + } + else { + // hand out credits for neutral and good events such as + // "headers look ok" events + + mPipeliningPenalty = std::max(mPipeliningPenalty - 1, 0); + mPipeliningClassPenalty[classification] = std::max(mPipeliningClassPenalty[classification] - 1, 0); + } + + if (mPipelineState == PS_RED && !mPipeliningPenalty) + { + LOG(("transition %s to yellow\n", mConnInfo->Origin())); + mPipelineState = PS_YELLOW; + mYellowConnection = nullptr; + } +} + +void +nsHttpConnectionMgr:: +nsConnectionEntry::SetYellowConnection(nsHttpConnection *conn) +{ + MOZ_ASSERT(!mYellowConnection && mPipelineState == PS_YELLOW, + "yellow connection already set or state is not yellow"); + mYellowConnection = conn; + mYellowGoodEvents = mYellowBadEvents = 0; +} + +void +nsHttpConnectionMgr:: +nsConnectionEntry::OnYellowComplete() +{ + if (mPipelineState == PS_YELLOW) { + if (mYellowGoodEvents && !mYellowBadEvents) { + LOG(("transition %s to green\n", mConnInfo->Origin())); + mPipelineState = PS_GREEN; + mGreenDepth = mInitialGreenDepth; + } + else { + // The purpose of the yellow state is to witness at least + // one successful pipelined transaction without seeing any + // kind of negative feedback before opening the flood gates. + // If we haven't confirmed that, then transfer back to red. + LOG(("transition %s to red from yellow return\n", + mConnInfo->Origin())); + mPipelineState = PS_RED; + } + } + + mYellowConnection = nullptr; +} + +void +nsHttpConnectionMgr:: +nsConnectionEntry::CreditPenalty() +{ + if (mLastCreditTime.IsNull()) + return; + + // Decrease penalty values by 1 for every 16 seconds + // (i.e 3.7 per minute, or 1000 every 4h20m) + + TimeStamp now = TimeStamp::Now(); + TimeDuration elapsedTime = now - mLastCreditTime; + uint32_t creditsEarned = + static_cast(elapsedTime.ToSeconds()) >> 4; + + bool failed = false; + if (creditsEarned > 0) { + mPipeliningPenalty = + std::max(int32_t(mPipeliningPenalty - creditsEarned), 0); + if (mPipeliningPenalty > 0) + failed = true; + + for (int32_t i = 0; i < nsAHttpTransaction::CLASS_MAX; ++i) { + mPipeliningClassPenalty[i] = + std::max(int32_t(mPipeliningClassPenalty[i] - creditsEarned), 0); + failed = failed || (mPipeliningClassPenalty[i] > 0); + } + + // update last credit mark to reflect elapsed time + mLastCreditTime += TimeDuration::FromSeconds(creditsEarned << 4); + } + else { + failed = true; /* just assume this */ + } + + // If we are no longer red then clear the credit counter - you only + // get credits for time spent in the red state + if (!failed) + mLastCreditTime = TimeStamp(); /* reset to null timestamp */ + + if (mPipelineState == PS_RED && !mPipeliningPenalty) + { + LOG(("transition %s to yellow based on time credit\n", + mConnInfo->Origin())); + mPipelineState = PS_YELLOW; + mYellowConnection = nullptr; + } +} + +uint32_t +nsHttpConnectionMgr:: +nsConnectionEntry::MaxPipelineDepth(nsAHttpTransaction::Classifier aClass) +{ + // Still subject to configuration limit no matter return value + + if ((mPipelineState == PS_RED) || (mPipeliningClassPenalty[aClass] > 0)) + return 0; + + if (mPipelineState == PS_YELLOW) + return kPipelineRestricted; + + return mGreenDepth; +} + bool nsHttpConnectionMgr::GetConnectionData(nsTArray *aArg) { diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.h b/netwerk/protocol/http/nsHttpConnectionMgr.h index 0b951a48e5bb..7ca2a2b28d03 100644 --- a/netwerk/protocol/http/nsHttpConnectionMgr.h +++ b/netwerk/protocol/http/nsHttpConnectionMgr.h @@ -48,7 +48,9 @@ public: MAX_CONNECTIONS, MAX_PERSISTENT_CONNECTIONS_PER_HOST, MAX_PERSISTENT_CONNECTIONS_PER_PROXY, - MAX_REQUEST_DELAY + MAX_REQUEST_DELAY, + MAX_PIPELINED_REQUESTS, + MAX_OPTIMISTIC_PIPELINED_REQUESTS }; //------------------------------------------------------------------------- @@ -60,7 +62,9 @@ public: nsresult Init(uint16_t maxConnections, uint16_t maxPersistentConnectionsPerHost, uint16_t maxPersistentConnectionsPerProxy, - uint16_t maxRequestDelay); + uint16_t maxRequestDelay, + uint16_t maxPipelinedRequests, + uint16_t maxOptimisticPipelinedRequests); nsresult Shutdown(); //------------------------------------------------------------------------- @@ -145,6 +149,70 @@ public: // clears the connection history mCT nsresult ClearConnectionHistory(); + // Pipielining Interfaces and Datatypes + + const static uint32_t kPipelineInfoTypeMask = 0xffff0000; + const static uint32_t kPipelineInfoIDMask = ~kPipelineInfoTypeMask; + + const static uint32_t kPipelineInfoTypeRed = 0x00010000; + const static uint32_t kPipelineInfoTypeBad = 0x00020000; + const static uint32_t kPipelineInfoTypeNeutral = 0x00040000; + const static uint32_t kPipelineInfoTypeGood = 0x00080000; + + enum PipelineFeedbackInfoType + { + // Used when an HTTP response less than 1.1 is received + RedVersionTooLow = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0001, + + // Used when a HTTP Server response header that is on the banned from + // pipelining list is received + RedBannedServer = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0002, + + // Used when a response is terminated early, when it fails an + // integrity check such as assoc-req or when a 304 contained a Last-Modified + // differnet than the entry being validated. + RedCorruptedContent = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0004, + + // Used when a pipeline is only partly satisfied - for instance if the + // server closed the connection after responding to the first + // request but left some requests unprocessed. + RedCanceledPipeline = kPipelineInfoTypeRed | kPipelineInfoTypeBad | 0x0005, + + // Used when a connection that we expected to stay persistently open + // was closed by the server. Not used when simply timed out. + BadExplicitClose = kPipelineInfoTypeBad | 0x0003, + + // Used when there is a gap of around 400 - 1200ms in between data being + // read from the server + BadSlowReadMinor = kPipelineInfoTypeBad | 0x0006, + + // Used when there is a gap of > 1200ms in between data being + // read from the server + BadSlowReadMajor = kPipelineInfoTypeBad | 0x0007, + + // Used when a response is received that is not framed with either chunked + // encoding or a complete content length. + BadInsufficientFraming = kPipelineInfoTypeBad | 0x0008, + + // Used when a very large response is recevied in a potential pipelining + // context. Large responses cause head of line blocking. + BadUnexpectedLarge = kPipelineInfoTypeBad | 0x000B, + + // Used when a response is received that has headers that appear to support + // pipelining. + NeutralExpectedOK = kPipelineInfoTypeNeutral | 0x0009, + + // Used when a response is received successfully to a pipelined request. + GoodCompletedOK = kPipelineInfoTypeGood | 0x000A + }; + + // called to provide information relevant to the pipelining manager + // may be called from any thread + void PipelineFeedbackInfo(nsHttpConnectionInfo *, + PipelineFeedbackInfoType info, + nsHttpConnection *, + uint32_t); + void ReportFailedToProcess(nsIURI *uri); // Causes a large amount of connection diagnostic information to be @@ -179,6 +247,8 @@ public: // bit different. void ReportSpdyConnection(nsHttpConnection *, bool usingSpdy); + bool SupportsPipelining(nsHttpConnectionInfo *); + bool GetConnectionData(nsTArray *); void ResetIPFamilyPreference(nsHttpConnectionInfo *); @@ -191,6 +261,23 @@ public: private: virtual ~nsHttpConnectionMgr(); + enum PipeliningState { + // Host has proven itself pipeline capable through past experience and + // large pipeline depths are allowed on multiple connections. + PS_GREEN, + + // Not enough information is available yet with this host to be certain + // of pipeline capability. Small pipelines on a single connection are + // allowed in order to decide whether or not to proceed to green. + PS_YELLOW, + + // One or more bad events has happened that indicate that pipelining + // to this host (or a particular type of transaction with this host) + // is a bad idea. Pipelining is not currently allowed, but time and + // other positive experiences will eventually allow it to try again. + PS_RED + }; + class nsHalfOpenSocket; // nsConnectionEntry @@ -220,6 +307,54 @@ private: // Remove a particular half open socket from the mHalfOpens array void RemoveHalfOpen(nsHalfOpenSocket *); + // Pipeline depths for various states + const static uint32_t kPipelineUnlimited = 1024; // fully open - extended green + const static uint32_t kPipelineOpen = 6; // 6 on each conn - normal green + const static uint32_t kPipelineRestricted = 2; // 2 on just 1 conn in yellow + + nsHttpConnectionMgr::PipeliningState PipelineState(); + void OnPipelineFeedbackInfo( + nsHttpConnectionMgr::PipelineFeedbackInfoType info, + nsHttpConnection *, uint32_t); + bool SupportsPipelining(); + uint32_t MaxPipelineDepth(nsAHttpTransaction::Classifier classification); + void CreditPenalty(); + + nsHttpConnectionMgr::PipeliningState mPipelineState; + + void SetYellowConnection(nsHttpConnection *); + void OnYellowComplete(); + uint32_t mYellowGoodEvents; + uint32_t mYellowBadEvents; + nsHttpConnection *mYellowConnection; + + // initialGreenDepth is the max depth of a pipeline when you first + // transition to green. Normally this is kPipelineOpen, but it can + // be kPipelineUnlimited in aggressive mode. + uint32_t mInitialGreenDepth; + + // greenDepth is the current max allowed depth of a pipeline when + // in the green state. Normally this starts as kPipelineOpen and + // grows to kPipelineUnlimited after a pipeline of depth 3 has been + // successfully transacted. + uint32_t mGreenDepth; + + // pipeliningPenalty is the current amount of penalty points this host + // entry has earned for participating in events that are not conducive + // to good pipelines - such as head of line blocking, canceled pipelines, + // etc.. penalties are paid back either through elapsed time or simply + // healthy transactions. Having penalty points means that this host is + // not currently eligible for pipelines. + int16_t mPipeliningPenalty; + + // some penalty points only apply to particular classifications of + // transactions - this allows a server that perhaps has head of line + // blocking problems on CGI queries to still serve JS pipelined. + int16_t mPipeliningClassPenalty[nsAHttpTransaction::CLASS_MAX]; + + // for calculating penalty repair credits + TimeStamp mLastCreditTime; + // Spdy sometimes resolves the address in the socket manager in order // to re-coalesce sharded HTTP hosts. The dotted decimal address is // combined with the Anonymous flag from the connection information @@ -364,6 +499,8 @@ private: uint16_t mMaxPersistConnsPerHost; uint16_t mMaxPersistConnsPerProxy; uint16_t mMaxRequestDelay; // in seconds + uint16_t mMaxPipelinedRequests; + uint16_t mMaxOptimisticPipelinedRequests; Atomic mIsShuttingDown; //------------------------------------------------------------------------- @@ -371,6 +508,8 @@ private: //------------------------------------------------------------------------- bool ProcessPendingQForEntry(nsConnectionEntry *, bool considerAll); + bool IsUnderPressure(nsConnectionEntry *ent, + nsHttpTransaction::Classifier classification); bool AtActiveConnectionLimit(nsConnectionEntry *, uint32_t caps); nsresult TryDispatchTransaction(nsConnectionEntry *ent, bool onlyReusedConnection, @@ -383,6 +522,9 @@ private: uint32_t, nsHttpConnection *, int32_t); + nsresult BuildPipeline(nsConnectionEntry *, + nsAHttpTransaction *, + nsHttpPipeline **); bool RestrictConnections(nsConnectionEntry *); nsresult ProcessNewTransaction(nsHttpTransaction *); nsresult EnsureSocketThreadTarget(); @@ -400,6 +542,10 @@ private: nsresult MakeNewConnection(nsConnectionEntry *ent, nsHttpTransaction *trans); + bool AddToShortestPipeline(nsConnectionEntry *ent, + nsHttpTransaction *trans, + nsHttpTransaction::Classifier classification, + uint16_t depthLimit); // Manage the preferred spdy connection entry for this address nsConnectionEntry *GetSpdyPreferredEnt(nsConnectionEntry *aOriginalEntry); diff --git a/netwerk/protocol/http/nsHttpHandler.cpp b/netwerk/protocol/http/nsHttpHandler.cpp index d0f48c5363ae..cd986ac22e85 100644 --- a/netwerk/protocol/http/nsHttpHandler.cpp +++ b/netwerk/protocol/http/nsHttpHandler.cpp @@ -41,6 +41,7 @@ #include "nsIObserverService.h" #include "nsISiteSecurityService.h" #include "nsIStreamConverterService.h" +#include "nsITimer.h" #include "nsCRT.h" #include "nsIMemoryReporter.h" #include "nsIParentalControlsService.h" @@ -173,6 +174,7 @@ nsHttpHandler::nsHttpHandler() , mReferrerXOriginTrimmingPolicy(0) , mReferrerXOriginPolicy(0) , mFastFallbackToIPv4(false) + , mProxyPipelining(true) , mIdleTimeout(PR_SecondsToInterval(10)) , mSpdyTimeout(PR_SecondsToInterval(180)) , mResponseTimeout(PR_SecondsToInterval(300)) @@ -182,12 +184,21 @@ nsHttpHandler::nsHttpHandler() , mMaxRequestDelay(10) , mIdleSynTimeout(250) , mH2MandatorySuiteEnabled(false) + , mPipeliningEnabled(false) , mMaxConnections(24) , mMaxPersistentConnectionsPerServer(2) , mMaxPersistentConnectionsPerProxy(4) + , mMaxPipelinedRequests(32) + , mMaxOptimisticPipelinedRequests(4) + , mPipelineAggressive(false) + , mMaxPipelineObjectSize(300000) + , mPipelineRescheduleOnTimeout(true) + , mPipelineRescheduleTimeout(PR_MillisecondsToInterval(1500)) + , mPipelineReadTimeout(PR_MillisecondsToInterval(30000)) , mRedirectionLimit(10) , mPhishyUserPassLength(1) , mQoSBits(0x00) + , mPipeliningOverSSL(false) , mEnforceAssocReq(false) , mLastUniqueID(NowInSeconds()) , mSessionStartTime(0) @@ -258,6 +269,11 @@ nsHttpHandler::~nsHttpHandler() // and it'll segfault. NeckoChild will get cleaned up by process exit. nsHttp::DestroyAtomTable(); + if (mPipelineTestTimer) { + mPipelineTestTimer->Cancel(); + mPipelineTestTimer = nullptr; + } + gHttpHandler = nullptr; } @@ -378,13 +394,11 @@ nsHttpHandler::Init() obsService->AddObserver(this, "net:prune-dead-connections", true); // Sent by the TorButton add-on in the Tor Browser obsService->AddObserver(this, "net:prune-all-connections", true); + obsService->AddObserver(this, "net:failed-to-process-uri-content", true); obsService->AddObserver(this, "last-pb-context-exited", true); obsService->AddObserver(this, "browser:purge-session-history", true); obsService->AddObserver(this, NS_NETWORK_LINK_TOPIC, true); obsService->AddObserver(this, "application-background", true); - - // disabled as its a nop right now - // obsService->AddObserver(this, "net:failed-to-process-uri-content", true); } MakeNewRequestTokenBucket(); @@ -429,7 +443,9 @@ nsHttpHandler::InitConnectionMgr() rv = mConnMgr->Init(mMaxConnections, mMaxPersistentConnectionsPerServer, mMaxPersistentConnectionsPerProxy, - mMaxRequestDelay); + mMaxRequestDelay, + mMaxPipelinedRequests, + mMaxOptimisticPipelinedRequests); return rv; } @@ -1141,6 +1157,96 @@ nsHttpHandler::PrefsChanged(nsIPrefBranch *prefs, const char *pref) } } + if (PREF_CHANGED(HTTP_PREF("pipelining"))) { + rv = prefs->GetBoolPref(HTTP_PREF("pipelining"), &cVar); + if (NS_SUCCEEDED(rv)) { + if (cVar) + mCapabilities |= NS_HTTP_ALLOW_PIPELINING; + else + mCapabilities &= ~NS_HTTP_ALLOW_PIPELINING; + mPipeliningEnabled = cVar; + } + } + + if (PREF_CHANGED(HTTP_PREF("pipelining.maxrequests"))) { + rv = prefs->GetIntPref(HTTP_PREF("pipelining.maxrequests"), &val); + if (NS_SUCCEEDED(rv)) { + mMaxPipelinedRequests = clamped(val, 1, 0xffff); + if (mConnMgr) + mConnMgr->UpdateParam(nsHttpConnectionMgr::MAX_PIPELINED_REQUESTS, + mMaxPipelinedRequests); + } + } + + if (PREF_CHANGED(HTTP_PREF("pipelining.max-optimistic-requests"))) { + rv = prefs-> + GetIntPref(HTTP_PREF("pipelining.max-optimistic-requests"), &val); + if (NS_SUCCEEDED(rv)) { + mMaxOptimisticPipelinedRequests = clamped(val, 1, 0xffff); + if (mConnMgr) + mConnMgr->UpdateParam + (nsHttpConnectionMgr::MAX_OPTIMISTIC_PIPELINED_REQUESTS, + mMaxOptimisticPipelinedRequests); + } + } + + if (PREF_CHANGED(HTTP_PREF("pipelining.aggressive"))) { + rv = prefs->GetBoolPref(HTTP_PREF("pipelining.aggressive"), &cVar); + if (NS_SUCCEEDED(rv)) + mPipelineAggressive = cVar; + } + + if (PREF_CHANGED(HTTP_PREF("pipelining.maxsize"))) { + rv = prefs->GetIntPref(HTTP_PREF("pipelining.maxsize"), &val); + if (NS_SUCCEEDED(rv)) { + mMaxPipelineObjectSize = + static_cast(clamped(val, 1000, 100000000)); + } + } + + // Determines whether or not to actually reschedule after the + // reschedule-timeout has expired + if (PREF_CHANGED(HTTP_PREF("pipelining.reschedule-on-timeout"))) { + rv = prefs->GetBoolPref(HTTP_PREF("pipelining.reschedule-on-timeout"), + &cVar); + if (NS_SUCCEEDED(rv)) + mPipelineRescheduleOnTimeout = cVar; + } + + // The amount of time head of line blocking is allowed (in ms) + // before the blocked transactions are moved to another pipeline + if (PREF_CHANGED(HTTP_PREF("pipelining.reschedule-timeout"))) { + rv = prefs->GetIntPref(HTTP_PREF("pipelining.reschedule-timeout"), + &val); + if (NS_SUCCEEDED(rv)) { + mPipelineRescheduleTimeout = + PR_MillisecondsToInterval((uint16_t) clamped(val, 500, 0xffff)); + } + } + + // The amount of time a pipelined transaction is allowed to wait before + // being canceled and retried in a non-pipeline connection + if (PREF_CHANGED(HTTP_PREF("pipelining.read-timeout"))) { + rv = prefs->GetIntPref(HTTP_PREF("pipelining.read-timeout"), &val); + if (NS_SUCCEEDED(rv)) { + mPipelineReadTimeout = + PR_MillisecondsToInterval((uint16_t) clamped(val, 5000, + 0xffff)); + } + } + + if (PREF_CHANGED(HTTP_PREF("pipelining.ssl"))) { + rv = prefs->GetBoolPref(HTTP_PREF("pipelining.ssl"), &cVar); + if (NS_SUCCEEDED(rv)) + mPipeliningOverSSL = cVar; + } + + if (PREF_CHANGED(HTTP_PREF("proxy.pipelining"))) { + rv = prefs->GetBoolPref(HTTP_PREF("proxy.pipelining"), &cVar); + if (NS_SUCCEEDED(rv)) + mProxyPipelining = cVar; + } + if (PREF_CHANGED(HTTP_PREF("qos"))) { rv = prefs->GetIntPref(HTTP_PREF("qos"), &val); if (NS_SUCCEEDED(rv)) @@ -1474,6 +1580,37 @@ nsHttpHandler::PrefsChanged(nsIPrefBranch *prefs, const char *pref) } } + // + // Test HTTP Pipelining (bug796192) + // If experiments are allowed and pipelining is Off, + // turn it On for just 10 minutes + // + if (mAllowExperiments && !mPipeliningEnabled && + PREF_CHANGED(HTTP_PREF("pipelining.abtest"))) { + rv = prefs->GetBoolPref(HTTP_PREF("pipelining.abtest"), &cVar); + if (NS_SUCCEEDED(rv)) { + // If option is enabled, only test for ~1% of sessions + if (cVar && !(rand() % 128)) { + mCapabilities |= NS_HTTP_ALLOW_PIPELINING; + if (mPipelineTestTimer) + mPipelineTestTimer->Cancel(); + mPipelineTestTimer = + do_CreateInstance("@mozilla.org/timer;1", &rv); + if (NS_SUCCEEDED(rv)) { + rv = mPipelineTestTimer->InitWithFuncCallback( + TimerCallback, this, 10*60*1000, // 10 minutes + nsITimer::TYPE_ONE_SHOT); + } + } else { + mCapabilities &= ~NS_HTTP_ALLOW_PIPELINING; + if (mPipelineTestTimer) { + mPipelineTestTimer->Cancel(); + mPipelineTestTimer = nullptr; + } + } + } + } + if (PREF_CHANGED(HTTP_PREF("pacing.requests.enabled"))) { rv = prefs->GetBoolPref(HTTP_PREF("pacing.requests.enabled"), &cVar); if (NS_SUCCEEDED(rv)) { @@ -1599,6 +1736,17 @@ nsHttpHandler::PrefsChanged(nsIPrefBranch *prefs, const char *pref) } +/** + * Static method called by mPipelineTestTimer when it expires. + */ +void +nsHttpHandler::TimerCallback(nsITimer * aTimer, void * aClosure) +{ + RefPtr thisObject = static_cast(aClosure); + if (!thisObject->mPipeliningEnabled) + thisObject->mCapabilities &= ~NS_HTTP_ALLOW_PIPELINING; +} + /** * Currently, only regularizes the case of subtags. */ @@ -1900,6 +2048,12 @@ nsHttpHandler::NewProxiedChannel2(nsIURI *uri, uint32_t caps = mCapabilities; + if (https) { + // enable pipelining over SSL if requested + if (mPipeliningOverSSL) + caps |= NS_HTTP_ALLOW_PIPELINING; + } + if (!IsNeckoChild()) { // HACK: make sure PSM gets initialized on the main thread. net_EnsurePSMInit(); @@ -2037,12 +2191,11 @@ nsHttpHandler::Observe(nsISupports *subject, mConnMgr->DoShiftReloadConnectionCleanup(nullptr); mConnMgr->PruneDeadConnections(); } -#if 0 } else if (!strcmp(topic, "net:failed-to-process-uri-content")) { - // nop right now - we used to cancel h1 pipelines based on this, - // but those are no longer implemented - nsCOMPtr uri = do_QueryInterface(subject); -#endif + nsCOMPtr uri = do_QueryInterface(subject); + if (uri && mConnMgr) { + mConnMgr->ReportFailedToProcess(uri); + } } else if (!strcmp(topic, "last-pb-context-exited")) { mPrivateAuthCache.ClearAll(); if (mConnMgr) { diff --git a/netwerk/protocol/http/nsHttpHandler.h b/netwerk/protocol/http/nsHttpHandler.h index 5d27f6dd5a0e..e6629ba06f27 100644 --- a/netwerk/protocol/http/nsHttpHandler.h +++ b/netwerk/protocol/http/nsHttpHandler.h @@ -28,6 +28,7 @@ class nsIRequestContextService; class nsISiteSecurityService; class nsIStreamConverterService; class nsIThrottlingService; +class nsITimer; class nsIUUIDGenerator; @@ -101,6 +102,7 @@ public: uint8_t GetQoSBits() { return mQoSBits; } uint16_t GetIdleSynTimeout() { return mIdleSynTimeout; } bool FastFallbackToIPv4() { return mFastFallbackToIPv4; } + bool ProxyPipelining() { return mProxyPipelining; } uint32_t MaxSocketCount(); bool EnforceAssocReq() { return mEnforceAssocReq; } @@ -317,6 +319,28 @@ public: static nsresult GenerateHostPort(const nsCString& host, int32_t port, nsACString& hostLine); + bool GetPipelineAggressive() { return mPipelineAggressive; } + void GetMaxPipelineObjectSize(int64_t *outVal) + { + *outVal = mMaxPipelineObjectSize; + } + + bool GetPipelineEnabled() + { + return mCapabilities & NS_HTTP_ALLOW_PIPELINING; + } + + bool GetPipelineRescheduleOnTimeout() + { + return mPipelineRescheduleOnTimeout; + } + + PRIntervalTime GetPipelineRescheduleTimeout() + { + return mPipelineRescheduleTimeout; + } + + PRIntervalTime GetPipelineTimeout() { return mPipelineReadTimeout; } SpdyInformation *SpdyInfo() { return &mSpdyInfo; } bool IsH2MandatorySuiteEnabled() { return mH2MandatorySuiteEnabled; } @@ -377,6 +401,7 @@ private: void NotifyObservers(nsIHttpChannel *chan, const char *event); + static void TimerCallback(nsITimer * aTimer, void * aClosure); private: // cached services @@ -408,6 +433,7 @@ private: uint8_t mReferrerXOriginPolicy; bool mFastFallbackToIPv4; + bool mProxyPipelining; PRIntervalTime mIdleTimeout; PRIntervalTime mSpdyTimeout; PRIntervalTime mResponseTimeout; @@ -418,9 +444,18 @@ private: uint16_t mIdleSynTimeout; bool mH2MandatorySuiteEnabled; + bool mPipeliningEnabled; uint16_t mMaxConnections; uint8_t mMaxPersistentConnectionsPerServer; uint8_t mMaxPersistentConnectionsPerProxy; + uint16_t mMaxPipelinedRequests; + uint16_t mMaxOptimisticPipelinedRequests; + bool mPipelineAggressive; + int64_t mMaxPipelineObjectSize; + bool mPipelineRescheduleOnTimeout; + PRIntervalTime mPipelineRescheduleTimeout; + PRIntervalTime mPipelineReadTimeout; + nsCOMPtr mPipelineTestTimer; uint8_t mRedirectionLimit; @@ -432,6 +467,7 @@ private: uint8_t mQoSBits; + bool mPipeliningOverSSL; bool mEnforceAssocReq; nsCString mAccept; diff --git a/netwerk/protocol/http/nsHttpPipeline.cpp b/netwerk/protocol/http/nsHttpPipeline.cpp new file mode 100644 index 000000000000..8789e96a5093 --- /dev/null +++ b/netwerk/protocol/http/nsHttpPipeline.cpp @@ -0,0 +1,912 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// HttpLog.h should generally be included first +#include "HttpLog.h" + +#include "nsHttpPipeline.h" +#include "nsHttpHandler.h" +#include "nsIOService.h" +#include "nsISocketTransport.h" +#include "nsIPipe.h" +#include "nsCOMPtr.h" +#include "nsSocketTransportService2.h" +#include + +#ifdef DEBUG +#include "prthread.h" +#endif + +namespace mozilla { +namespace net { + +//----------------------------------------------------------------------------- +// nsHttpPushBackWriter +//----------------------------------------------------------------------------- + +class nsHttpPushBackWriter : public nsAHttpSegmentWriter +{ +public: + nsHttpPushBackWriter(const char *buf, uint32_t bufLen) + : mBuf(buf) + , mBufLen(bufLen) + { } + virtual ~nsHttpPushBackWriter() {} + + nsresult OnWriteSegment(char *buf, uint32_t count, uint32_t *countWritten) + { + if (mBufLen == 0) + return NS_BASE_STREAM_CLOSED; + + if (count > mBufLen) + count = mBufLen; + + memcpy(buf, mBuf, count); + + mBuf += count; + mBufLen -= count; + *countWritten = count; + return NS_OK; + } + +private: + const char *mBuf; + uint32_t mBufLen; +}; + +//----------------------------------------------------------------------------- +// nsHttpPipeline +//----------------------------------------------------------------------------- + +nsHttpPipeline::nsHttpPipeline() + : mStatus(NS_OK) + , mRequestIsPartial(false) + , mResponseIsPartial(false) + , mClosed(false) + , mUtilizedPipeline(false) + , mPushBackBuf(nullptr) + , mPushBackLen(0) + , mPushBackMax(0) + , mHttp1xTransactionCount(0) + , mReceivingFromProgress(0) + , mSendingToProgress(0) + , mSuppressSendEvents(true) +{ +} + +nsHttpPipeline::~nsHttpPipeline() +{ + // make sure we aren't still holding onto any transactions! + Close(NS_ERROR_ABORT); + + if (mPushBackBuf) + free(mPushBackBuf); +} + +nsresult +nsHttpPipeline::AddTransaction(nsAHttpTransaction *trans) +{ + LOG(("nsHttpPipeline::AddTransaction [this=%p trans=%p]\n", this, trans)); + + if (mRequestQ.Length() || mResponseQ.Length()) + mUtilizedPipeline = true; + + // A reference to the actual transaction is held by the pipeline transaction + // in either the request or response queue + mRequestQ.AppendElement(trans); + uint32_t qlen = PipelineDepth(); + + if (qlen != 1) { + trans->SetPipelinePosition(qlen); + } + else { + // do it for this case in case an idempotent cancellation + // is being repeated and an old value needs to be cleared + trans->SetPipelinePosition(0); + } + + // trans->SetConnection() needs to be updated to point back at + // the pipeline object. + trans->SetConnection(this); + + if (mConnection && !mClosed && mRequestQ.Length() == 1) + mConnection->ResumeSend(); + + return NS_OK; +} + +uint32_t +nsHttpPipeline::PipelineDepth() +{ + return mRequestQ.Length() + mResponseQ.Length(); +} + +nsresult +nsHttpPipeline::SetPipelinePosition(int32_t position) +{ + nsAHttpTransaction *trans = Response(0); + if (trans) + return trans->SetPipelinePosition(position); + return NS_OK; +} + +int32_t +nsHttpPipeline::PipelinePosition() +{ + nsAHttpTransaction *trans = Response(0); + if (trans) + return trans->PipelinePosition(); + + // The response queue is empty, so return oldest request + if (mRequestQ.Length()) + return Request(mRequestQ.Length() - 1)->PipelinePosition(); + + // No transactions in the pipeline + return 0; +} + +nsHttpPipeline * +nsHttpPipeline::QueryPipeline() +{ + return this; +} + +//----------------------------------------------------------------------------- +// nsHttpPipeline::nsISupports +//----------------------------------------------------------------------------- + +NS_IMPL_ADDREF(nsHttpPipeline) +NS_IMPL_RELEASE(nsHttpPipeline) + +// multiple inheritance fun :-) +NS_INTERFACE_MAP_BEGIN(nsHttpPipeline) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsAHttpConnection) +NS_INTERFACE_MAP_END + + +//----------------------------------------------------------------------------- +// nsHttpPipeline::nsAHttpConnection +//----------------------------------------------------------------------------- + +nsresult +nsHttpPipeline::OnHeadersAvailable(nsAHttpTransaction *trans, + nsHttpRequestHead *requestHead, + nsHttpResponseHead *responseHead, + bool *reset) +{ + LOG(("nsHttpPipeline::OnHeadersAvailable [this=%p]\n", this)); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + MOZ_ASSERT(mConnection, "no connection"); + + RefPtr ci; + GetConnectionInfo(getter_AddRefs(ci)); + MOZ_ASSERT(ci); + + if (!ci) { + return NS_ERROR_UNEXPECTED; + } + + bool pipeliningBefore = gHttpHandler->ConnMgr()->SupportsPipelining(ci); + + // trans has now received its response headers; forward to the real connection + nsresult rv = mConnection->OnHeadersAvailable(trans, + requestHead, + responseHead, + reset); + + if (!pipeliningBefore && gHttpHandler->ConnMgr()->SupportsPipelining(ci)) { + // The received headers have expanded the eligible + // pipeline depth for this connection + gHttpHandler->ConnMgr()->ProcessPendingQForEntry(ci); + } + + return rv; +} + +void +nsHttpPipeline::CloseTransaction(nsAHttpTransaction *aTrans, nsresult reason) +{ + LOG(("nsHttpPipeline::CloseTransaction [this=%p trans=%p reason=%" PRIx32 "]\n", + this, aTrans, static_cast(reason))); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + MOZ_ASSERT(NS_FAILED(reason), "expecting failure code"); + + // the specified transaction is to be closed with the given "reason" + RefPtr trans(aTrans); + int32_t index; + bool killPipeline = false; + + if ((index = mRequestQ.IndexOf(trans)) >= 0) { + if (index == 0 && mRequestIsPartial) { + // the transaction is in the request queue. check to see if any of + // its data has been written out yet. + killPipeline = true; + } + mRequestQ.RemoveElementAt(index); + } else if ((index = mResponseQ.IndexOf(trans)) >= 0) { + mResponseQ.RemoveElementAt(index); + // while we could avoid killing the pipeline if this transaction is the + // last transaction in the pipeline, there doesn't seem to be that much + // value in doing so. most likely if this transaction is going away, + // the others will be shortly as well. + killPipeline = true; + } + + // Marking this connection as non-reusable prevents other items from being + // added to it and causes it to be torn down soon. + DontReuse(); + + trans->Close(reason); + trans = nullptr; + + if (killPipeline) { + // reschedule anything from this pipeline onto a different connection + CancelPipeline(reason); + } + + // If all the transactions have been removed then we can close the connection + // right away. + if (!mRequestQ.Length() && !mResponseQ.Length() && mConnection) + mConnection->CloseTransaction(this, reason); +} + +nsresult +nsHttpPipeline::TakeTransport(nsISocketTransport **aTransport, + nsIAsyncInputStream **aInputStream, + nsIAsyncOutputStream **aOutputStream) +{ + return mConnection->TakeTransport(aTransport, aInputStream, aOutputStream); +} + +bool +nsHttpPipeline::IsPersistent() +{ + return true; // pipelining requires this +} + +bool +nsHttpPipeline::IsReused() +{ + if (!mUtilizedPipeline && mConnection) + return mConnection->IsReused(); + return true; +} + +void +nsHttpPipeline::DontReuse() +{ + if (mConnection) + mConnection->DontReuse(); +} + +nsresult +nsHttpPipeline::PushBack(const char *data, uint32_t length) +{ + LOG(("nsHttpPipeline::PushBack [this=%p len=%u]\n", this, length)); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + MOZ_ASSERT(mPushBackLen == 0, "push back buffer already has data!"); + + // If we have no chance for a pipeline (e.g. due to an Upgrade) + // then push this data down to original connection + if (!mConnection->IsPersistent()) + return mConnection->PushBack(data, length); + + // PushBack is called recursively from WriteSegments + + // XXX we have a design decision to make here. either we buffer the data + // and process it when we return to WriteSegments, or we attempt to move + // onto the next transaction from here. doing so adds complexity with the + // benefit of eliminating the extra buffer copy. the buffer is at most + // 4096 bytes, so it is really unclear if there is any value in the added + // complexity. besides simplicity, buffering this data has the advantage + // that we'll call close on the transaction sooner, which will wake up + // the HTTP channel sooner to continue with its work. + + if (!mPushBackBuf) { + mPushBackMax = length; + mPushBackBuf = (char *) malloc(mPushBackMax); + if (!mPushBackBuf) + return NS_ERROR_OUT_OF_MEMORY; + } + else if (length > mPushBackMax) { + // grow push back buffer as necessary. + MOZ_ASSERT(length <= nsIOService::gDefaultSegmentSize, "too big"); + mPushBackMax = length; + mPushBackBuf = (char *) realloc(mPushBackBuf, mPushBackMax); + if (!mPushBackBuf) + return NS_ERROR_OUT_OF_MEMORY; + } + + memcpy(mPushBackBuf, data, length); + mPushBackLen = length; + + return NS_OK; +} + +already_AddRefed +nsHttpPipeline::TakeHttpConnection() +{ + if (mConnection) + return mConnection->TakeHttpConnection(); + return nullptr; +} + +nsAHttpTransaction::Classifier +nsHttpPipeline::Classification() +{ + if (mConnection) + return mConnection->Classification(); + + LOG(("nsHttpPipeline::Classification this=%p " + "has null mConnection using CLASS_SOLO default", this)); + return nsAHttpTransaction::CLASS_SOLO; +} + +void +nsHttpPipeline::SetProxyConnectFailed() +{ + nsAHttpTransaction *trans = Request(0); + + if (trans) + trans->SetProxyConnectFailed(); +} + +nsHttpRequestHead * +nsHttpPipeline::RequestHead() +{ + nsAHttpTransaction *trans = Request(0); + + if (trans) + return trans->RequestHead(); + return nullptr; +} + +uint32_t +nsHttpPipeline::Http1xTransactionCount() +{ + return mHttp1xTransactionCount; +} + +nsresult +nsHttpPipeline::TakeSubTransactions( + nsTArray > &outTransactions) +{ + LOG(("nsHttpPipeline::TakeSubTransactions [this=%p]\n", this)); + + if (mResponseQ.Length() || mRequestIsPartial) + return NS_ERROR_ALREADY_OPENED; + + int32_t i, count = mRequestQ.Length(); + for (i = 0; i < count; ++i) { + nsAHttpTransaction *trans = Request(i); + // set the transaction connection object back to the underlying + // nsHttpConnectionHandle + trans->SetConnection(mConnection); + outTransactions.AppendElement(trans); + } + mRequestQ.Clear(); + + LOG((" took %d\n", count)); + return NS_OK; +} + +//----------------------------------------------------------------------------- +// nsHttpPipeline::nsAHttpTransaction +//----------------------------------------------------------------------------- + +void +nsHttpPipeline::SetConnection(nsAHttpConnection *conn) +{ + LOG(("nsHttpPipeline::SetConnection [this=%p conn=%p]\n", this, conn)); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + MOZ_ASSERT(!conn || !mConnection, "already have a connection"); + + mConnection = conn; +} + +nsAHttpConnection * +nsHttpPipeline::Connection() +{ + LOG(("nsHttpPipeline::Connection [this=%p conn=%p]\n", this, mConnection.get())); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + return mConnection; +} + +void +nsHttpPipeline::GetSecurityCallbacks(nsIInterfaceRequestor **result) +{ + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + // depending on timing this could be either the request or the response + // that is needed - but they both go to the same host. A request for these + // callbacks directly in nsHttpTransaction would not make a distinction + // over whether the the request had been transmitted yet. + nsAHttpTransaction *trans = Request(0); + if (!trans) + trans = Response(0); + if (trans) + trans->GetSecurityCallbacks(result); + else { + *result = nullptr; + } +} + +void +nsHttpPipeline::OnTransportStatus(nsITransport* transport, + nsresult status, int64_t progress) +{ + LOG(("nsHttpPipeline::OnStatus [this=%p status=%" PRIx32 " progress=%" PRId64 "]\n", + this, static_cast(status), progress)); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + nsAHttpTransaction *trans; + int32_t i, count; + + switch (status) { + + case NS_NET_STATUS_RESOLVING_HOST: + case NS_NET_STATUS_RESOLVED_HOST: + case NS_NET_STATUS_CONNECTING_TO: + case NS_NET_STATUS_CONNECTED_TO: + // These should only appear at most once per pipeline. + // Deliver to the first transaction. + + trans = Request(0); + if (!trans) + trans = Response(0); + if (trans) + trans->OnTransportStatus(transport, status, progress); + + break; + + case NS_NET_STATUS_SENDING_TO: + // This is generated by the socket transport when (part) of + // a transaction is written out + // + // In pipelining this is generated out of FillSendBuf(), but it cannot do + // so until the connection is confirmed by CONNECTED_TO. + // See patch for bug 196827. + // + + if (mSuppressSendEvents) { + mSuppressSendEvents = false; + + // catch up by sending the event to all the transactions that have + // moved from request to response and any that have been partially + // sent. Also send WAITING_FOR to those that were completely sent + count = mResponseQ.Length(); + for (i = 0; i < count; ++i) { + Response(i)->OnTransportStatus(transport, + NS_NET_STATUS_SENDING_TO, + progress); + Response(i)->OnTransportStatus(transport, + NS_NET_STATUS_WAITING_FOR, + progress); + } + if (mRequestIsPartial && Request(0)) + Request(0)->OnTransportStatus(transport, + NS_NET_STATUS_SENDING_TO, + progress); + mSendingToProgress = progress; + } + // otherwise ignore it + break; + + case NS_NET_STATUS_WAITING_FOR: + // Created by nsHttpConnection when request pipeline has been totally + // sent. Ignore it here because it is simulated in FillSendBuf() when + // a request is moved from request to response. + + // ignore it + break; + + case NS_NET_STATUS_RECEIVING_FROM: + // Forward this only to the transaction currently recieving data. It is + // normally generated by the socket transport, but can also + // be repeated by the pushbackwriter if necessary. + mReceivingFromProgress = progress; + if (Response(0)) + Response(0)->OnTransportStatus(transport, status, progress); + break; + + default: + // forward other notifications to all request transactions + count = mRequestQ.Length(); + for (i = 0; i < count; ++i) + Request(i)->OnTransportStatus(transport, status, progress); + break; + } +} + +nsHttpConnectionInfo * +nsHttpPipeline::ConnectionInfo() +{ + nsAHttpTransaction *trans = Request(0) ? Request(0) : Response(0); + if (!trans) { + return nullptr; + } + return trans->ConnectionInfo(); +} + +bool +nsHttpPipeline::IsDone() +{ + bool done = true; + + uint32_t i, count = mRequestQ.Length(); + for (i = 0; done && (i < count); i++) + done = Request(i)->IsDone(); + + count = mResponseQ.Length(); + for (i = 0; done && (i < count); i++) + done = Response(i)->IsDone(); + + return done; +} + +nsresult +nsHttpPipeline::Status() +{ + return mStatus; +} + +uint32_t +nsHttpPipeline::Caps() +{ + nsAHttpTransaction *trans = Request(0); + if (!trans) + trans = Response(0); + + return trans ? trans->Caps() : 0; +} + +void +nsHttpPipeline::SetDNSWasRefreshed() +{ + nsAHttpTransaction *trans = Request(0); + if (!trans) + trans = Response(0); + + if (trans) + trans->SetDNSWasRefreshed(); +} + +uint64_t +nsHttpPipeline::Available() +{ + uint64_t result = 0; + + int32_t i, count = mRequestQ.Length(); + for (i=0; iAvailable(); + return result; +} + +nsresult +nsHttpPipeline::ReadFromPipe(nsIInputStream *stream, + void *closure, + const char *buf, + uint32_t offset, + uint32_t count, + uint32_t *countRead) +{ + nsHttpPipeline *self = (nsHttpPipeline *) closure; + return self->mReader->OnReadSegment(buf, count, countRead); +} + +nsresult +nsHttpPipeline::ReadSegments(nsAHttpSegmentReader *reader, + uint32_t count, + uint32_t *countRead) +{ + LOG(("nsHttpPipeline::ReadSegments [this=%p count=%u]\n", this, count)); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + if (mClosed) { + *countRead = 0; + return mStatus; + } + + nsresult rv; + uint64_t avail = 0; + if (mSendBufIn) { + rv = mSendBufIn->Available(&avail); + if (NS_FAILED(rv)) return rv; + } + + if (avail == 0) { + rv = FillSendBuf(); + if (NS_FAILED(rv)) return rv; + + rv = mSendBufIn->Available(&avail); + if (NS_FAILED(rv)) return rv; + + // return EOF if send buffer is empty + if (avail == 0) { + *countRead = 0; + return NS_OK; + } + } + + // read no more than what was requested + if (avail > count) + avail = count; + + mReader = reader; + + // avail is under 4GB, so casting to uint32_t is safe + rv = mSendBufIn->ReadSegments(ReadFromPipe, this, (uint32_t)avail, countRead); + + mReader = nullptr; + return rv; +} + +nsresult +nsHttpPipeline::WriteSegments(nsAHttpSegmentWriter *writer, + uint32_t count, + uint32_t *countWritten) +{ + LOG(("nsHttpPipeline::WriteSegments [this=%p count=%u]\n", this, count)); + + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + if (mClosed) + return NS_SUCCEEDED(mStatus) ? NS_BASE_STREAM_CLOSED : mStatus; + + nsAHttpTransaction *trans; + nsresult rv; + + trans = Response(0); + // This code deals with the establishment of a CONNECT tunnel through + // an HTTP proxy. It allows the connection to do the CONNECT/200 + // HTTP transaction to establish a tunnel as a precursor to the + // actual pipeline of regular HTTP transactions. + if (!trans && mRequestQ.Length() && + mConnection->IsProxyConnectInProgress()) { + LOG(("nsHttpPipeline::WriteSegments [this=%p] Forced Delegation\n", + this)); + trans = Request(0); + } + + if (!trans) { + if (mRequestQ.Length() > 0) + rv = NS_BASE_STREAM_WOULD_BLOCK; + else + rv = NS_BASE_STREAM_CLOSED; + } else { + // + // ask the transaction to consume data from the connection. + // PushBack may be called recursively. + // + rv = trans->WriteSegments(writer, count, countWritten); + + if (rv == NS_BASE_STREAM_CLOSED || trans->IsDone()) { + trans->Close(NS_OK); + + // Release the transaction if it is not IsProxyConnectInProgress() + if (trans == Response(0)) { + mResponseQ.RemoveElementAt(0); + mResponseIsPartial = false; + ++mHttp1xTransactionCount; + } + + // ask the connection manager to add additional transactions + // to our pipeline. + RefPtr ci; + GetConnectionInfo(getter_AddRefs(ci)); + if (ci) + gHttpHandler->ConnMgr()->ProcessPendingQForEntry(ci); + } + else + mResponseIsPartial = true; + } + + if (mPushBackLen) { + nsHttpPushBackWriter pushBackWriter(mPushBackBuf, mPushBackLen); + uint32_t len = mPushBackLen, n; + mPushBackLen = 0; + + // This progress notification has previously been sent from + // the socket transport code, but it was delivered to the + // previous transaction on the pipeline. + nsITransport *transport = Transport(); + if (transport) + OnTransportStatus(transport, NS_NET_STATUS_RECEIVING_FROM, + mReceivingFromProgress); + + // the push back buffer is never larger than NS_HTTP_SEGMENT_SIZE, + // so we are guaranteed that the next response will eat the entire + // push back buffer (even though it might again call PushBack). + rv = WriteSegments(&pushBackWriter, len, &n); + } + + return rv; +} + +uint32_t +nsHttpPipeline::CancelPipeline(nsresult originalReason) +{ + uint32_t i, reqLen, respLen, total; + nsAHttpTransaction *trans; + + reqLen = mRequestQ.Length(); + respLen = mResponseQ.Length(); + total = reqLen + respLen; + + // don't count the first response, if presnet + if (respLen) + total--; + + if (!total) + return 0; + + // any pending requests can ignore this error and be restarted + // unless it is during a CONNECT tunnel request + for (i = 0; i < reqLen; ++i) { + trans = Request(i); + if (mConnection && mConnection->IsProxyConnectInProgress()) + trans->Close(originalReason); + else + trans->Close(NS_ERROR_NET_RESET); + } + mRequestQ.Clear(); + + // any pending responses can be restarted except for the first one, + // that we might want to finish on this pipeline or cancel individually. + // Higher levels of callers ensure that we don't process non-idempotent + // tranasction with the NS_HTTP_ALLOW_PIPELINING bit set + for (i = 1; i < respLen; ++i) { + trans = Response(i); + trans->Close(NS_ERROR_NET_RESET); + } + + if (respLen > 1) + mResponseQ.TruncateLength(1); + + DontReuse(); + Classify(nsAHttpTransaction::CLASS_SOLO); + + return total; +} + +void +nsHttpPipeline::Close(nsresult reason) +{ + LOG(("nsHttpPipeline::Close [this=%p reason=%" PRIx32 "]\n", + this, static_cast(reason))); + + if (mClosed) { + LOG((" already closed\n")); + return; + } + + // the connection is going away! + mStatus = reason; + mClosed = true; + + RefPtr ci; + GetConnectionInfo(getter_AddRefs(ci)); + uint32_t numRescheduled = CancelPipeline(reason); + + // numRescheduled can be 0 if there is just a single response in the + // pipeline object. That isn't really a meaningful pipeline that + // has been forced to be rescheduled so it does not need to generate + // negative feedback. + if (ci && numRescheduled) + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + ci, nsHttpConnectionMgr::RedCanceledPipeline, nullptr, 0); + + nsAHttpTransaction *trans = Response(0); + if (!trans) + return; + + // The current transaction can be restarted via reset + // if the response has not started to arrive and the reason + // for failure is innocuous (e.g. not an SSL error) + if (!mResponseIsPartial && + (reason == NS_ERROR_NET_RESET || + reason == NS_OK || + reason == NS_ERROR_NET_TIMEOUT || + reason == NS_BASE_STREAM_CLOSED)) { + trans->Close(NS_ERROR_NET_RESET); + } + else { + trans->Close(reason); + } + + mResponseQ.Clear(); +} + +nsresult +nsHttpPipeline::OnReadSegment(const char *segment, + uint32_t count, + uint32_t *countRead) +{ + return mSendBufOut->Write(segment, count, countRead); +} + +nsresult +nsHttpPipeline::FillSendBuf() +{ + // reads from request queue, moving transactions to response queue + // when they have been completely read. + + nsresult rv; + + if (!mSendBufIn) { + // allocate a single-segment pipe + rv = NS_NewPipe(getter_AddRefs(mSendBufIn), + getter_AddRefs(mSendBufOut), + nsIOService::gDefaultSegmentSize, /* segment size */ + nsIOService::gDefaultSegmentSize, /* max size */ + true, true); + if (NS_FAILED(rv)) return rv; + } + + uint32_t n; + uint64_t avail; + RefPtr trans; + nsITransport *transport = Transport(); + + while ((trans = Request(0)) != nullptr) { + avail = trans->Available(); + if (avail) { + // if there is already a response in the responseq then this + // new data comprises a pipeline. Update the transaction in the + // response queue to reflect that if necessary. We are now sending + // out a request while we haven't received all responses. + nsAHttpTransaction *response = Response(0); + if (response && !response->PipelinePosition()) + response->SetPipelinePosition(1); + rv = trans->ReadSegments(this, (uint32_t)std::min(avail, (uint64_t)UINT32_MAX), &n); + if (NS_FAILED(rv)) return rv; + + if (n == 0) { + LOG(("send pipe is full")); + break; + } + + mSendingToProgress += n; + if (!mSuppressSendEvents && transport) { + // Simulate a SENDING_TO event + trans->OnTransportStatus(transport, + NS_NET_STATUS_SENDING_TO, + mSendingToProgress); + } + } + + avail = trans->Available(); + if (avail == 0) { + // move transaction from request queue to response queue + mRequestQ.RemoveElementAt(0); + mResponseQ.AppendElement(trans); + mRequestIsPartial = false; + + if (!mSuppressSendEvents && transport) { + // Simulate a WAITING_FOR event + trans->OnTransportStatus(transport, + NS_NET_STATUS_WAITING_FOR, + mSendingToProgress); + } + + // It would be good to re-enable data read handlers via ResumeRecv() + // except the read handler code can be synchronously dispatched on + // the stack. + } + else + mRequestIsPartial = true; + } + return NS_OK; +} + +} // namespace net +} // namespace mozilla diff --git a/netwerk/protocol/http/nsHttpPipeline.h b/netwerk/protocol/http/nsHttpPipeline.h new file mode 100644 index 000000000000..6dc027f19bc6 --- /dev/null +++ b/netwerk/protocol/http/nsHttpPipeline.h @@ -0,0 +1,105 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsHttpPipeline_h__ +#define nsHttpPipeline_h__ + +#include "nsAHttpConnection.h" +#include "nsAHttpTransaction.h" +#include "nsTArray.h" +#include "nsCOMPtr.h" + +class nsIInputStream; +class nsIOutputStream; + +namespace mozilla { namespace net { + +class nsHttpPipeline final : public nsAHttpConnection + , public nsAHttpTransaction + , public nsAHttpSegmentReader +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSAHTTPCONNECTION(mConnection) + NS_DECL_NSAHTTPTRANSACTION + NS_DECL_NSAHTTPSEGMENTREADER + + nsHttpPipeline(); + + bool ResponseTimeoutEnabled() const override final { + return true; + } + +private: + virtual ~nsHttpPipeline(); + + nsresult FillSendBuf(); + + static nsresult ReadFromPipe(nsIInputStream *, void *, const char *, + uint32_t, uint32_t, uint32_t *); + + // convenience functions + nsAHttpTransaction *Request(int32_t i) + { + if (mRequestQ.Length() == 0) + return nullptr; + + return mRequestQ[i]; + } + nsAHttpTransaction *Response(int32_t i) + { + if (mResponseQ.Length() == 0) + return nullptr; + + return mResponseQ[i]; + } + + // overload of nsAHttpTransaction::QueryPipeline() + nsHttpPipeline *QueryPipeline() override; + + RefPtr mConnection; + nsTArray > mRequestQ; + nsTArray > mResponseQ; + nsresult mStatus; + + // these flags indicate whether or not the first request or response + // is partial. a partial request means that Request(0) has been + // partially written out to the socket. a partial response means + // that Response(0) has been partially read in from the socket. + bool mRequestIsPartial; + bool mResponseIsPartial; + + // indicates whether or not the pipeline has been explicitly closed. + bool mClosed; + + // indicates whether or not a true pipeline (more than 1 request without + // a synchronous response) has been formed. + bool mUtilizedPipeline; + + // used when calling ReadSegments/WriteSegments on a transaction. + nsAHttpSegmentReader *mReader; + + // send buffer + nsCOMPtr mSendBufIn; + nsCOMPtr mSendBufOut; + + // the push back buffer. not exceeding nsIOService::gDefaultSegmentSize bytes. + char *mPushBackBuf; + uint32_t mPushBackLen; + uint32_t mPushBackMax; + + // The number of transactions completed on this pipeline. + uint32_t mHttp1xTransactionCount; + + // For support of OnTransportStatus() + int64_t mReceivingFromProgress; + int64_t mSendingToProgress; + bool mSuppressSendEvents; +}; + +} // namespace net +} // namespace mozilla + +#endif // nsHttpPipeline_h__ diff --git a/netwerk/protocol/http/nsHttpTransaction.cpp b/netwerk/protocol/http/nsHttpTransaction.cpp index 7138974481e1..bc15e8376e08 100644 --- a/netwerk/protocol/http/nsHttpTransaction.cpp +++ b/netwerk/protocol/http/nsHttpTransaction.cpp @@ -104,6 +104,8 @@ nsHttpTransaction::nsHttpTransaction() , mPriority(0) , mRestartCount(0) , mCaps(0) + , mClassification(CLASS_GENERAL) + , mPipelinePosition(0) , mHttpVersion(NS_HTTP_VERSION_UNKNOWN) , mHttpResponseCode(0) , mCurrentHttpResponseHeaderSize(0) @@ -143,6 +145,7 @@ nsHttpTransaction::nsHttpTransaction() , mTransportStatus(NS_OK) { LOG(("Creating nsHttpTransaction @%p\n", this)); + gHttpHandler->GetMaxPipelineObjectSize(&mMaxPipelineObjectSize); #ifdef MOZ_VALGRIND memset(&mSelfAddr, 0, sizeof(NetAddr)); @@ -178,6 +181,45 @@ nsHttpTransaction::~nsHttpTransaction() ReleaseBlockingTransaction(); } +nsHttpTransaction::Classifier +nsHttpTransaction::Classify() +{ + if (!(mCaps & NS_HTTP_ALLOW_PIPELINING)) + return (mClassification = CLASS_SOLO); + + if (mRequestHead->HasHeader(nsHttp::If_Modified_Since) || + mRequestHead->HasHeader(nsHttp::If_None_Match)) + return (mClassification = CLASS_REVALIDATION); + + nsAutoCString accept; + bool hasAccept = NS_SUCCEEDED(mRequestHead->GetHeader(nsHttp::Accept, accept)); + if (hasAccept && StringBeginsWith(accept, NS_LITERAL_CSTRING("image/"))) { + return (mClassification = CLASS_IMAGE); + } + + if (hasAccept && StringBeginsWith(accept, NS_LITERAL_CSTRING("text/css"))) { + return (mClassification = CLASS_SCRIPT); + } + + mClassification = CLASS_GENERAL; + + nsAutoCString requestURI; + mRequestHead->RequestURI(requestURI); + int32_t queryPos = requestURI.FindChar('?'); + if (queryPos == kNotFound) { + if (StringEndsWith(requestURI, + NS_LITERAL_CSTRING(".js"))) + mClassification = CLASS_SCRIPT; + } + else if (queryPos >= 3 && + Substring(requestURI, queryPos - 3, 3). + EqualsLiteral(".js")) { + mClassification = CLASS_SCRIPT; + } + + return mClassification; +} + nsresult nsHttpTransaction::Init(uint32_t caps, nsHttpConnectionInfo *cinfo, @@ -373,6 +415,8 @@ nsHttpTransaction::Init(uint32_t caps, MOZ_DIAGNOSTIC_ASSERT(*vtable != 0); #endif // WIN32 + Classify(); + nsCOMPtr tmp(mPipeIn); tmp.forget(responseBody); return NS_OK; @@ -399,7 +443,7 @@ nsHttpTransaction::TakeResponseHead() { MOZ_ASSERT(!mResponseHeadTaken, "TakeResponseHead called 2x"); - // Lock TakeResponseHead() against main thread + // Lock RestartInProgress() and TakeResponseHead() against main thread MutexAutoLock lock(*nsHttp::GetLock()); mResponseHeadTaken = true; @@ -582,15 +626,16 @@ nsHttpTransaction::OnTransportStatus(nsITransport* transport, PR_Now(), 0, EmptyCString()); // report the status and progress - mActivityDistributor->ObserveActivity( - mChannel, - NS_HTTP_ACTIVITY_TYPE_SOCKET_TRANSPORT, - static_cast(status), - PR_Now(), - progress, - EmptyCString()); + if (!mRestartInProgressVerifier.IsDiscardingContent()) + mActivityDistributor->ObserveActivity( + mChannel, + NS_HTTP_ACTIVITY_TYPE_SOCKET_TRANSPORT, + static_cast(status), + PR_Now(), + progress, + EmptyCString()); } - + // nsHttpChannel synthesizes progress events in OnDataAvailable if (status == NS_NET_STATUS_RECEIVING_FROM) return; @@ -984,9 +1029,24 @@ nsHttpTransaction::Close(nsresult reason) // if restarting fails, then we must proceed to close the pipe, // which will notify the channel that the transaction failed. + if (mPipelinePosition) { + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::RedCanceledPipeline, + nullptr, 0); + } if (NS_SUCCEEDED(Restart())) return; } + else if (!mResponseIsComplete && mPipelinePosition && + reason == NS_ERROR_NET_RESET) { + // due to unhandled rst on a pipeline - safe to + // restart as only idempotent is found there + + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::RedCorruptedContent, nullptr, 0); + if (NS_SUCCEEDED(RestartInProgress())) + return; + } } if ((mChunkedDecoder || (mContentLength >= int64_t(0))) && @@ -1017,6 +1077,20 @@ nsHttpTransaction::Close(nsresult reason) bool relConn = true; if (NS_SUCCEEDED(reason)) { + if (!mResponseIsComplete) { + // The response has not been delimited with a high-confidence + // algorithm like Content-Length or Chunked Encoding. We + // need to use a strong framing mechanism to pipeline. + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::BadInsufficientFraming, + nullptr, mClassification); + } + else if (mPipelinePosition) { + // report this success as feedback + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::GoodCompletedOK, + nullptr, mPipelinePosition); + } // the server has not sent the final \r\n terminating the header // section, and there may still be a header line unparsed. let's make @@ -1085,6 +1159,31 @@ nsHttpTransaction::ConnectionInfo() return mConnInfo.get(); } +nsresult +nsHttpTransaction::AddTransaction(nsAHttpTransaction *trans) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +uint32_t +nsHttpTransaction::PipelineDepth() +{ + return IsDone() ? 0 : 1; +} + +nsresult +nsHttpTransaction::SetPipelinePosition(int32_t position) +{ + mPipelinePosition = position; + return NS_OK; +} + +int32_t +nsHttpTransaction::PipelinePosition() +{ + return mPipelinePosition; +} + bool // NOTE BASE CLASS nsAHttpTransaction::ResponseTimeoutEnabled() const { @@ -1107,6 +1206,71 @@ nsHttpTransaction::ResponseTimeoutEnabled() const // nsHttpTransaction //----------------------------------------------------------------------------- +nsresult +nsHttpTransaction::RestartInProgress() +{ + MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread); + + if ((mRestartCount + 1) >= gHttpHandler->MaxRequestAttempts()) { + LOG(("nsHttpTransaction::RestartInProgress() " + "reached max request attempts, failing transaction %p\n", this)); + return NS_ERROR_NET_RESET; + } + + // Lock RestartInProgress() and TakeResponseHead() against main thread + MutexAutoLock lock(*nsHttp::GetLock()); + + // Don't try and RestartInProgress() things that haven't gotten a response + // header yet. Those should be handled under the normal restart() path if + // they are eligible. + if (!mHaveAllHeaders) + return NS_ERROR_NET_RESET; + + if (mCaps & NS_HTTP_STICKY_CONNECTION) { + return NS_ERROR_NET_RESET; + } + + // don't try and restart 0.9 or non 200/Get HTTP/1 + if (!mRestartInProgressVerifier.IsSetup()) + return NS_ERROR_NET_RESET; + + LOG(("Will restart transaction %p and skip first %" PRId64 " bytes, " + "old Content-Length %" PRId64, + this, mContentRead, mContentLength)); + + mRestartInProgressVerifier.SetAlreadyProcessed( + std::max(mRestartInProgressVerifier.AlreadyProcessed(), mContentRead)); + + if (!mResponseHeadTaken && !mForTakeResponseHead) { + // TakeResponseHeader() has not been called yet and this + // is the first restart. Store the resp headers exclusively + // for TakeResponseHead() which is called from the main thread and + // could happen at any time - so we can't continue to modify those + // headers (which restarting will effectively do) + mForTakeResponseHead = mResponseHead; + mResponseHead = nullptr; + } + + if (mResponseHead) { + mResponseHead->Reset(); + } + + mContentRead = 0; + mContentLength = -1; + delete mChunkedDecoder; + mChunkedDecoder = nullptr; + mHaveStatusLine = false; + mHaveAllHeaders = false; + mHttpResponseMatched = false; + mResponseIsComplete = false; + mDidContentStart = false; + mNoContent = false; + mSentData = false; + mReceivedData = false; + + return Restart(); +} + nsresult nsHttpTransaction::Restart() { @@ -1140,6 +1304,12 @@ nsHttpTransaction::Restart() // to the next mReuseOnRestart = false; + // disable pipelining for the next attempt in case pipelining caused the + // reset. this is being overly cautious since we don't know if pipelining + // was the problem here. + mCaps &= ~NS_HTTP_ALLOW_PIPELINING; + SetPipelinePosition(0); + if (!mConnInfo->GetRoutedHost().IsEmpty()) { MutexAutoLock lock(*nsHttp::GetLock()); RefPtr ci; @@ -1269,6 +1439,9 @@ nsHttpTransaction::ParseLineSegment(char *segment, uint32_t len) nsresult rv = ParseLine(mLineBuf); mLineBuf.Truncate(); if (NS_FAILED(rv)) { + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::RedCorruptedContent, + nullptr, 0); return rv; } } @@ -1447,7 +1620,8 @@ nsHttpTransaction::HandleContentStart() // notify the connection, give it a chance to cause a reset. bool reset = false; - mConnection->OnHeadersAvailable(this, mRequestHead, mResponseHead, &reset); + if (!mRestartInProgressVerifier.IsSetup()) + mConnection->OnHeadersAvailable(this, mRequestHead, mResponseHead, &reset); // looks like we should ignore this response, resetting... if (reset) { @@ -1492,13 +1666,21 @@ nsHttpTransaction::HandleContentStart() mNoContent = true; } mConnection->SetLastTransactionExpectedNoContent(mNoContent); + if (mInvalidResponseBytesRead) + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, nsHttpConnectionMgr::BadInsufficientFraming, + nullptr, mClassification); - if (mNoContent) { + if (mNoContent) mContentLength = 0; - } else { + else { // grab the content-length from the response headers mContentLength = mResponseHead->ContentLength(); + if ((mClassification != CLASS_SOLO) && + (mContentLength > mMaxPipelineObjectSize)) + CancelPipeline(nsHttpConnectionMgr::BadUnexpectedLarge); + // handle chunked encoding here, so we'll know immediately when // we're done with the socket. please note that _all_ other // decoding is done when the channel receives the content data @@ -1520,9 +1702,20 @@ nsHttpTransaction::HandleContentStart() else if (mContentLength == int64_t(-1)) LOG(("waiting for the server to close the connection.\n")); } + if (mRestartInProgressVerifier.IsSetup() && + !mRestartInProgressVerifier.Verify(mContentLength, mResponseHead)) { + LOG(("Restart in progress subsequent transaction failed to match")); + return NS_ERROR_ABORT; + } } mDidContentStart = true; + + // The verifier only initializes itself once (from the first iteration of + // a transaction that gets far enough to have response headers) + if (mRequestHead->IsGet()) + mRestartInProgressVerifier.Set(mContentLength, mResponseHead); + return NS_OK; } @@ -1583,6 +1776,21 @@ nsHttpTransaction::HandleContent(char *buf, *contentRead = count; } + int64_t toReadBeforeRestart = + mRestartInProgressVerifier.ToReadBeforeRestart(); + + if (toReadBeforeRestart && *contentRead) { + uint32_t ignore = + static_cast(std::min(toReadBeforeRestart, UINT32_MAX)); + ignore = std::min(*contentRead, ignore); + LOG(("Due To Restart ignoring %d of remaining %" PRId64, + ignore, toReadBeforeRestart)); + *contentRead -= ignore; + mContentRead += ignore; + mRestartInProgressVerifier.HaveReadBeforeRestart(ignore); + memmove(buf, buf + ignore, *contentRead + *contentRemaining); + } + if (*contentRead) { // update count of content bytes read and report progress... mContentRead += *contentRead; @@ -1591,6 +1799,15 @@ nsHttpTransaction::HandleContent(char *buf, LOG(("nsHttpTransaction::HandleContent [this=%p count=%u read=%u mContentRead=%" PRId64 " mContentLength=%" PRId64 "]\n", this, count, *contentRead, mContentRead, mContentLength)); + // Check the size of chunked responses. If we exceed the max pipeline size + // for this response reschedule the pipeline + if ((mClassification != CLASS_SOLO) && + mChunkedDecoder && + ((mContentRead + mChunkedDecoder->GetChunkRemaining()) > + mMaxPipelineObjectSize)) { + CancelPipeline(nsHttpConnectionMgr::BadUnexpectedLarge); + } + // check for end-of-file if ((mContentRead == mContentLength) || (mChunkedDecoder && mChunkedDecoder->ReachedEOF())) { @@ -1683,7 +1900,7 @@ nsHttpTransaction::ProcessData(char *buf, uint32_t count, uint32_t *countRead) // // count : bytes read from the socket // countRead : bytes corresponding to this transaction - // countRemaining : bytes corresponding to next transaction on conn + // countRemaining : bytes corresponding to next pipelined transaction // // NOTE: // count > countRead + countRemaining <==> chunked transfer encoding @@ -1707,6 +1924,24 @@ nsHttpTransaction::ProcessData(char *buf, uint32_t count, uint32_t *countRead) return NS_OK; } +void +nsHttpTransaction::CancelPipeline(uint32_t reason) +{ + // reason is casted through a uint to avoid compiler header deps + gHttpHandler->ConnMgr()->PipelineFeedbackInfo( + mConnInfo, + static_cast(reason), + nullptr, mClassification); + + mConnection->CancelPipeline(NS_ERROR_ABORT); + + // Avoid pipelining this transaction on restart by classifying it as solo. + // This also prevents BadUnexpectedLarge from being reported more + // than one time per transaction. + mClassification = CLASS_SOLO; +} + + void nsHttpTransaction::SetRequestContext(nsIRequestContext *aRequestContext) { @@ -1715,8 +1950,8 @@ nsHttpTransaction::SetRequestContext(nsIRequestContext *aRequestContext) } // Called when the transaction marked for blocking is associated with a connection -// (i.e. added to a new h1 conn, an idle http connection, etc..) -// It is safe to call this multiple times with it only +// (i.e. added to a new h1 conn, an idle http connection, or placed into +// a http pipeline). It is safe to call this multiple times with it only // having an effect once. void nsHttpTransaction::DispatchedAsBlocking() @@ -2092,6 +2327,95 @@ nsHttpTransaction::OnOutputStreamReady(nsIAsyncOutputStream *out) return NS_OK; } +// nsHttpTransaction::RestartVerifier + +static bool +matchOld(nsHttpResponseHead *newHead, nsCString &old, + nsHttpAtom headerAtom) +{ + nsAutoCString val; + + newHead->GetHeader(headerAtom, val); + if (!val.IsEmpty() && old.IsEmpty()) + return false; + if (val.IsEmpty() && !old.IsEmpty()) + return false; + if (!val.IsEmpty() && !old.Equals(val)) + return false; + return true; +} + +bool +nsHttpTransaction::RestartVerifier::Verify(int64_t contentLength, + nsHttpResponseHead *newHead) +{ + if (mContentLength != contentLength) + return false; + + if (newHead->Status() != 200) + return false; + + if (!matchOld(newHead, mContentRange, nsHttp::Content_Range)) + return false; + + if (!matchOld(newHead, mLastModified, nsHttp::Last_Modified)) + return false; + + if (!matchOld(newHead, mETag, nsHttp::ETag)) + return false; + + if (!matchOld(newHead, mContentEncoding, nsHttp::Content_Encoding)) + return false; + + if (!matchOld(newHead, mTransferEncoding, nsHttp::Transfer_Encoding)) + return false; + + return true; +} + +void +nsHttpTransaction::RestartVerifier::Set(int64_t contentLength, + nsHttpResponseHead *head) +{ + if (mSetup) + return; + + // If mSetup does not transition to true RestartInPogress() is later + // forbidden + + // Only RestartInProgress with 200 response code + if (!head || (head->Status() != 200)) { + return; + } + + mContentLength = contentLength; + + nsAutoCString val; + if (NS_SUCCEEDED(head->GetHeader(nsHttp::ETag, val))) { + mETag = val; + } + if (NS_SUCCEEDED(head->GetHeader(nsHttp::Last_Modified, val))) { + mLastModified = val; + } + if (NS_SUCCEEDED(head->GetHeader(nsHttp::Content_Range, val))) { + mContentRange = val; + } + if (NS_SUCCEEDED(head->GetHeader(nsHttp::Content_Encoding, val))) { + mContentEncoding = val; + } + if (NS_SUCCEEDED(head->GetHeader(nsHttp::Transfer_Encoding, val))) { + mTransferEncoding = val; + } + + // We can only restart with any confidence if we have a stored etag or + // last-modified header + if (mETag.IsEmpty() && mLastModified.IsEmpty()) { + return; + } + + mSetup = true; +} + void nsHttpTransaction::GetNetworkAddresses(NetAddr &self, NetAddr &peer) { diff --git a/netwerk/protocol/http/nsHttpTransaction.h b/netwerk/protocol/http/nsHttpTransaction.h index 84094f3f3662..6df966294d86 100644 --- a/netwerk/protocol/http/nsHttpTransaction.h +++ b/netwerk/protocol/http/nsHttpTransaction.h @@ -119,11 +119,14 @@ public: void SetPriority(int32_t priority) { mPriority = priority; } int32_t Priority() { return mPriority; } + enum Classifier Classification() { return mClassification; } + void PrintDiagnostics(nsCString &log); // Sets mPendingTime to the current time stamp or to a null time stamp (if now is false) void SetPendingTime(bool now = true) { mPendingTime = now ? TimeStamp::Now() : TimeStamp(); } const TimeStamp GetPendingTime() { return mPendingTime; } + bool UsesPipelining() const { return mCaps & NS_HTTP_ALLOW_PIPELINING; } // overload of nsAHttpTransaction::RequestContext() nsIRequestContext *RequestContext() override { return mRequestContext.get(); } @@ -171,6 +174,7 @@ private: virtual ~nsHttpTransaction(); nsresult Restart(); + nsresult RestartInProgress(); char *LocateHttpStart(char *buf, uint32_t len, bool aAllowPartialMatch); nsresult ParseLine(nsACString &line); @@ -182,6 +186,9 @@ private: void DeleteSelfOnConsumerThread(); void ReleaseBlockingTransaction(); + Classifier Classify(); + void CancelPipeline(uint32_t reason); + static nsresult ReadRequestSegment(nsIInputStream *, void *, const char *, uint32_t, uint32_t, uint32_t *); static nsresult WritePipeSegment(nsIOutputStream *, void *, char *, @@ -272,6 +279,9 @@ private: uint16_t mRestartCount; // the number of times this transaction has been restarted uint32_t mCaps; + enum Classifier mClassification; + int32_t mPipelinePosition; + int64_t mMaxPipelineObjectSize; nsHttpVersion mHttpVersion; uint16_t mHttpResponseCode; @@ -328,6 +338,68 @@ private: // The time when the transaction was submitted to the Connection Manager TimeStamp mPendingTime; + class RestartVerifier + { + + // When a idemptotent transaction has received part of its response body + // and incurs an error it can be restarted. To do this we mark the place + // where we stopped feeding the body to the consumer and start the + // network call over again. If everything we track (headers, length, etc..) + // matches up to the place where we left off then the consumer starts being + // fed data again with the new information. This can be done N times up + // to the normal restart (i.e. with no response info) limit. + + public: + RestartVerifier() + : mContentLength(-1) + , mAlreadyProcessed(0) + , mToReadBeforeRestart(0) + , mSetup(false) + {} + ~RestartVerifier() {} + + void Set(int64_t contentLength, nsHttpResponseHead *head); + bool Verify(int64_t contentLength, nsHttpResponseHead *head); + bool IsDiscardingContent() { return mToReadBeforeRestart != 0; } + bool IsSetup() { return mSetup; } + int64_t AlreadyProcessed() { return mAlreadyProcessed; } + void SetAlreadyProcessed(int64_t val) { + mAlreadyProcessed = val; + mToReadBeforeRestart = val; + } + int64_t ToReadBeforeRestart() { return mToReadBeforeRestart; } + void HaveReadBeforeRestart(uint32_t amt) + { + MOZ_ASSERT(amt <= mToReadBeforeRestart, + "too large of a HaveReadBeforeRestart deduction"); + mToReadBeforeRestart -= amt; + } + + private: + // This is the data from the first complete response header + // used to make sure that all subsequent response headers match + + int64_t mContentLength; + nsCString mETag; + nsCString mLastModified; + nsCString mContentRange; + nsCString mContentEncoding; + nsCString mTransferEncoding; + + // This is the amount of data that has been passed to the channel + // from previous iterations of the transaction and must therefore + // be skipped in the new one. + int64_t mAlreadyProcessed; + + // The amount of data that must be discarded in the current iteration + // (where iteration > 0) to reach the mAlreadyProcessed high water + // mark. + int64_t mToReadBeforeRestart; + + // true when ::Set has been called with a response header + bool mSetup; + } mRestartInProgressVerifier; + // For Rate Pacing via an EventTokenBucket public: // called by the connection manager to run this transaction through the diff --git a/netwerk/protocol/http/nsIHttpChannel.idl b/netwerk/protocol/http/nsIHttpChannel.idl index 60c83af035d6..bb6b6c7963ef 100644 --- a/netwerk/protocol/http/nsIHttpChannel.idl +++ b/netwerk/protocol/http/nsIHttpChannel.idl @@ -206,7 +206,16 @@ interface nsIHttpChannel : nsIChannel void visitNonDefaultRequestHeaders(in nsIHttpHeaderVisitor aVisitor); /** - * This attribute no longer has any effect, it remains for backwards compat + * This attribute is a hint to the channel to indicate whether or not + * the underlying HTTP transaction should be allowed to be pipelined + * with other transactions. This should be set to FALSE, for example, + * if the application knows that the corresponding document is likely + * to be very large. + * + * This attribute is true by default, though other factors may prevent + * pipelining. + * + * This attribute may only be set before the channel is opened. * * @throws NS_ERROR_FAILURE if set after the channel has been opened. */ diff --git a/netwerk/test/unit/test_assoc.js b/netwerk/test/unit/test_assoc.js new file mode 100644 index 000000000000..ded2e3d5ab07 --- /dev/null +++ b/netwerk/test/unit/test_assoc.js @@ -0,0 +1,102 @@ +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource://gre/modules/NetUtil.jsm"); + +var httpserver = new HttpServer(); +var currentTestIndex = 0; + +XPCOMUtils.defineLazyGetter(this, "port", function() { + return httpserver.identity.primaryPort; +}); + +XPCOMUtils.defineLazyGetter(this, "tests", function() { + return [ + // this is valid + {url: "/assoc/assoctest?valid", + responseheader: ["Assoc-Req: GET http://localhost:" + port + + "/assoc/assoctest?valid", + "Pragma: X-Verify-Assoc-Req"], + flags: 0}, + + // this is invalid because the method is wrong + {url: "/assoc/assoctest?invalid", + responseheader: ["Assoc-Req: POST http://localhost:" + port + + "/assoc/assoctest?invalid", + "Pragma: X-Verify-Assoc-Req"], + flags: CL_EXPECT_LATE_FAILURE}, + + // this is invalid because the url is wrong + {url: "/assoc/assoctest?notvalid", + responseheader: ["Assoc-Req: GET http://localhost:" + port + + "/wrongpath/assoc/assoctest?notvalid", + "Pragma: X-Verify-Assoc-Req"], + flags: CL_EXPECT_LATE_FAILURE}, + + // this is invalid because the space between method and URL is missing + {url: "/assoc/assoctest?invalid2", + responseheader: ["Assoc-Req: GEThttp://localhost:" + port + + "/assoc/assoctest?invalid2", + "Pragma: X-Verify-Assoc-Req"], + flags: CL_EXPECT_LATE_FAILURE}, + ]; +}); + +var oldPrefVal; +var domBranch; + +function setupChannel(url) +{ + return NetUtil.newChannel({ + uri: "http://localhost:" + port + url, + loadUsingSystemPrincipal: true + }); +} + +function startIter() +{ + var channel = setupChannel(tests[currentTestIndex].url); + channel.asyncOpen2(new ChannelListener(completeIter, + channel, tests[currentTestIndex].flags)); +} + +function completeIter(request, data, ctx) +{ + if (++currentTestIndex < tests.length ) { + startIter(); + } else { + domBranch.setBoolPref("enforce", oldPrefVal); + httpserver.stop(do_test_finished); + } +} + +function run_test() +{ + var prefService = + Components.classes["@mozilla.org/preferences-service;1"] + .getService(Components.interfaces.nsIPrefService); + domBranch = prefService.getBranch("network.http.assoc-req."); + oldPrefVal = domBranch.getBoolPref("enforce"); + domBranch.setBoolPref("enforce", true); + + httpserver.registerPathHandler("/assoc/assoctest", handler); + httpserver.start(-1); + + startIter(); + do_test_pending(); +} + +function handler(metadata, response) +{ + var body = "thequickbrownfox"; + response.setHeader("Content-Type", "text/plain", false); + + var header = tests[currentTestIndex].responseheader; + if (header != undefined) { + for (var i = 0; i < header.length; i++) { + var splitHdr = header[i].split(": "); + response.setHeader(splitHdr[0], splitHdr[1], false); + } + } + + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} diff --git a/netwerk/test/unit/xpcshell.ini b/netwerk/test/unit/xpcshell.ini index f62d79f5b725..e380ba7dc15a 100644 --- a/netwerk/test/unit/xpcshell.ini +++ b/netwerk/test/unit/xpcshell.ini @@ -90,6 +90,7 @@ requesttimeoutfactor = 2 # Intermittent time-outs on Android, bug 1285020 requesttimeoutfactor = 2 [test_aboutblank.js] +[test_assoc.js] [test_auth_jar.js] [test_auth_proxy.js] [test_authentication.js] diff --git a/toolkit/components/telemetry/Histograms.json b/toolkit/components/telemetry/Histograms.json index afc83dc35826..9e5da65b43dd 100644 --- a/toolkit/components/telemetry/Histograms.json +++ b/toolkit/components/telemetry/Histograms.json @@ -2570,6 +2570,13 @@ "n_buckets": 100, "description": "Time from submission to dispatch of HTTP transaction (ms)" }, + "TRANSACTION_WAIT_TIME_HTTP_PIPELINES": { + "expires_in_version": "never", + "kind": "exponential", + "high": 5000, + "n_buckets": 100, + "description": "Time from submission to dispatch of HTTP with pipelines transaction (ms)" + }, "TRANSACTION_WAIT_TIME_SPDY": { "expires_in_version": "never", "kind": "exponential", diff --git a/toolkit/components/telemetry/histogram-whitelists.json b/toolkit/components/telemetry/histogram-whitelists.json index 03639b825395..8e91f9195468 100644 --- a/toolkit/components/telemetry/histogram-whitelists.json +++ b/toolkit/components/telemetry/histogram-whitelists.json @@ -666,6 +666,7 @@ "TOP_LEVEL_CONTENT_DOCUMENTS_DESTROYED", "TOUCH_ENABLED_DEVICE", "TRANSACTION_WAIT_TIME_HTTP", + "TRANSACTION_WAIT_TIME_HTTP_PIPELINES", "TRANSACTION_WAIT_TIME_SPDY", "TRANSLATED_CHARACTERS", "TRANSLATED_PAGES", @@ -1586,6 +1587,7 @@ "TOTAL_COUNT_LOW_ERRORS", "TOUCH_ENABLED_DEVICE", "TRANSACTION_WAIT_TIME_HTTP", + "TRANSACTION_WAIT_TIME_HTTP_PIPELINES", "TRANSACTION_WAIT_TIME_SPDY", "TRANSLATED_CHARACTERS", "TRANSLATED_PAGES",